diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b60247499909..b86cdb84f909 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.50.37 +current_version = 0.50.53 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? @@ -10,6 +10,4 @@ serialize = [bumpversion:file:gradle.properties] -[bumpversion:file:docs/operator-guides/upgrading-airbyte.md] - [bumpversion:file:run-ab-platform.sh] diff --git a/.devcontainer/java-connectors-generic/devcontainer.json b/.devcontainer/java-connectors-generic/devcontainer.json index b7041313bbc6..c35b8502dd77 100644 --- a/.devcontainer/java-connectors-generic/devcontainer.json +++ b/.devcontainer/java-connectors-generic/devcontainer.json @@ -1,6 +1,6 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the { - "name": "Connector Development DevContainer (Generic)", + "name": "Java Development DevContainer (Generic)", "image": "mcr.microsoft.com/devcontainers/java:0-17", "features": { diff --git a/.devcontainer/python-connectors-generic/devcontainer.json b/.devcontainer/python-connectors-generic/devcontainer.json new file mode 100644 index 000000000000..539a80499800 --- /dev/null +++ b/.devcontainer/python-connectors-generic/devcontainer.json @@ -0,0 +1,65 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +{ + "name": "Python Development DevContainer (Generic)", + + "image": "mcr.microsoft.com/devcontainers/python:0-3.10", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker": {}, + "ghcr.io/devcontainers/features/python:1": { + "installGradle": true, + "version": "3.10", + "installTools": true + }, + "ghcr.io/devcontainers-contrib/features/poetry:2": {} + }, + + // Deterministic order reduces cache busting + "overrideFeatureInstallOrder": [ + "ghcr.io/devcontainers/features/docker-in-docker", + "ghcr.io/devcontainers/features/python", + "ghcr.io/devcontainers-contrib/features/poetry" + ], + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + // Python extensions: + "charliermarsh.ruff", + "matangover.mypy", + "ms-python.python", + "ms-python.vscode-pylance", + + // Toml support + "tamasfe.even-better-toml", + + // Yaml and JSON Schema support: + "redhat.vscode-yaml", + + // Contributing: + "GitHub.vscode-pull-request-github" + ], + "settings": { + "extensions.ignoreRecommendations": true, + "git.openRepositoryInParentFolders": "always" + } + } + }, + + // Mark the root directory as 'safe' for git. + "initializeCommand": "git config --add safe.directory /workspaces/airbyte", + + // Setup airbyte-ci on the container: + "postCreateCommand": "make tools.airbyte-ci-dev.install", + + "containerEnv": { + // Deterministic Poetry virtual env location: `./.venv` + "POETRY_VIRTUALENVS_IN_PROJECT": "true" + } + + // Override to change the directory that the IDE opens by default: + // "workspaceFolder": "/workspaces/airbyte" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1c76017e418d..b3b9368f31d4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,7 +9,7 @@ # CDK and Connector Acceptance Tests /airbyte-cdk/python @airbytehq/connector-extensibility /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility -/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-operations @lazebnyi @oustynova +/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova # Protocol related items /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers @@ -62,7 +62,7 @@ airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations /airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations # Build customization file change -/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations +/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility # airbyte-ci -/airbyte-ci @airbytehq/connector-operations +/airbyte-ci @airbytehq/connector-extensibility diff --git a/.github/actions/airbyte-ci-requirements/action.yml b/.github/actions/airbyte-ci-requirements/action.yml new file mode 100644 index 000000000000..cb3ae4688c48 --- /dev/null +++ b/.github/actions/airbyte-ci-requirements/action.yml @@ -0,0 +1,104 @@ +name: "Get airbyte-ci runner name" +description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command" +inputs: + runner_type: + description: "Type of runner to get requirements for. One of: format, test, nightly, publish" + required: true + runner_size: + description: "One of: format, test, nightly, publish" + required: true + airbyte_ci_command: + description: "airbyte-ci command to get CI requirements for." + required: true + runner_name_prefix: + description: "Prefix of runner name" + required: false + default: ci-runner-connector + github_token: + description: "GitHub token" + required: true + sentry_dsn: + description: "Sentry DSN" + required: false + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + +runs: + using: "composite" + steps: + - name: Check if PR is from a fork + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "PR is from a fork. Exiting workflow..." + exit 78 + fi + + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + pipelines: + - 'airbyte-ci/connectors/pipelines/**' + + - name: Determine how Airbyte CI should be installed + shell: bash + id: determine-install-mode + run: | + if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + echo "install-mode=dev" >> $GITHUB_OUTPUT + else + echo "install-mode=production" >> $GITHUB_OUTPUT + fi + + - name: Install airbyte-ci binary + id: install-airbyte-ci + if: steps.determine-install-mode.outputs.install-mode == 'production' + shell: bash + run: | + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + if: steps.determine-install-mode.outputs.install-mode == 'dev' + with: + python-version: "3.10" + token: ${{ inputs.github_token }} + + - name: Install ci-connector-ops package + if: steps.determine-install-mode.outputs.install-mode == 'dev' + shell: bash + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/pipelines/ + + - name: Get dagger version from airbyte-ci + id: get-dagger-version + shell: bash + run: | + dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version') + echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT" + + - name: Get runner name + id: get-runner-name + shell: bash + run: | + runner_name_prefix=${{ inputs.runner_name_prefix }} + runner_type=${{ inputs.runner_type }} + runner_size=${{ inputs.runner_size }} + dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-') + runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}" + echo ${runner_name} + echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT" +outputs: + runner_name: + description: "Name of self hosted CI runner to use" + value: ${{ steps.get-runner-name.outputs.runner_name }} diff --git a/.github/actions/get-dagger-engine-image/action.yml b/.github/actions/get-dagger-engine-image/action.yml new file mode 100644 index 000000000000..7f03269b4bc8 --- /dev/null +++ b/.github/actions/get-dagger-engine-image/action.yml @@ -0,0 +1,53 @@ +name: "Get Dagger Engine Image" +description: "Pulls the Dagger Engine Image or load from cache" + +inputs: + dagger_engine_image: + description: "Image name of the Dagger Engine" + required: true + path_to_dagger_engine_image_cache: + description: "Path to the Dagger Engine image cache" + required: false + default: "/home/runner/dagger-engine-image-cache" + +runs: + using: "composite" + steps: + - name: Create local image cache directory + id: create-dagger-engine-image-cache-dir + shell: bash + run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }} + + - name: Restore dagger engine image cache + id: dagger-engine-image-cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.path_to_dagger_engine_image_cache }} + key: ${{ inputs.dagger_engine_image }} + + # If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory + - name: Pull dagger engine image + id: pull-dagger-engine-image + if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true' + shell: bash + run: | + set -x + docker pull ${{ inputs.dagger_engine_image }} + docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }} + + # If no GitHub Action cache hit, save the path to the image cache directory to the Github Action cache + - name: Save dagger engine image cache + id: dagger-engine-image-cache-save + if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ inputs.path_to_dagger_engine_image_cache }} + key: ${{ inputs.dagger_engine_image }} + + # If GitHub Action cache hit, load the image tar restored from the cache + - name: Load dagger engine image from cache + if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true' + shell: bash + run: | + set -x + docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml new file mode 100644 index 000000000000..4c1e0bf10ec5 --- /dev/null +++ b/.github/actions/install-airbyte-ci/action.yml @@ -0,0 +1,81 @@ +name: "Install Airbyte CI" +description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci." + +inputs: + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + path_to_airbyte_ci_source: + description: "Path to airbyte-ci source" + required: false + default: airbyte-ci/connectors/pipelines +runs: + using: "composite" + steps: + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + pipelines: + - '${{ inputs.path_to_airbyte_ci_source }}/**' + + - name: Determine how Airbyte CI should be installed + shell: bash + id: determine-install-mode + run: | + if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + echo "install-mode=source" >> $GITHUB_OUTPUT + else + echo "install-mode=binary" >> $GITHUB_OUTPUT + fi + + - name: Install Airbyte CI from binary + id: install-airbyte-ci-binary + if: steps.determine-install-mode.outputs.install-mode == 'binary' + shell: bash + run: | + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci + + - name: Install Python 3.10 + id: install-python-3-10 + uses: actions/setup-python@v4 + if: steps.determine-install-mode.outputs.install-mode == 'source' + with: + python-version: "3.10" + token: ${{ inputs.github_token }} + + - name: Install Airbyte CI from source + id: install-airbyte-ci-source + if: steps.determine-install-mode.outputs.install-mode == 'source' + shell: bash + run: | + pip install --upgrade pip + pip install pipx + pipx ensurepath + pipx install ${{ inputs.path_to_airbyte_ci_source }} + + - name: Get dagger engine image name + id: get-dagger-engine-image-name + shell: bash + run: | + dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image') + echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT" + + - name: Get dagger engine image + id: get-dagger-engine-image + uses: ./.github/actions/get-dagger-engine-image + with: + dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }} + +outputs: + install_mode: + description: "Whether Airbyte CI was installed from source or from a binary" + value: ${{ steps.determine-install-mode.outputs.install-mode }} + dagger_engine_image_name: + description: "Dagger engine image name" + value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }} diff --git a/.github/actions/run-airbyte-ci/action.yml b/.github/actions/run-airbyte-ci/action.yml new file mode 100644 index 000000000000..87d8b6c8f787 --- /dev/null +++ b/.github/actions/run-airbyte-ci/action.yml @@ -0,0 +1,152 @@ +name: "Run Dagger pipeline" +description: "Runs a given dagger pipeline" +inputs: + subcommand: + description: "Subcommand for airbyte-ci" + required: true + context: + description: "CI context (e.g., pull_request, manual)" + required: true + github_token: + description: "GitHub token" + required: true + dagger_cloud_token: + description: "Dagger Cloud token" + required: true + docker_hub_username: + description: "Dockerhub username" + required: true + docker_hub_password: + description: "Dockerhub password" + required: true + options: + description: "Options for the subcommand" + required: false + production: + description: "Whether to run in production mode" + required: false + default: "True" + report_bucket_name: + description: "Bucket name for CI reports" + required: false + default: "airbyte-ci-reports-multi" + gcp_gsm_credentials: + description: "GCP credentials for GCP Secret Manager" + required: false + default: "" + git_branch: + description: "Git branch to checkout" + required: false + git_revision: + description: "Git revision to checkout" + required: false + slack_webhook_url: + description: "Slack webhook URL" + required: false + metadata_service_gcs_credentials: + description: "GCP credentials for metadata service" + required: false + metadata_service_bucket_name: + description: "Bucket name for metadata service" + required: false + default: "prod-airbyte-cloud-connector-metadata-service" + sentry_dsn: + description: "Sentry DSN" + required: false + spec_cache_bucket_name: + description: "Bucket name for GCS spec cache" + required: false + default: "io-airbyte-cloud-spec-cache" + spec_cache_gcs_credentials: + description: "GCP credentials for GCS spec cache" + required: false + gcs_credentials: + description: "GCP credentials for GCS" + required: false + ci_job_key: + description: "CI job key" + required: false + s3_build_cache_access_key_id: + description: "Gradle S3 Build Cache AWS access key ID" + required: false + s3_build_cache_secret_key: + description: "Gradle S3 Build Cache AWS secret key" + required: false + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + python_registry_token: + description: "Python registry API token to publish python package" + required: false + +runs: + using: "composite" + steps: + - name: Get start timestamp + id: get-start-timestamp + shell: bash + run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT + + - name: Check if PR is from a fork + id: check-if-pr-is-from-fork + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "PR is from a fork. Exiting workflow..." + exit 78 + fi + + - name: Docker login + id: docker-login + uses: docker/login-action@v3 + with: + username: ${{ inputs.docker_hub_username }} + password: ${{ inputs.docker_hub_password }} + + - name: Install Airbyte CI + id: install-airbyte-ci + uses: ./.github/actions/install-airbyte-ci + with: + airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url }} + + - name: Run airbyte-ci + id: run-airbyte-ci + shell: bash + run: | + airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} + env: + CI_CONTEXT: "${{ inputs.context }}" + CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }} + CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }} + CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }} + CI_JOB_KEY: ${{ inputs.ci_job_key }} + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }} + CI: "True" + DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}" + DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }} + DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }} + GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }} + GCS_CREDENTIALS: ${{ inputs.gcs_credentials }} + METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }} + METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }} + PRODUCTION: ${{ inputs.production }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }} + PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }} + PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }} + S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }} + SENTRY_DSN: ${{ inputs.sentry_dsn }} + SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }} + SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }} + SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }} + SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }} + # give the Dagger Engine more time to push cache data to Dagger Cloud + - name: Stop Engine + id: stop-engine + if: always() + shell: bash + run: docker stop --time 300 $(docker ps --filter name="dagger-engine-*" -q) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml deleted file mode 100644 index 2847f3e90726..000000000000 --- a/.github/actions/run-dagger-pipeline/action.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: "Run Dagger pipeline" -description: "Runs a given dagger pipeline" -inputs: - subcommand: - description: "Subcommand for airbyte-ci" - required: true - context: - description: "CI context (e.g., pull_request, manual)" - required: true - github_token: - description: "GitHub token" - required: true - docker_hub_username: - description: "Dockerhub username" - required: true - docker_hub_password: - description: "Dockerhub password" - required: true - docker_registry_mirror_url: - description: "Docker registry mirror URL (not including http or https)" - required: false - # Do not use http or https here - default: "ci-dockerhub-registry.airbyte.com" - options: - description: "Options for the subcommand" - required: false - production: - description: "Whether to run in production mode" - required: false - default: "True" - report_bucket_name: - description: "Bucket name for CI reports" - required: false - default: "airbyte-ci-reports-multi" - gcp_gsm_credentials: - description: "GCP credentials for GCP Secret Manager" - required: false - default: "" - git_branch: - description: "Git branch to checkout" - required: false - git_revision: - description: "Git revision to checkout" - required: false - slack_webhook_url: - description: "Slack webhook URL" - required: false - metadata_service_gcs_credentials: - description: "GCP credentials for metadata service" - required: false - metadata_service_bucket_name: - description: "Bucket name for metadata service" - required: false - default: "prod-airbyte-cloud-connector-metadata-service" - sentry_dsn: - description: "Sentry DSN" - required: false - spec_cache_bucket_name: - description: "Bucket name for GCS spec cache" - required: false - default: "io-airbyte-cloud-spec-cache" - spec_cache_gcs_credentials: - description: "GCP credentials for GCS spec cache" - required: false - gcs_credentials: - description: "GCP credentials for GCS" - required: false - ci_job_key: - description: "CI job key" - required: false - s3_build_cache_access_key_id: - description: "Gradle S3 Build Cache AWS access key ID" - required: false - s3_build_cache_secret_key: - description: "Gradle S3 Build Cache AWS secret key" - required: false - tailscale_auth_key: - description: "Tailscale auth key" - airbyte_ci_binary_url: - description: "URL to airbyte-ci binary" - required: false - default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci - -runs: - using: "composite" - steps: - - name: Check if PR is from a fork - if: github.event_name == 'pull_request' - shell: bash - run: | - if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then - echo "PR is from a fork. Exiting workflow..." - exit 78 - fi - - name: Docker login - uses: docker/login-action@v1 - with: - username: ${{ inputs.docker_hub_username }} - password: ${{ inputs.docker_hub_password }} - - name: Get start timestamp - id: get-start-timestamp - shell: bash - run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT - - name: Install airbyte-ci binary - id: install-airbyte-ci - shell: bash - run: | - curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin - sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci - sudo chmod +x /usr/local/bin/airbyte-ci - - name: Run airbyte-ci - shell: bash - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - CI_CONTEXT: "${{ inputs.context }}" - CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }} - CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }} - CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }} - CI_JOB_KEY: ${{ inputs.ci_job_key }} - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }} - GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }} - GCS_CREDENTIALS: ${{ inputs.gcs_credentials }} - METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }} - METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }} - PRODUCTION: ${{ inputs.production }} - PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - SENTRY_DSN: ${{ inputs.sentry_dsn }} - SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }} - SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }} - SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }} - DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }} - DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }} - S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }} - S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }} - CI: "True" - TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }} - DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }} diff --git a/.github/actions/runner-prepare-for-build/action.yml b/.github/actions/runner-prepare-for-build/action.yml index cb5a890a968a..15c85386aa8e 100644 --- a/.github/actions/runner-prepare-for-build/action.yml +++ b/.github/actions/runner-prepare-for-build/action.yml @@ -20,7 +20,7 @@ runs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - if: inputs.install_node == 'true' uses: actions/setup-node@v3 diff --git a/.github/labeler.yml b/.github/labeler.yml index 09aed0435b97..23e0950d448a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -11,10 +11,6 @@ area/documentation: - docs/* - docs/**/* -area/octavia-cli: - - octavia-cli/* - - octavia-cli/**/* - CDK: - airbyte-cdk/* - airbyte-cdk/**/* diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index ed450ee04b95..37240d132791 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -1,4 +1,4 @@ -name: Connector Ops CI - Pipeline Unit Test +name: Internal Poetry packages CI concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -7,19 +7,51 @@ concurrency: on: workflow_dispatch: inputs: - airbyte_ci_binary_url: - description: "URL to airbyte-ci binary" - required: false - default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + airbyte_ci_subcommand: + description: "Subcommand to pass to the 'airbyte-ci test' command" + default: "--poetry-package-path=airbyte-ci/connectors/pipelines" pull_request: types: - opened - reopened - synchronize jobs: - run-airbyte-ci-tests: + changes: + runs-on: ubuntu-latest + outputs: + internal_poetry_packages: ${{ steps.changes.outputs.internal_poetry_packages }} + + steps: + - name: Checkout Airbyte + if: github.event_name != 'pull_request' + uses: actions/checkout@v3 + - id: changes + uses: dorny/paths-filter@v2 + with: + # Note: expressions within a filter are OR'ed + filters: | + internal_poetry_packages: + - airbyte-lib/** + - airbyte-ci/connectors/pipelines/** + - airbyte-ci/connectors/base_images/** + - airbyte-ci/connectors/common_utils/** + - airbyte-ci/connectors/connector_ops/** + - airbyte-ci/connectors/connectors_qa/** + - airbyte-ci/connectors/ci_credentials/** + - airbyte-ci/connectors/metadata_service/lib/** + - airbyte-ci/connectors/metadata_service/orchestrator/** + - airbyte-integrations/bases/connector-acceptance-test/** + + run-tests: + needs: changes + if: needs.changes.outputs.internal_poetry_packages == 'true' + #name: Internal Poetry packages CI + # To rename in a follow up PR name: Run Airbyte CI tests - runs-on: "ci-runner-connector-test-large-dagger-0-6-4" + runs-on: tooling-test-large + permissions: + pull-requests: read + statuses: write steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -27,108 +59,50 @@ jobs: fetch-depth: 0 ref: ${{ github.event.pull_request.head.ref }} - # IMPORTANT! This is nessesary to make sure that a status is reported on the PR - # even if the workflow is skipped. If we used github actions filters, the workflow - # would not be reported as skipped, but instead would be forever pending. - # - # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. - # - # Also it gets worse - # - # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. - # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. - - name: Get changed files - uses: tj-actions/changed-files@v39 - id: changes - with: - files_yaml: | - ops: - - 'airbyte-ci/connectors/connector_ops/**' - - '!**/*.md' - base_images: - - 'airbyte-ci/connectors/connector_ops/**' - - 'airbyte-ci/connectors/base_images/**' - - '!**/*.md' - pipelines: - - 'airbyte-ci/connectors/connector_ops/**' - - 'airbyte-ci/connectors/base_images/**' - - 'airbyte-ci/connectors/pipelines/**' - - '!**/*.md' - metadata_lib: - - 'airbyte-ci/connectors/metadata_service/lib/**' - - '!**/*.md' - metadata_orchestrator: - - 'airbyte-ci/connectors/metadata_service/lib/**' - - 'airbyte-ci/connectors/metadata_service/orchestrator/**' - - '!**/*.md' + - name: Extract branch name [WORKFLOW DISPATCH] + shell: bash + if: github.event_name == 'workflow_dispatch' + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Fetch last commit id from remote branch [PULL REQUESTS] + if: github.event_name == 'pull_request' + id: fetch_last_commit_id_pr + run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT + - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + id: fetch_last_commit_id_wd + run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT - - name: Run airbyte-ci/connectors/connector_ops tests - if: steps.changes.outputs.ops_any_changed == 'true' - id: run-airbyte-ci-connectors-connector-ops-tests - uses: ./.github/actions/run-dagger-pipeline + - name: Run poe tasks for modified internal packages [PULL REQUEST] + if: github.event_name == 'pull_request' + id: run-airbyte-ci-test-pr + uses: ./.github/actions/run-airbyte-ci with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + git_branch: ${{ github.head_ref }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ github.token }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/connector_ops" - airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "test --modified" - - name: Run airbyte-ci/connectors/pipelines tests - id: run-airbyte-ci-connectors-pipelines-tests - if: steps.changes.outputs.pipelines_any_changed == 'true' - uses: ./.github/actions/run-dagger-pipeline + - name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH] + id: run-airbyte-ci-test-workflow-dispatch + if: github.event_name == 'workflow_dispatch' + uses: ./.github/actions/run-airbyte-ci with: - context: "pull_request" + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + git_branch: ${{ steps.extract_branch.outputs.branch }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ github.token }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/pipelines" - airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - - - name: Run airbyte-ci/connectors/base_images tests - id: run-airbyte-ci-connectors-base-images-tests - if: steps.changes.outputs.base_images_any_changed == 'true' - uses: ./.github/actions/run-dagger-pipeline - with: - context: "pull_request" - docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/base_images" - airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - - - name: Run test pipeline for the metadata lib - id: metadata_lib-test-pipeline - if: steps.changes.outputs.metadata_lib_any_changed == 'true' - uses: ./.github/actions/run-dagger-pipeline - with: - subcommand: "test airbyte-ci/connectors/metadata_service/lib/" - context: "pull_request" - github_token: ${{ secrets.GITHUB_TOKEN }} - docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - - - name: Run test for the metadata orchestrator - id: metadata_orchestrator-test-pipeline - if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' - uses: ./.github/actions/run-dagger-pipeline - with: - subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/" - context: "pull_request" - github_token: ${{ secrets.GITHUB_TOKEN }} - docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "test ${{ inputs.airbyte_ci_subcommand}}" diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml deleted file mode 100644 index 553bf1373f8d..000000000000 --- a/.github/workflows/cat-tests.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Connector Ops CI - CAT Unit Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - types: - - opened - - reopened - - synchronize - paths: - - airbyte-integrations/bases/connector-acceptance-test/** -jobs: - run-cat-unit-tests: - name: Run CAT unit tests - runs-on: "ci-runner-connector-test-large-dagger-0-6-4" - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Run CAT unit tests - id: run-cat-unit-tests - uses: ./.github/actions/run-dagger-pipeline - with: - context: "pull_request" - docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} - gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests" - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} diff --git a/.github/workflows/community_ci.yml b/.github/workflows/community_ci.yml new file mode 100644 index 000000000000..cc4c4ce313ba --- /dev/null +++ b/.github/workflows/community_ci.yml @@ -0,0 +1,129 @@ +name: Community CI Spike + +concurrency: + # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. + # + # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests + # ensuring that only one run per pull request is active at a time. + # + # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. + # This allows us to run multiple workflow dispatches in parallel. + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + +on: + workflow_dispatch: + inputs: + test-connectors-options: + description: "Options to pass to the 'airbyte-ci connectors test' command" + default: "--modified" + pull_request_target: +jobs: + determine_runner_environment: + runs-on: ubuntu-latest + name: Determine runner and environment + steps: + # Checkout is required here to: + # - fetch the local actions stored in .github/actions + # - install airbyte-ci in dev mode if the PR modified airbyte-ci + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + # This checkouts the fork + # /!\ untrusted code + # It's deemed safe as the following step is not executing code from forks + ref: ${{ github.head_ref }} + # Ensures that the git token is not persisted + # It helps prevent access to token from code executed in the workflow + persist-credentials: false + fetch-depth: 1 + + # Disabling this step for safety during the spike + # - name: Get CI runner + # id: get_ci_runner + # uses: ./.github/actions/airbyte-ci-requirements + # with: + # runner_type: "test" + # runner_size: "large" + # airbyte_ci_command: "connectors test" + # is_fork: ${{ github.event.pull_request.head.repo.fork }} + + # We set the environment to community-ci if the PR is from a fork + # The community-ci environment requires manual reviewer approval to run + # This is a safety measure to prevent untrusted code from running on our infrastructure + # The internal-ci environment is reserved for internal PRs (non-forked PRs) + - name: Determine environment + id: determine_environment + if: github.event_name == 'pull_request_target' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "environment=community-ci" >> $GITHUB_OUTPUT + else + echo "environment=internal-ci" >> $GITHUB_OUTPUT + fi + outputs: + environment: ${{ steps.get_ci_runner.outputs.environment }} + runner_name: ci-runner-connector-test-large-dagger-0-9-6 + #runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} + + connectors_ci: + name: Connectors CI + needs: determine_runner_environment + environment: ${{ needs.determine_runner_environment.outputs.environment }} + runs-on: ${{ needs.determine_runner_environment.outputs.runner_name }} + timeout-minutes: 1440 # 24 hours + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + # This can checkouts forks + # /!\ untrusted code + # It's deemed safe as the community-ci environment requires manual reviewer approval to run + ref: ${{ github.head_ref }} + fetch-depth: 1 + - name: Extract branch name [WORKFLOW DISPATCH] + shell: bash + if: github.event_name == 'workflow_dispatch' + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Fetch last commit id from remote branch [PULL REQUESTS] + if: github.event_name == 'pull_request_target' + id: fetch_last_commit_id_pr + run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT + - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + id: fetch_last_commit_id_wd + run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT + - name: Test connectors [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + uses: ./.github/actions/run-airbyte-ci + with: + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ steps.extract_branch.outputs.branch }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ env.PAT }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" + - name: Test connectors [PULL REQUESTS] + if: github.event_name == 'pull_request_target' + uses: ./.github/actions/run-airbyte-ci + with: + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ github.head_ref }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ env.PAT }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + subcommand: "connectors --modified test" + is_fork: ${{ github.event.pull_request.head.repo.fork }} diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 3ed30a4ceb77..3679b848ded9 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -159,7 +159,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/connector-performance-cron.yml b/.github/workflows/connector-performance-cron.yml index a097e1674a68..fc50d6b4ee4f 100644 --- a/.github/workflows/connector-performance-cron.yml +++ b/.github/workflows/connector-performance-cron.yml @@ -3,7 +3,7 @@ on: schedule: # * is a special character in YAML so you have to quote this string - # Twice a week, Monday and Thursday. - - cron: "* * * * 1,4" + - cron: "0 0 * * 1,4" workflow_dispatch: # for manual triggers jobs: diff --git a/.github/workflows/connector_checklist.yml b/.github/workflows/connector_checklist.yml deleted file mode 100644 index 127ac5ae521d..000000000000 --- a/.github/workflows/connector_checklist.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Add Connector Merge Checklist -on: - pull_request_target: - types: [opened, reopened] - paths: - - "airbyte-integrations/connectors/source-**" - - "airbyte-integrations/connectors/destination-**" - - "airbyte-integrations/connectors/third-party/**" -jobs: - checklist_job: - name: Add Connector Merge Checklist Job - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Checklist - uses: wyozi/contextual-qa-checklist-action@master - with: - comment-header: "### Before Merging a Connector Pull Request \n\n Wow! What a great pull request you have here! 🎉 \n\n To merge this PR, ensure the following has been done/considered for each connector added or updated: \n\n" - comment-footer: "If the checklist is complete, but the CI check is failing, \n\n1. Check for hidden checklists in your PR description \n\n2. Toggle the github label `checklist-action-run` on/off to re-run the checklist CI." - show-paths: false - input-file: airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml - gh-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/connector_checklist_require.yml b/.github/workflows/connector_checklist_require.yml deleted file mode 100644 index 073cc69842c7..000000000000 --- a/.github/workflows/connector_checklist_require.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Require Connector Checklist -on: - pull_request: - types: - [ - opened, - edited, - synchronize, - labeled, - unlabeled, - reopened, - ready_for_review, - ] - paths: - - "airbyte-integrations/connectors/source-**" - - "airbyte-integrations/connectors/destination-**" - - "airbyte-integrations/connectors/third-party/**" -jobs: - check_for_required: - name: Require Connector Merge Checklist Job - runs-on: ubuntu-latest - steps: - - name: Ensure All Checklist Checked - uses: mheap/require-checklist-action@v2 - with: - requireChecklist: false # TODO (ben) reenable in one week once pull request templates have been updated - - name: Send Error Message - if: failure() - run: | - echo "::error::All checklist items not checked. Review your PR description and comments for unchecked items." - exit 1 diff --git a/.github/workflows/connector_code_freeze.yml b/.github/workflows/connector_code_freeze.yml new file mode 100644 index 000000000000..a5ac4c8a3661 --- /dev/null +++ b/.github/workflows/connector_code_freeze.yml @@ -0,0 +1,79 @@ +# This workflow is meant to be used to prevent/discourage merging to master during code freeze. +# The code freeze dates are set in the env variables CODE_FREEZE_START_DATE and CODE_FREEZE_END_DATE. +# If any connector code has been changed we display a warning message reminding merging is blocked and who to contact. +# If no connector connector code has been changed we only display a warning message reminding merging is discouraged. +# The Code freeze check job will be set as a required check for PRs in branch protection rules. + +name: Code freeze + +on: + pull_request: + types: + - opened + - synchronize + - ready_for_review + +env: + CODE_FREEZE_START_DATE: "2023-12-21" + CODE_FREEZE_END_DATE: "2024-01-02" +jobs: + code-freeze-check: + runs-on: ubuntu-latest + name: Code freeze check + permissions: + # This is required to be able to comment on PRs and list changed files + pull-requests: write + + steps: + # Check if code freeze is in effect by comparing the current date with the start and end date of the code freeze + - name: Check code freeze in effect + id: check-code-freeze-in-effect + run: | + start_date=$(date -d "$CODE_FREEZE_START_DATE" +%s) + end_date=$(date -d "$CODE_FREEZE_END_DATE" +%s) + current_date=$(date +%s) + + if [ "$current_date" -ge "$start_date" ] && [ "$current_date" -le "$end_date" ]; then + echo "Code freeze is in effect" + echo "::set-output name=is_in_code_freeze::true" + else + echo "Code freeze is not in effect" + echo "::set-output name=is_in_code_freeze::false" + fi + + # Use GitHub PR Api to get the list of changed files + # Filter the list to only keep the connectors files + - name: Get changed files + if: steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + id: changed-files + uses: tj-actions/changed-files@v40 + with: + files_yaml: | + connectors: + - 'airbyte-integrations/connectors/**' + - '!**/*.md' + + # If any connector code has been changed we display a warning message reminding merging is blocked and who to contact + - name: Code freeze comment on PR + if: steps.changed-files.outputs.connectors_any_changed == 'true' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + uses: thollander/actions-comment-pull-request@v2 + with: + comment_tag: code_freeze_warning + message: | + > [!WARNING] + > 🚨 Connector code freeze is in effect until ${{ env.CODE_FREEZE_END_DATE }}. This PR is changing connector code. Please contact the current OC engineers if you want to merge this change to master. + + # If no connector code has been changed we only display a warning message reminding merging is discouraged + - name: Code freeze comment on PR + if: steps.changed-files.outputs.connectors_any_changed == 'false' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + uses: thollander/actions-comment-pull-request@v2 + with: + comment_tag: code_freeze_warning + message: | + > [!WARNING] + > Soft code freeze is in effect until ${{ env.CODE_FREEZE_END_DATE }}. Please avoid merging to master. #freedom-and-responsibility + + # Fail the workflow if connector code has been changed to prevent merging to master + - name: Fail workflow if connector code has been changed + if: steps.changed-files.outputs.connectors_any_changed == 'true' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + run: echo "Connector code freeze is in effect. Please contact the current OC engineers if you want to merge this change." && exit 1 diff --git a/.github/workflows/connector_metadata_checks.yml b/.github/workflows/connector_metadata_checks.yml deleted file mode 100644 index 2af441a2bbe9..000000000000 --- a/.github/workflows/connector_metadata_checks.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Connector Ops CI - Connector Metadata Checks - -on: - pull_request: - paths: - - "airbyte-integrations/connectors/source-**" -jobs: - connector-metadata-checks: - name: "Check Connector Metadata" - runs-on: ubuntu-latest - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install ci-connector-ops package - run: | - pip install pipx - pipx ensurepath - pipx install airbyte-ci/connectors/connector_ops/ - - name: Check test strictness level - run: check-test-strictness-level - - name: Check allowed hosts - run: allowed-hosts-checks diff --git a/.github/workflows/connector_teams_review_requirements.yml b/.github/workflows/connector_teams_review_requirements.yml index 3964f748955e..206e7e46e91d 100644 --- a/.github/workflows/connector_teams_review_requirements.yml +++ b/.github/workflows/connector_teams_review_requirements.yml @@ -9,9 +9,11 @@ on: - synchronize paths: - "airbyte-integrations/connectors/source-**" + - "airbyte-integrations/connectors/destination-**" pull_request_review: paths: - "airbyte-integrations/connectors/source-**" + - "airbyte-integrations/connectors/destination-**" jobs: check-review-requirements: name: "Check if a review is required from Connector teams" diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml index a47efb2fe43a..e39b687c7158 100644 --- a/.github/workflows/connectors_nightly_build.yml +++ b/.github/workflows/connectors_nightly_build.yml @@ -6,21 +6,17 @@ on: - cron: "0 0 * * *" workflow_dispatch: inputs: - runs-on: - type: string - default: ci-runner-connector-nightly-xlarge-dagger-0-6-4 - required: true test-connectors-options: default: --concurrency=5 --support-level=certified required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" jobs: test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" timeout-minutes: 720 # 12 hours - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }} + runs-on: connector-nightly-xlarge steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -32,10 +28,11 @@ jobs: run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT id: extract_branch - name: Test connectors - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "master" ci_job_key: "nightly_builds" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 5ecb61ea6b1b..8cd907140cab 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -17,23 +17,53 @@ on: test-connectors-options: description: "Options to pass to the 'airbyte-ci connectors test' command" default: "--modified" - runner: - description: "The runner to use for this job" - default: "ci-runner-connector-test-large-dagger-0-6-4" - airbyte_ci_binary_url: - description: "The URL to download the airbyte-ci binary from" - required: false - default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci pull_request: types: - opened - synchronize - - ready_for_review jobs: + changes: + runs-on: ubuntu-latest + outputs: + connectors: ${{ steps.changes.outputs.connectors }} + permissions: + statuses: write + steps: + - name: Checkout Airbyte + if: github.event_name != 'pull_request' + uses: actions/checkout@v3 + - id: changes + uses: dorny/paths-filter@v2 + with: + # Note: expressions within a filter are OR'ed + filters: | + connectors: + - '*' + - 'airbyte-ci/**/*' + - 'airbyte-integrations/connectors/**/*' + - 'airbyte-cdk/**/*' + - 'buildSrc/**/*' + # The Connector CI Tests is a status check emitted by airbyte-ci + # We make it pass once we have determined that there are no changes to the connectors + - name: "Skip Connectors CI tests" + if: steps.changes.outputs.connectors != 'true' && github.event_name == 'pull_request' + run: | + curl --request POST \ + --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' \ + --data '{ + "state": "success", + "context": "Connectors CI tests", + "target_url": "${{ github.event.workflow_run.html_url }}" + }' \ + connectors_ci: + needs: changes + if: needs.changes.outputs.connectors == 'true' name: Connectors CI + runs-on: connector-test-large timeout-minutes: 1440 # 24 hours - runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-6-4'}} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -57,9 +87,10 @@ jobs: run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT - name: Test connectors [WORKFLOW DISPATCH] if: github.event_name == 'workflow_dispatch' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -70,13 +101,12 @@ jobs: s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" - airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - name: Test connectors [PULL REQUESTS] if: github.event_name == 'pull_request' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -86,5 +116,4 @@ jobs: github_token: ${{ env.PAT }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --modified test" diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml index c7795c6139fc..aee4f83aa064 100644 --- a/.github/workflows/connectors_weekly_build.yml +++ b/.github/workflows/connectors_weekly_build.yml @@ -6,21 +6,17 @@ on: - cron: "0 12 * * 0" workflow_dispatch: inputs: - runs-on: - type: string - default: ci-runner-connector-nightly-xlarge-dagger-0-6-4 - required: true test-connectors-options: default: --concurrency=3 --support-level=community required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" jobs: test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" timeout-minutes: 8640 # 6 days - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }} + runs-on: connector-weekly-xlarge steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -32,14 +28,14 @@ jobs: run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT id: extract_branch - name: Test connectors - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "master" ci_job_key: "weekly_alpha_test" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} git_branch: ${{ steps.extract_branch.outputs.branch }} github_token: ${{ secrets.GITHUB_TOKEN }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: '--show-dagger-logs connectors ${{ inputs.test-connectors-options || ''--concurrency=3 --metadata-query="(data.ab_internal.ql > 100) & (data.ab_internal.sl < 200)"'' }} test' diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 7e6be7dc972d..2eabc9a87463 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -2,6 +2,7 @@ name: Check for formatting errors run-name: Check for formatting errors on ${{ github.ref }} on: workflow_dispatch: + push: branches: - master @@ -9,59 +10,59 @@ on: jobs: format-check: - runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" # IMPORTANT: This name must match the require check name on the branch protection settings name: "Check for formatting errors" + runs-on: ubuntu-latest steps: - name: Checkout Airbyte uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - + fetch-depth: 1 - name: Run airbyte-ci format check [MASTER] id: airbyte_ci_format_check_all_master if: github.ref == 'refs/heads/master' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci continue-on-error: true with: context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "format check all" - name: Run airbyte-ci format check [PULL REQUEST] id: airbyte_ci_format_check_all_pr if: github.event_name == 'pull_request' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci continue-on-error: false with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "format check all" - name: Run airbyte-ci format check [WORKFLOW DISPATCH] id: airbyte_ci_format_check_all_manual if: github.event_name == 'workflow_dispatch' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci continue-on-error: false with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "format check all" - name: Match GitHub User to Slack User [MASTER] diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml index 22e39d345c88..547b2ef797d3 100644 --- a/.github/workflows/format_fix.yml +++ b/.github/workflows/format_fix.yml @@ -10,8 +10,8 @@ on: workflow_dispatch: jobs: format-fix: - runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" name: "Run airbyte-ci format fix all" + runs-on: ubuntu-latest steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -22,16 +22,16 @@ jobs: token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - name: Run airbyte-ci format fix all - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci continue-on-error: true with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "format fix all" # This is helpful in the case that we change a previously committed generated file to be ignored by git. diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index c6de5ead2257..45cfc134b28a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1,4 +1,4 @@ -name: Airbyte CI - Repository Health Check +name: Connector Ops CI - Gradle Check concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -20,75 +20,46 @@ on: - synchronize jobs: - # In case of self-hosted EC2 errors, remove this block. - start-check-runner: - name: Start EC2 Runner - timeout-minutes: 10 + changes: runs-on: ubuntu-latest outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + java: ${{ steps.changes.outputs.java }} + steps: - name: Checkout Airbyte + if: github.event_name != 'pull_request' uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + - id: changes + uses: dorny/paths-filter@v2 with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - # Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost. - # When gradle runs on this instance, it will use up all the available cores anyway. - # There should be little to no difference in total cost, however the job latency will be improved. - # At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes). - # This is largely thanks to the gradle cache. - ec2-instance-type: "c5.4xlarge" - github-token: ${{ env.PAT }} + # Note: expressions within a filter are OR'ed + filters: | + java: + - '**/*.java' + - '**/*.gradle' + - 'airbyte-cdk/java/**/*' run-check: - # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-check-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner + needs: + - changes + if: needs.changes.outputs.java == 'true' + # The gradle check task which we will run is embarrassingly parallelizable. + # We therefore run this on a machine with a maximum number of cores. + # We pay per time and per core, so there should be little difference in total cost. + # The latency overhead of setting up gradle prior to running the actual task adds up to about a minute. + runs-on: connector-test-xxlarge name: Gradle Check timeout-minutes: 30 steps: - name: Checkout Airbyte uses: actions/checkout@v3 - - # IMPORTANT! This is nessesary to make sure that a status is reported on the PR - # even if the workflow is skipped. If we used github actions filters, the workflow - # would not be reported as skipped, but instead would be forever pending. - # - # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. - # - # Also it gets worse - # - # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. - # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. - - name: Get changed files - uses: tj-actions/changed-files@v39 - id: changes - with: - files_yaml: | - gradlecheck: - - '**/*' - - '!**/*.md' - - '!.github/*' - - uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Pip - if: steps.changes.outputs.gradlecheck_any_changed == 'true' run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3 - name: Install Pyenv - if: steps.changes.outputs.gradlecheck_any_changed == 'true' run: python3 -m pip install virtualenv --user - name: Docker login # Some tests use testcontainers which pull images from DockerHub. @@ -97,47 +68,16 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Run Gradle Check - if: steps.changes.outputs.gradlecheck_any_changed == 'true' uses: burrunan/gradle-cache-action@v1 env: CI: true with: + job-id: gradle-check read-only: ${{ github.ref != 'refs/heads/master' }} + gradle-distribution-sha-256-sum-warning: false + concurrent: true # TODO: be able to remove the skipSlowTests property - arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true - - # In case of self-hosted EC2 errors, remove this block. - stop-check-runner: - name: Stop EC2 Runner - timeout-minutes: 10 - needs: - - start-check-runner # required to get output from the start-runner job - - run-check # required to wait when the main job is done - runs-on: ubuntu-latest - # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. - # Thus, we check for skipped here. - if: ${{ always() && needs.start-check-runner.result != 'skipped'}} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-check-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }} + arguments: --scan check -DskipSlowTests=true set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml deleted file mode 100644 index 3a92e92ba462..000000000000 --- a/.github/workflows/jacoco_report.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Measure Java Test Coverage - -on: - pull_request: - branches: - - master - paths: - - "airbyte-integrations/connectors/source-postgres/**" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - name: Publish CDK to MavenLocal - run: | - ./gradlew :airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded - - - name: Run Coverage - run: | - ./gradlew :airbyte-integrations:connectors:source-postgres:jacocoTestReport - - - name: Add coverage to PR - id: jacoco - uses: madrapps/jacoco-report@v1.3 - with: - paths: ${{ github.workspace }}/airbyte-integrations/connectors/source-postgres/build/reports/jacoco/test/jacocoTestReport.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 64 - title: Coverage report for source-postgres - update-comment: true diff --git a/.github/workflows/legacy-publish-command.yml b/.github/workflows/legacy-publish-command.yml index e221f155b06c..e20bd25fd77f 100644 --- a/.github/workflows/legacy-publish-command.yml +++ b/.github/workflows/legacy-publish-command.yml @@ -244,7 +244,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/legacy-test-command.yml b/.github/workflows/legacy-test-command.yml index f71339a21e2f..3d530b81e143 100644 --- a/.github/workflows/legacy-test-command.yml +++ b/.github/workflows/legacy-test-command.yml @@ -92,7 +92,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Python uses: actions/setup-python@v4 with: @@ -113,9 +113,6 @@ jobs: fi env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Build Java CDK Snapshot if Needed - # If a snapshot version is specified for the Java CDK, build publish locally. Otherwise, do nothing. - run: ./gradlew :airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded - name: Test ${{ github.event.inputs.connector }} id: test env: diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml index f1136d805251..12fb0487355f 100644 --- a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml +++ b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml @@ -10,20 +10,20 @@ on: jobs: connector_metadata_service_deploy_orchestrator: name: Connector metadata service deploy orchestrator - runs-on: ci-runner-connector-test-large-dagger-0-6-4 + runs-on: tooling-publish-medium steps: - name: Checkout Airbyte uses: actions/checkout@v2 - name: Deploy the metadata orchestrator id: metadata-orchestrator-deploy-orchestrator-pipeline - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: subcommand: "metadata deploy orchestrator" context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} github_token: ${{ secrets.GITHUB_TOKEN }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} env: DAGSTER_CLOUD_METADATA_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_METADATA_API_TOKEN }} diff --git a/.github/workflows/publish-airbyte-lib-command-manually.yml b/.github/workflows/publish-airbyte-lib-command-manually.yml new file mode 100644 index 000000000000..e596444414d3 --- /dev/null +++ b/.github/workflows/publish-airbyte-lib-command-manually.yml @@ -0,0 +1,57 @@ +name: Publish AirbyteLib Manually +on: workflow_dispatch + +concurrency: + group: publish-airbyte-lib + cancel-in-progress: false + +jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + # Getting ci requirements for connectors publish command as there is no special one for poetry publish + airbyte_ci_command: "connectors publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} + publish_connectors: + name: Publish airbyte-lib + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Publish + id: publish-airbyte-lib + uses: ./.github/actions/run-airbyte-ci + with: + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "poetry --package-path=airbyte-lib publish" + python_registry_token: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index a5b18a4327ec..4f206f609bab 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -66,14 +66,14 @@ jobs: - uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Checkout Airbyte uses: actions/checkout@v3 with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} - name: Build CDK Package - run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build + run: (cd airbyte-cdk/python; ./gradlew --no-daemon --no-build-cache :build) - name: Post failure to Slack channel dev-connectors-extensibility if: ${{ failure() }} uses: slackapi/slack-github-action@v1.23.0 @@ -228,7 +228,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml deleted file mode 100644 index a99757ba4ebe..000000000000 --- a/.github/workflows/publish-command.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Deprecation message for publish slash command -on: - workflow_dispatch: - inputs: - repo: - description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." - required: false - default: "airbytehq/airbyte" - gitref: - description: "The git ref to check out from the specified repository." - required: false - default: master - connector: - description: "Airbyte Connector" - required: true - comment-id: - description: "The comment-id of the slash command. Used to update the comment with the status." - required: false - parallel: - description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" - required: true - default: "false" - run-tests: - description: "Should run tests when publishing" - required: true - default: "true" - pre-release: - description: "Should publish a pre-release version" - required: true - default: "false" - -jobs: - write-deprecation-message: - name: Set up git comment - if: github.event.inputs.comment-id - runs-on: ubuntu-latest - steps: - - name: Print deprecation message - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :warning: The publish slash command is now deprecated.
- The connector publication happens on merge to the master branch.
- Please use /legacy-publish if you need to publish normalization images.
- Please join the #connector-publish-updates slack channel to track ongoing publish pipelines.
- Please reach out to the @dev-connector-ops team if you need support in publishing a connector. diff --git a/.github/workflows/publish-java-cdk-command.yml b/.github/workflows/publish-java-cdk-command.yml index 8ee3d25329c2..878f7273faf5 100644 --- a/.github/workflows/publish-java-cdk-command.yml +++ b/.github/workflows/publish-java-cdk-command.yml @@ -8,17 +8,10 @@ # /publish-java-cdk force=true # Force-publish if needing to replace an already published version name: Publish Java CDK on: - # Temporarily run on commits to the 'java-cdk/publish-workflow' branch. - # TODO: Remove this 'push' trigger before merging to master. - push: - branches: - - java-cdk/publish-workflow - workflow_dispatch: inputs: repo: description: "Repo to check out code from. Defaults to the main airbyte repo." - # TODO: If publishing from forks is needed, we'll need to revert type to `string` of `choice`. type: choice required: true default: airbytehq/airbyte @@ -30,7 +23,7 @@ on: type: boolean default: false force: - description: "Force release (ignore existing)" + description: "Force release (overwrite existing)" required: true type: boolean default: false @@ -40,9 +33,6 @@ on: comment-id: description: "Optional comment-id of the slash command. Ignore if not applicable." required: false - # uuid: - # description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id." - # required: false concurrency: group: publish-airbyte-cdk @@ -50,59 +40,32 @@ concurrency: env: # Use the provided GITREF or default to the branch triggering the workflow. - REPO: ${{ github.event.inputs.repo }} GITREF: ${{ github.event.inputs.gitref || github.ref }} FORCE: "${{ github.event.inputs.force == null && 'false' || github.event.inputs.force }}" DRY_RUN: "${{ github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}" CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties" + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} jobs: - # We are using these runners because they are the same as the one for `publish-command.yml` - # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in - # `manage.sh` (specifically `apt-get`) - start-publish-docker-image-runner-0: - name: Start Build EC2 Runner 0 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: airbytehq/airbyte - ref: master - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - label: ${{ github.run_id }}-publisher - publish-cdk: name: Publish Java CDK - needs: start-publish-docker-image-runner-0 - runs-on: ubuntu-latest + runs-on: connector-test-large + timeout-minutes: 30 steps: - - name: Link comment to workflow run + - name: Link comment to Workflow Run if: github.event.inputs.comment-id uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | > :clock2: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: ${{ env.REPO }} ref: ${{ env.GITREF }} + - name: Read Target Java CDK version id: read-target-java-cdk-version run: | @@ -112,31 +75,57 @@ jobs: exit 1 fi echo "CDK_VERSION=${cdk_version}" >> $GITHUB_ENV + - name: Setup Java uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" - - name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }}) - if: ${{ !(env.FORCE == 'true') }} - run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished + java-version: "21" + + - name: Docker login + # Some tests use testcontainers which pull images from DockerHub. + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Build Java CDK - run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build - - name: Publish Java Modules to MavenLocal (Dry-Run) - if: ${{ !(env.DRY_RUN == 'false') }} - run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publishToMavenLocal - - name: Upload jars as artifacts - if: ${{ !(env.DRY_RUN == 'false') }} - uses: actions/upload-artifact@v2 + uses: burrunan/gradle-cache-action@v1 + env: + CI: true with: - name: mavenlocal-jars - path: ~/.m2/repository/io/airbyte/ - - name: Publish Java Modules to CloudRepo + job-id: cdk-publish + read-only: ${{ !(env.DRY_RUN == 'false') }} + concurrent: true + gradle-distribution-sha-256-sum-warning: false + arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkBuild + + - name: Check for Existing Version + if: ${{ !(env.FORCE == 'true') }} + uses: burrunan/gradle-cache-action@v1 + env: + CI: true + with: + job-id: cdk-publish + read-only: true + concurrent: true + gradle-distribution-sha-256-sum-warning: false + arguments: --scan :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished + + - name: Publish Poms and Jars to CloudRepo if: ${{ env.DRY_RUN == 'false' }} - run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publish + uses: burrunan/gradle-cache-action@v1 env: + CI: true CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }} CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }} + with: + job-id: cdk-publish + read-only: true + concurrent: true + execution-only-caches: true + gradle-distribution-sha-256-sum-warning: false + arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkPublish - name: Add Success Comment if: github.event.inputs.comment-id && success() @@ -146,6 +135,7 @@ jobs: edit-mode: append body: | > :white_check_mark: Successfully published Java CDK ${{ env.CDK_VERSION }}! + - name: Add Failure Comment if: github.event.inputs.comment-id && failure() uses: peter-evans/create-or-update-comment@v1 @@ -154,7 +144,8 @@ jobs: edit-mode: append body: | > :x: Publish Java CDK ${{ env.CDK_VERSION }} failed! - - name: "Post failure to Slack channel `#dev-connectors-extensibility-releases`" + + - name: "Post failure to Slack channel" if: ${{ env.DRY_RUN == 'false' && failure() }} uses: slackapi/slack-github-action@v1.23.0 continue-on-error: true @@ -182,7 +173,8 @@ jobs: } env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - - name: "Post success to Slack channel `#dev-connectors-extensibility-releases`" + + - name: "Post success to Slack channel" if: ${{ env.DRY_RUN == 'false' && !failure() }} uses: slackapi/slack-github-action@v1.23.0 continue-on-error: true @@ -210,33 +202,3 @@ jobs: } env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - - # In case of self-hosted EC2 errors, remove this block. - stop-publish-docker-image-runner-0: - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - name: Stop Build EC2 Runner - needs: - - start-publish-docker-image-runner-0 # required to get output from the start-runner job - - publish-cdk # required to wait when the main job is done - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: airbytehq/ec2-github-runner@base64v1.1.0 - with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 3cf9b0089f72..6472cf037a6a 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -14,27 +14,23 @@ on: publish-options: description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " default: "--pre-release" - runs-on: - type: string - default: ci-runner-connector-publish-large-dagger-0-6-4 - required: true - airbyte-ci-binary-url: - description: "URL to airbyte-ci binary" - required: false - default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + airbyte_ci_binary_url: + description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci." + default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci" jobs: publish_connectors: name: Publish connectors - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }} + runs-on: connector-publish-large steps: - name: Checkout Airbyte uses: actions/checkout@v3 - name: Publish modified connectors [On merge to master] id: publish-modified-connectors if: github.event_name == 'push' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -46,15 +42,16 @@ jobs: spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" + python_registry_token: ${{ secrets.PYPI_TOKEN }} - name: Publish connectors [manual] id: publish-connectors if: github.event_name == 'workflow_dispatch' - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -66,9 +63,9 @@ jobs: spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" - airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }} + python_registry_token: ${{ secrets.PYPI_TOKEN }} + airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 05deefcb7845..1d38dcab8017 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -5,12 +5,12 @@ on: inputs: runs-on: type: string - default: ci-runner-connector-publish-large-dagger-0-6-4 + default: ci-runner-connector-publish-large-dagger-0-9-5 required: true jobs: no-op: name: No-op - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }} steps: - run: echo 'hi!' diff --git a/.github/workflows/python_cdk_tests.yml b/.github/workflows/python_cdk_tests.yml new file mode 100644 index 000000000000..0fc8a9b5bacb --- /dev/null +++ b/.github/workflows/python_cdk_tests.yml @@ -0,0 +1,137 @@ +# THIS WORKFLOW SHOULD BE REPLACED BY A CLEANER ONE ONCE THE PYTHON CDK TESTS CAN BE RUN WITH POETRY +name: Python CDK Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - master + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + python_cdk: ${{ steps.changes.outputs.python_cdk }} + steps: + - name: Checkout Airbyte + if: github.event_name != 'pull_request' + uses: actions/checkout@v3 + - id: changes + uses: dorny/paths-filter@v2 + with: + # Note: expressions within a filter are OR'ed + filters: | + python_cdk: + - 'airbyte-cdk/python/**/*' + + run-python-cdk-check: + needs: + - changes + if: needs.changes.outputs.python_cdk == 'true' + runs-on: connector-test-large + name: Python CDK Tests + timeout-minutes: 30 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "21" + - name: Install Pip + run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3 + - name: Install Pyenv + run: python3 -m pip install virtualenv --user + - name: Run Gradle Check on Python CDK + run: (cd airbyte-cdk/python && CI=true ./gradlew --no-daemon --no-build-cache --scan check) + + set-instatus-incident-on-failure: + name: Create Instatus Incident on Failure + runs-on: ubuntu-latest + needs: + - run-python-cdk-check + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "down", "status": "HASISSUES" }' + + set-instatus-incident-on-success: + name: Create Instatus Incident on Success + runs-on: ubuntu-latest + needs: + - run-python-cdk-check + if: ${{ success() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "up" }' + + notify-failure-slack-channel: + name: "Notify Slack Channel on Build Failures" + runs-on: ubuntu-latest + needs: + - run-python-cdk-check + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Match GitHub User to Slack User + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to OSS Build Failure Slack Channel + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} + + notify-failure-slack-channel-fixed-broken-build: + name: "Notify Slack Channel on Build Fixes" + runs-on: ubuntu-latest + needs: + - run-python-cdk-check + if: success() + steps: + - name: Get Previous Workflow Status + uses: Mercymeilya/last-workflow-status@v0.3 + id: last_status + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. + - name: Publish Build Fixed Message to OSS Build Failure Slack Channel + if: ${{ steps.last_status.outputs.last_status == 'failure' }} + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: :octavia-rocket: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml index ea36f6aea9bf..b4fbfc9e3255 100644 --- a/.github/workflows/release-airbyte-os.yml +++ b/.github/workflows/release-airbyte-os.yml @@ -49,7 +49,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - uses: actions/setup-node@v3 with: @@ -184,7 +184,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - uses: actions/setup-python@v4 with: @@ -198,29 +198,6 @@ jobs: PART_TO_BUMP: ${{ github.event.inputs.partToBump }} run: ./tools/bin/release_version_octavia.sh - - name: Publish Python Package to test.pypi.org - if: github.event.inputs.skip-publish-test != 'true' - uses: mariamrf/py-package-publish-action@v1.1.0 - with: - # specify the same version as in ~/.python-version - python_version: "3.10" - pip_version: "23.2" - subdir: "octavia-cli/" - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" - - name: Publish Python Package - uses: mariamrf/py-package-publish-action@v1.1.0 - with: - # specify the same version as in ~/.python-version - python_version: "3.10" - pip_version: "23.2" - subdir: "octavia-cli/" - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - # In case of self-hosted EC2 errors, remove this block. stop-release-airbyte-runner: name: "Release Airbyte: Stop EC2 Runner" diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 0fa936b646b8..46678cfa7f1e 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -84,7 +84,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "zulu" - java-version: "17" + java-version: "21" - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.gitignore b/.gitignore index b97efca12e81..bc6841f7b081 100644 --- a/.gitignore +++ b/.gitignore @@ -69,13 +69,6 @@ resources/examples/airflow/logs/* # Summary.md keeps getting added and we just don't like it docs/SUMMARY.md -# Files generated by unit tests -**/specs_secrets_mask.yaml - -# Files generated when downloading connector registry -**/init-oss/src/main/resources/seed/oss_registry.json -**/init-oss/src/main/resources/seed/oss_catalog.json - # Output Files generated by scripts lowcode_connector_names.txt num_lowcode_connectors.csv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8c41a749100c..4fd1a68cdc3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: hooks: - id: format-fix-all-on-push always_run: true - entry: airbyte-ci format fix all + entry: airbyte-ci --disable-update-check format fix all language: system name: Run airbyte-ci format fix on git push (~30s) pass_filenames: false diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000000..b556b2b63c60 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,11 @@ +{ + "overrides": [ + { + "files": "*.md", + "options": { + "printWidth": 100, + "proseWrap": "always" + } + } + ] +} diff --git a/README.md b/README.md index bdf4db5d1544..af1cd195469c 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ See the [LICENSE](docs/project-overview/licenses/) file for licensing informatio ### Thank You -Airbyte would not be possible without the support and assistance of other open-source tools and companies. Visit our [thank you page](THANK-YOU.md) to learn more about how we build Airbyte. +Airbyte would not be possible without the support and assistance of other open-source tools and companies! Visit our [thank you page](THANK-YOU.md) to learn more about how we build Airbyte. diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 30f3c517c523..adf174f5a256 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -2,21 +2,23 @@ This page will walk through the process of developing with the Java CDK. -- [Developing with the Java CDK](#developing-with-the-java-cdk) - - [Intro to the Java CDK](#intro-to-the-java-cdk) - - [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) - - [How is the CDK published?](#how-is-the-cdk-published) - - [Using the Java CDK](#using-the-java-cdk) - - [Building the CDK](#building-the-cdk) - - [Bumping the CDK version](#bumping-the-cdk-version) - - [Publishing the CDK](#publishing-the-cdk) - - [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) - - [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) - - [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) - - [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) - - [Common Debugging Tips](#common-debugging-tips) - - [Changelog](#changelog) - - [Java CDK](#java-cdk) +* [Developing with the Java CDK](#developing-with-the-java-cdk) + * [Intro to the Java CDK](#intro-to-the-java-cdk) + * [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) + * [How is the CDK published?](#how-is-the-cdk-published) + * [Using the Java CDK](#using-the-java-cdk) + * [Building the CDK](#building-the-cdk) + * [Bumping the CDK version](#bumping-the-cdk-version) + * [Publishing the CDK](#publishing-the-cdk) + * [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) + * [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) + * [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) + * [Publishing the CDK and switching to a pinned CDK reference](#publishing-the-cdk-and-switching-to-a-pinned-cdk-reference) + * [Troubleshooting CDK Dependency Caches](#troubleshooting-cdk-dependency-caches) + * [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) + * [Common Debugging Tips](#common-debugging-tips) + * [Changelog](#changelog) + * [Java CDK](#java-cdk) ## Intro to the Java CDK @@ -121,6 +123,8 @@ When modifying the CDK and a connector in the same PR or branch, please use the After the above, you can build and test your connector as usual. Gradle will automatically use the local CDK code files while you are working on the connector. +### Publishing the CDK and switching to a pinned CDK reference + Once you are done developing and testing your CDK changes: 1. Publish the CDK using the instructions here in this readme. @@ -129,6 +133,12 @@ Once you are done developing and testing your CDK changes: _Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ +### Troubleshooting CDK Dependency Caches + +Note: after switching between a local and a pinned CDK reference, you may need to refresh dependency caches in Gradle and/or your IDE. + +In Gradle, you can use the CLI arg `--refresh-dependencies` the next time you build or test your connector, which will ensure that the correct version of the CDK is used after toggling the `useLocalCdk` value. + ### Developing a connector against a pinned CDK version You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`. @@ -156,7 +166,66 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.7.2 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | Fix DV2 JDBC type mappings (code changes in [\#33307](https://github.com/airbytehq/airbyte/pull/33307)). | +| 0.23.2 | 2024-02-22 | [\#35385](https://github.com/airbytehq/airbyte/pull/35342) | Bugfix: inverted logic of disableTypeDedupe flag | +| 0.23.1 | 2024-02-22 | [\#35527](https://github.com/airbytehq/airbyte/pull/35527) | reduce shutdow timeouts | +| 0.23.0 | 2024-02-22 | [\#35342](https://github.com/airbytehq/airbyte/pull/35342) | Consolidate and perform upfront gathering of DB metadata state | +| 0.21.4 | 2024-02-21 | [\#35511](https://github.com/airbytehq/airbyte/pull/35511) | Reduce CDC state compression limit to 1MB | +| 0.21.3 | 2024-02-20 | [\#35394](https://github.com/airbytehq/airbyte/pull/35394) | Add Junit progress information to the test logs | +| 0.21.2 | 2024-02-20 | [\#34978](https://github.com/airbytehq/airbyte/pull/34978) | Reduce log noise in NormalizationLogParser. | +| 0.21.1 | 2024-02-20 | [\#35199](https://github.com/airbytehq/airbyte/pull/35199) | Add thread names to the logs. | +| 0.21.0 | 2024-02-16 | [\#35314](https://github.com/airbytehq/airbyte/pull/35314) | Delete S3StreamCopier classes. These have been superseded by the async destinations framework. | +| 0.20.9 | 2024-02-15 | [\#35240](https://github.com/airbytehq/airbyte/pull/35240) | Make state emission to platform inside state manager itself. | +| 0.20.8 | 2024-02-15 | [\#35285](https://github.com/airbytehq/airbyte/pull/35285) | Improve blobstore module structure. | +| 0.20.7 | 2024-02-13 | [\#35236](https://github.com/airbytehq/airbyte/pull/35236) | output logs to files in addition to stdout when running tests | +| 0.20.6 | 2024-02-12 | [\#35036](https://github.com/airbytehq/airbyte/pull/35036) | Add trace utility to emit analytics messages. | +| 0.20.5 | 2024-02-13 | [\#34869](https://github.com/airbytehq/airbyte/pull/34869) | Don't emit final state in SourceStateIterator there is an underlying stream failure. | +| 0.20.4 | 2024-02-12 | [\#35042](https://github.com/airbytehq/airbyte/pull/35042) | Use delegate's isDestinationV2 invocation in SshWrappedDestination. | +| 0.20.3 | 2024-02-09 | [\#34580](https://github.com/airbytehq/airbyte/pull/34580) | Support special chars in mysql/mssql database name. | +| 0.20.2 | 2024-02-12 | [\#35111](https://github.com/airbytehq/airbyte/pull/35144) | Make state emission from async framework synchronized. | +| 0.20.1 | 2024-02-11 | [\#35111](https://github.com/airbytehq/airbyte/pull/35111) | Fix GlobalAsyncStateManager stats counting logic. | +| 0.20.0 | 2024-02-09 | [\#34562](https://github.com/airbytehq/airbyte/pull/34562) | Add new test cases to BaseTypingDedupingTest to exercise special characters. | +| 0.19.0 | 2024-02-01 | [\#34745](https://github.com/airbytehq/airbyte/pull/34745) | Reorganize CDK module structure. | +| 0.18.0 | 2024-02-08 | [\#33606](https://github.com/airbytehq/airbyte/pull/33606) | Add updated Initial and Incremental Stream State definitions for DB Sources. | +| 0.17.1 | 2024-02-08 | [\#35027](https://github.com/airbytehq/airbyte/pull/35027) | Make state handling thread safe in async destination framework. | +| 0.17.0 | 2024-02-08 | [\#34502](https://github.com/airbytehq/airbyte/pull/34502) | Enable configuring async destination batch size. | +| 0.16.6 | 2024-02-07 | [\#34892](https://github.com/airbytehq/airbyte/pull/34892) | Improved testcontainers logging and support for unshared containers. | +| 0.16.5 | 2024-02-07 | [\#34948](https://github.com/airbytehq/airbyte/pull/34948) | Fix source state stats counting logic | +| 0.16.4 | 2024-02-01 | [\#34727](https://github.com/airbytehq/airbyte/pull/34727) | Add future based stdout consumer in BaseTypingDedupingTest | +| 0.16.3 | 2024-01-30 | [\#34669](https://github.com/airbytehq/airbyte/pull/34669) | Fix org.apache.logging.log4j:log4j-slf4j-impl version conflicts. | +| 0.16.2 | 2024-01-29 | [\#34630](https://github.com/airbytehq/airbyte/pull/34630) | expose NamingTransformer to sub-classes in destinations JdbcSqlGenerator. | +| 0.16.1 | 2024-01-29 | [\#34533](https://github.com/airbytehq/airbyte/pull/34533) | Add a safe method to execute DatabaseMetadata's Resultset returning queries. | +| 0.16.0 | 2024-01-26 | [\#34573](https://github.com/airbytehq/airbyte/pull/34573) | Untangle Debezium harness dependencies. | +| 0.15.2 | 2024-01-25 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Improve airbyte-api build performance. | +| 0.15.1 | 2024-01-25 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Async destinations: Better logging when we fail to parse an AirbyteMessage | +| 0.15.0 | 2024-01-23 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Removed connector registry and micronaut dependencies. | +| 0.14.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Handle case-sensitivity in sentry error grouping | +| 0.14.1 | 2024-01-24 | [\#34468](https://github.com/airbytehq/airbyte/pull/34468) | Add wait for process to be done before ending sync in destination BaseTDTest | +| 0.14.0 | 2024-01-23 | [\#34461](https://github.com/airbytehq/airbyte/pull/34461) | Revert non backward compatible signature changes from 0.13.1 | +| 0.13.3 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Denote if destinations fully support Destinations V2 | +| 0.13.2 | 2024-01-18 | [\#34364](https://github.com/airbytehq/airbyte/pull/34364) | Better logging in mongo db source connector | +| 0.13.1 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Add postCreateTable hook in destination JdbcSqlGenerator | +| 0.13.0 | 2024-01-16 | [\#34177](https://github.com/airbytehq/airbyte/pull/34177) | Add `useExpensiveSafeCasting` param in JdbcSqlGenerator methods; add JdbcTypingDedupingTest fixture; other DV2-related changes | +| 0.12.1 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Add hook for additional destination specific checks to JDBC destination check method | +| 0.12.0 | 2024-01-10 | [\#33875](https://github.com/airbytehq/airbyte/pull/33875) | Upgrade sshd-mina to 2.11.1 | +| 0.11.5 | 2024-01-10 | [\#34119](https://github.com/airbytehq/airbyte/pull/34119) | Remove wal2json support for postgres+debezium. | +| 0.11.4 | 2024-01-09 | [\#33305](https://github.com/airbytehq/airbyte/pull/33305) | Source stats in incremental syncs | +| 0.11.3 | 2023-01-09 | [\#33658](https://github.com/airbytehq/airbyte/pull/33658) | Always fail when debezium fails, even if it happened during the setup phase. | +| 0.11.2 | 2024-01-09 | [\#33969](https://github.com/airbytehq/airbyte/pull/33969) | Destination state stats implementation | +| 0.11.1 | 2024-01-04 | [\#33727](https://github.com/airbytehq/airbyte/pull/33727) | SSH bastion heartbeats for Destinations | +| 0.11.0 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | DV2 T+D uses Sql struct to represent transactions; other T+D-related changes | +| 0.10.4 | 2023-12-20 | [\#33071](https://github.com/airbytehq/airbyte/pull/33071) | Add the ability to parse JDBC parameters with another delimiter than '&' | +| 0.10.3 | 2024-01-03 | [\#33312](https://github.com/airbytehq/airbyte/pull/33312) | Send out count in AirbyteStateMessage | +| 0.10.1 | 2023-12-21 | [\#33723](https://github.com/airbytehq/airbyte/pull/33723) | Make memory-manager log message less scary | +| 0.10.0 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | JdbcDestinationHandler now properly implements `getInitialRawTableState`; reenable SqlGenerator test | +| 0.9.0 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation, exclude the T&D module from the CDK | +| 0.8.0 | 2023-12-18 | [\#33506](https://github.com/airbytehq/airbyte/pull/33506) | Improve async destination shutdown logic; more JDBC async migration work; improve DAT test schema handling | +| 0.7.9 | 2023-12-18 | [\#33549](https://github.com/airbytehq/airbyte/pull/33549) | Improve MongoDB logging. | +| 0.7.8 | 2023-12-18 | [\#33365](https://github.com/airbytehq/airbyte/pull/33365) | Emit stream statuses more consistently | +| 0.7.7 | 2023-12-18 | [\#33434](https://github.com/airbytehq/airbyte/pull/33307) | Remove LEGACY state | +| 0.7.6 | 2023-12-14 | [\#32328](https://github.com/airbytehq/airbyte/pull/33307) | Add schema less mode for mongodb CDC. Fixes for non standard mongodb id type. | +| 0.7.4 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Track stream record count during sync; only run T+D if a stream had nonzero records or the previous sync left unprocessed records. | +| 0.7.3 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Extract shared JDBC T+D code. | +| 0.7.2 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | Fix DV2 JDBC type mappings (code changes in [\#33307](https://github.com/airbytehq/airbyte/pull/33307)). | | 0.7.1 | 2023-12-01 | [\#33027](https://github.com/airbytehq/airbyte/pull/33027) | Add the abstract DB source debugger. | | 0.7.0 | 2023-12-07 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Destinations V2 changes for JDBC destinations | | 0.6.4 | 2023-12-06 | [\#33082](https://github.com/airbytehq/airbyte/pull/33082) | Improvements to schema snapshot error handling + schema snapshot history scope (scoped to configured DB). | diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle deleted file mode 100644 index 0a12cf304887..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle +++ /dev/null @@ -1,50 +0,0 @@ -plugins { - id "java-library" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-try" - } -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - - implementation group: 'joda-time', name: 'joda-time', version: '2.12.5' - implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'com.auth0:java-jwt:3.19.2' - implementation libs.guava - implementation(libs.temporal.sdk) { - exclude module: 'guava' - } - implementation 'org.apache.ant:ant:1.10.10' - implementation 'org.apache.commons:commons-text:1.10.0' - implementation libs.bundles.datadog - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol') - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testAnnotationProcessor libs.jmh.annotations - - testImplementation libs.bundles.micronaut.test - testImplementation 'com.jayway.jsonpath:json-path:2.7.0' - testImplementation 'org.mockito:mockito-inline:4.7.0' - testImplementation libs.postgresql - testImplementation libs.testcontainers - testImplementation libs.testcontainers.postgresql - testImplementation libs.jmh.core - testImplementation libs.jmh.annotations - testImplementation 'com.github.docker-java:docker-java:3.2.8' - testImplementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' -} diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java deleted file mode 100644 index 9d7941952735..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.internal; - -import io.airbyte.commons.protocol.AirbyteMessageVersionedMigrator; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.io.BufferedWriter; -import java.io.IOException; -import java.util.Optional; - -public class VersionedAirbyteMessageBufferedWriter extends DefaultAirbyteMessageBufferedWriter { - - private final AirbyteMessageSerializer serializer; - private final AirbyteMessageVersionedMigrator migrator; - private final Optional configuredAirbyteCatalog; - - public VersionedAirbyteMessageBufferedWriter(final BufferedWriter writer, - final AirbyteMessageSerializer serializer, - final AirbyteMessageVersionedMigrator migrator, - final Optional configuredAirbyteCatalog) { - super(writer); - this.serializer = serializer; - this.migrator = migrator; - this.configuredAirbyteCatalog = configuredAirbyteCatalog; - } - - @Override - public void write(final AirbyteMessage message) throws IOException { - final T downgradedMessage = migrator.downgrade(message, configuredAirbyteCatalog); - writer.write(serializer.serialize(downgradedMessage)); - writer.newLine(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java deleted file mode 100644 index 32a848f84131..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.internal; - -import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; -import io.airbyte.commons.protocol.AirbyteProtocolVersionedMigratorFactory; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.io.BufferedWriter; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class VersionedAirbyteMessageBufferedWriterFactory implements AirbyteMessageBufferedWriterFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(VersionedAirbyteMessageBufferedWriterFactory.class); - - private final AirbyteMessageSerDeProvider serDeProvider; - private final AirbyteProtocolVersionedMigratorFactory migratorFactory; - private final Version protocolVersion; - private final Optional configuredAirbyteCatalog; - - public VersionedAirbyteMessageBufferedWriterFactory(final AirbyteMessageSerDeProvider serDeProvider, - final AirbyteProtocolVersionedMigratorFactory migratorFactory, - final Version protocolVersion, - final Optional configuredAirbyteCatalog) { - this.serDeProvider = serDeProvider; - this.migratorFactory = migratorFactory; - this.protocolVersion = protocolVersion; - this.configuredAirbyteCatalog = configuredAirbyteCatalog; - } - - @Override - public AirbyteMessageBufferedWriter createWriter(BufferedWriter bufferedWriter) { - final boolean needMigration = !protocolVersion.getMajorVersion().equals(migratorFactory.getMostRecentVersion().getMajorVersion()); - LOGGER.info( - "Writing messages to protocol version {}{}", - protocolVersion.serialize(), - needMigration ? ", messages will be downgraded from protocol version " + migratorFactory.getMostRecentVersion().serialize() : ""); - return new VersionedAirbyteMessageBufferedWriter<>( - bufferedWriter, - serDeProvider.getSerializer(protocolVersion).orElseThrow(), - migratorFactory.getAirbyteMessageMigrator(protocolVersion), - configuredAirbyteCatalog); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle deleted file mode 100644 index 2db31d830e9c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle +++ /dev/null @@ -1,222 +0,0 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - id "org.openapi.generator" version "6.2.1" - id "java-library" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-deprecation" - } -} - -def specFile = "$projectDir/src/main/openapi/config.yaml" - -def generate = tasks.register('generate') - -// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer -def generateApiServerLegacy = tasks.register('generateApiServerLegacy', GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" - - inputs.file specFile - outputs.dir serverOutputDir - - generatorName = "jaxrs-spec" - inputSpec = specFile - outputDir = serverOutputDir - - apiPackage = "io.airbyte.api.generated" - invokerPackage = "io.airbyte.api.invoker.generated" - modelPackage = "io.airbyte.api.model.generated" - - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) - - generateApiDocumentation = false - - configOptions.set([ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - ]) -} -generate.configure { - dependsOn generateApiServerLegacy -} - -def generateApiServer = tasks.register('generateApiServer', GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" - - inputs.file specFile - outputs.dir serverOutputDir - - generatorName = "jaxrs-spec" - inputSpec = specFile - outputDir = serverOutputDir - - apiPackage = "io.airbyte.api.generated" - invokerPackage = "io.airbyte.api.invoker.generated" - modelPackage = "io.airbyte.api.model.generated" - - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) - - generateApiDocumentation = false - - configOptions.set([ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - - // Generate separate classes for each endpoint "domain" - useTags: "true" - ]) -} -generate.configure { - dependsOn generateApiServer -} - -def generateApiClient = tasks.register('generateApiClient', GenerateTask) { - def clientOutputDir = "$buildDir/generated/api/client" - - inputs.file specFile - outputs.dir clientOutputDir - - generatorName = "java" - inputSpec = specFile - outputDir = clientOutputDir - - apiPackage = "io.airbyte.api.client.generated" - invokerPackage = "io.airbyte.api.client.invoker.generated" - modelPackage = "io.airbyte.api.client.model.generated" - - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) - - library = "native" - - generateApiDocumentation = false - - configOptions.set([ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true" - ]) -} -generate.configure { - dependsOn generateApiClient -} - -def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { - def docsOutputDir = "$buildDir/generated/api/docs" - - generatorName = "html" - inputSpec = specFile - outputDir = docsOutputDir - - apiPackage = "io.airbyte.api.client.generated" - invokerPackage = "io.airbyte.api.client.invoker.generated" - modelPackage = "io.airbyte.api.client.model.generated" - - schemaMappings.set([ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ]) - - generateApiDocumentation = false - - configOptions.set([ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true" - ]) - - doLast { - def target = file(rootProject.file("docs/reference/api/generated-api-html")) - delete target - mkdir target - copy { - from outputDir - include "**/*.html" - includeEmptyDirs = false - into target - } - } -} -generate.configure { - dependsOn generateApiDocs -} - -dependencies { - implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' - - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - - implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' - implementation group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.1.1' - implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' - - implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1' -} - -sourceSets { - main { - java { - srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java" - } - resources { - srcDir "$projectDir/src/main/openapi/" - } - } -} - -tasks.named('compileJava').configure { - dependsOn generate -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md deleted file mode 100644 index 33ffeeb918dd..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# airbyte-api - -Defines the OpenApi configuration for the Airbyte Configuration API. It also is responsible for generating the following from the API spec: -* Java API client -* Java API server - this generated code is used in `airbyte-server` to allow us to implement the Configuration API in a type safe way. See `ConfigurationApi.java` in `airbyte-server` -* API docs - -## Key Files -* src/openapi/config.yaml - Defines the config API interface using OpenApi3 -* AirbyteApiClient.java - wraps all api clients so that they can be dependency injected together -* PatchedLogsApi.java - fixes generated code for log api. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle deleted file mode 100644 index 3dbb175d2ad9..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle +++ /dev/null @@ -1,7 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'commons-cli:commons-cli:1.4' -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md deleted file mode 100644 index 81aa7feb0b33..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# airbyte-commons-cli - -This module houses utility functions for the `commons-cli` library. It is separate from `commons`, because it depends on external library `commons-cli` which we do not want to introduce as a dependency to every module. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle deleted file mode 100644 index ae8a69d75513..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -java { - compileJava { - options.compilerArgs += "-Xlint:-unchecked" - } -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - implementation libs.bundles.micronaut.annotation - testImplementation libs.bundles.micronaut.test - - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java deleted file mode 100644 index 3d0bcaf97008..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.protocol.migrations.MigrationContainer; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -/** - * AirbyteProtocol Message Migrator - * - * This class is intended to apply the transformations required to go from one version of the - * AirbyteProtocol to another. - */ -@Singleton -public class AirbyteMessageMigrator { - - private final MigrationContainer> migrationContainer; - - public AirbyteMessageMigrator(final List> migrations) { - migrationContainer = new MigrationContainer<>(migrations); - } - - @PostConstruct - public void initialize() { - migrationContainer.initialize(); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, - final Version target, - final Optional configuredAirbyteCatalog) { - return migrationContainer.downgrade(message, target, (migration, msg) -> applyDowngrade(migration, msg, configuredAirbyteCatalog)); - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, - final Version source, - final Optional configuredAirbyteCatalog) { - return migrationContainer.upgrade(message, source, (migration, msg) -> applyUpgrade(migration, msg, configuredAirbyteCatalog)); - } - - public Version getMostRecentVersion() { - return migrationContainer.getMostRecentVersion(); - } - - // Helper function to work around type casting - private static PreviousVersion applyDowngrade(final AirbyteMessageMigration migration, - final Object message, - final Optional configuredAirbyteCatalog) { - return migration.downgrade((CurrentVersion) message, configuredAirbyteCatalog); - } - - // Helper function to work around type casting - private static CurrentVersion applyUpgrade(final AirbyteMessageMigration migration, - final Object message, - final Optional configuredAirbyteCatalog) { - return migration.upgrade((PreviousVersion) message, configuredAirbyteCatalog); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getMigrationKeys() { - return migrationContainer.getMigrationKeys(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java deleted file mode 100644 index 8ead1378bc02..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.commons.version.Version; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * AirbyteProtocol Message Serializer/Deserializer provider - * - * This class is intended to help access the serializer/deserializer for a given version of the - * Airbyte Protocol. - */ -@Singleton -public class AirbyteMessageSerDeProvider { - - private final List> deserializersToRegister; - private final List> serializersToRegister; - - private final Map> deserializers = new HashMap<>(); - private final Map> serializers = new HashMap<>(); - - public AirbyteMessageSerDeProvider(final List> deserializers, - final List> serializers) { - deserializersToRegister = deserializers; - serializersToRegister = serializers; - } - - public AirbyteMessageSerDeProvider() { - this(Collections.emptyList(), Collections.emptyList()); - } - - @PostConstruct - public void initialize() { - deserializersToRegister.forEach(this::registerDeserializer); - serializersToRegister.forEach(this::registerSerializer); - } - - /** - * Returns the Deserializer for the version if known else empty - */ - public Optional> getDeserializer(final Version version) { - return Optional.ofNullable(deserializers.get(version.getMajorVersion())); - } - - /** - * Returns the Serializer for the version if known else empty - */ - public Optional> getSerializer(final Version version) { - return Optional.ofNullable(serializers.get(version.getMajorVersion())); - } - - @VisibleForTesting - void registerDeserializer(final AirbyteMessageDeserializer deserializer) { - final String key = deserializer.getTargetVersion().getMajorVersion(); - if (!deserializers.containsKey(key)) { - deserializers.put(key, deserializer); - } else { - throw new RuntimeException(String.format("Trying to register a deserializer for protocol version {} when {} already exists", - deserializer.getTargetVersion().serialize(), deserializers.get(key).getTargetVersion().serialize())); - } - } - - @VisibleForTesting - void registerSerializer(final AirbyteMessageSerializer serializer) { - final String key = serializer.getTargetVersion().getMajorVersion(); - if (!serializers.containsKey(key)) { - serializers.put(key, serializer); - } else { - throw new RuntimeException(String.format("Trying to register a serializer for protocol version {} when {} already exists", - serializer.getTargetVersion().serialize(), serializers.get(key).getTargetVersion().serialize())); - } - } - - // Used for inspection of the injection - @VisibleForTesting - Set getDeserializerKeys() { - return deserializers.keySet(); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getSerializerKeys() { - return serializers.keySet(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java deleted file mode 100644 index f3339fcd5e38..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.Optional; - -/** - * Wraps message migration from a fixed version to the most recent version - */ -public class AirbyteMessageVersionedMigrator { - - private final AirbyteMessageMigrator migrator; - private final Version version; - - public AirbyteMessageVersionedMigrator(final AirbyteMessageMigrator migrator, final Version version) { - this.migrator = migrator; - this.version = version; - } - - public OriginalMessageType downgrade(final AirbyteMessage message, final Optional configuredAirbyteCatalog) { - return migrator.downgrade(message, version, configuredAirbyteCatalog); - } - - public AirbyteMessage upgrade(final OriginalMessageType message, final Optional configuredAirbyteCatalog) { - return migrator.upgrade(message, version, configuredAirbyteCatalog); - } - - public Version getVersion() { - return version; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java deleted file mode 100644 index 2388e95e4a08..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.version.Version; -import jakarta.inject.Singleton; - -/** - * Factory to build AirbyteMessageVersionedMigrator - */ -@Singleton -public class AirbyteProtocolVersionedMigratorFactory { - - private final AirbyteMessageMigrator airbyteMessageMigrator; - private final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - - public AirbyteProtocolVersionedMigratorFactory(final AirbyteMessageMigrator airbyteMessageMigrator, - final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator) { - this.airbyteMessageMigrator = airbyteMessageMigrator; - this.configuredAirbyteCatalogMigrator = configuredAirbyteCatalogMigrator; - } - - public AirbyteMessageVersionedMigrator getAirbyteMessageMigrator(final Version version) { - return new AirbyteMessageVersionedMigrator<>(airbyteMessageMigrator, version); - } - - public final VersionedProtocolSerializer getProtocolSerializer(final Version version) { - return new VersionedProtocolSerializer(configuredAirbyteCatalogMigrator, version); - } - - public Version getMostRecentVersion() { - return airbyteMessageMigrator.getMostRecentVersion(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java deleted file mode 100644 index c61522bfad9c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.migrations.ConfiguredAirbyteCatalogMigration; -import io.airbyte.commons.protocol.migrations.MigrationContainer; -import io.airbyte.commons.version.Version; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Singleton; -import java.util.List; -import java.util.Set; - -@Singleton -public class ConfiguredAirbyteCatalogMigrator { - - private final MigrationContainer> migrationContainer; - - public ConfiguredAirbyteCatalogMigrator(final List> migrations) { - migrationContainer = new MigrationContainer<>(migrations); - } - - @PostConstruct - public void initialize() { - migrationContainer.initialize(); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, final Version target) { - return migrationContainer.downgrade(message, target, ConfiguredAirbyteCatalogMigrator::applyDowngrade); - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, final Version source) { - return migrationContainer.upgrade(message, source, ConfiguredAirbyteCatalogMigrator::applyUpgrade); - } - - public Version getMostRecentVersion() { - return migrationContainer.getMostRecentVersion(); - } - - // Helper function to work around type casting - private static PreviousVersion applyDowngrade(final ConfiguredAirbyteCatalogMigration migration, - final Object message) { - return migration.downgrade((CurrentVersion) message); - } - - // Helper function to work around type casting - private static CurrentVersion applyUpgrade(final ConfiguredAirbyteCatalogMigration migration, - final Object message) { - return migration.upgrade((PreviousVersion) message); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getMigrationKeys() { - return migrationContainer.getMigrationKeys(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java deleted file mode 100644 index db04473eecba..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; - -/** - * Serialize a ConfiguredAirbyteCatalog to the specified version - *

- * This Serializer expects a ConfiguredAirbyteCatalog from the Current version of the platform, - * converts it to the target protocol version before serializing it. - */ -public class VersionedProtocolSerializer implements ProtocolSerializer { - - private final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - private final Version protocolVersion; - - public VersionedProtocolSerializer(final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator, final Version protocolVersion) { - this.configuredAirbyteCatalogMigrator = configuredAirbyteCatalogMigrator; - this.protocolVersion = protocolVersion; - } - - @Override - public String serialize(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - return Jsons.serialize(configuredAirbyteCatalogMigrator.downgrade(configuredAirbyteCatalog, protocolVersion)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java deleted file mode 100644 index c49f9db1e665..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.Optional; - -/** - * AirbyteProtocol message migration interface - * - * @param The Old AirbyteMessage type - * @param The New AirbyteMessage type - */ -public interface AirbyteMessageMigration extends Migration { - - /** - * Downgrades a message to from the new version to the old version - * - * @param message: the message to downgrade - * @param configuredAirbyteCatalog: the ConfiguredAirbyteCatalog of the connection when applicable - * @return the downgraded message - */ - PreviousVersion downgrade(final CurrentVersion message, final Optional configuredAirbyteCatalog); - - /** - * Upgrades a message from the old version to the new version - * - * @param message: the message to upgrade - * @param configuredAirbyteCatalog: the ConfiguredAirbyteCatalog of the connection when applicable - * @return the upgrade message - */ - CurrentVersion upgrade(final PreviousVersion message, final Optional configuredAirbyteCatalog); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java deleted file mode 100644 index 4634c4464e7c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -public interface ConfiguredAirbyteCatalogMigration extends Migration { - - /** - * Downgrades a ConfiguredAirbyteCatalog from the new version to the old version - * - * @param message: the ConfiguredAirbyteCatalog to downgrade - * @return the downgraded ConfiguredAirbyteCatalog - */ - PreviousVersion downgrade(final CurrentVersion message); - - /** - * Upgrades a ConfiguredAirbyteCatalog from the old version to the new version - * - * @param message: the ConfiguredAirbyteCatalog to upgrade - * @return the upgraded ConfiguredAirbyteCatalog - */ - CurrentVersion upgrade(final PreviousVersion message); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java deleted file mode 100644 index 47b1469b1a0e..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.commons.version.Version; - -public interface Migration { - - /** - * The Old version, note that due to semver, the important piece of information is the Major. - */ - Version getPreviousVersion(); - - /** - * The New version, note that due to semver, the important piece of information is the Major. - */ - Version getCurrentVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java deleted file mode 100644 index c421742d5f38..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.commons.version.Version; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.BiFunction; - -public class MigrationContainer { - - private final List migrationsToRegister; - private final SortedMap migrations = new TreeMap<>(); - - // mostRecentMajorVersion defaults to v0 as no migration is required - private String mostRecentMajorVersion = "0"; - - public MigrationContainer(final List migrations) { - this.migrationsToRegister = migrations; - } - - public void initialize() { - migrationsToRegister.forEach(this::registerMigration); - } - - public Version getMostRecentVersion() { - return new Version(mostRecentMajorVersion, "0", "0"); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, - final Version target, - final BiFunction applyDowngrade) { - if (target.getMajorVersion().equals(mostRecentMajorVersion)) { - return (PreviousVersion) message; - } - - Object result = message; - Object[] selectedMigrations = selectMigrations(target).toArray(); - for (int i = selectedMigrations.length; i > 0; --i) { - result = applyDowngrade.apply((T) selectedMigrations[i - 1], result); - } - return (PreviousVersion) result; - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, - final Version source, - final BiFunction applyUpgrade) { - if (source.getMajorVersion().equals(mostRecentMajorVersion)) { - return (CurrentVersion) message; - } - - Object result = message; - for (var migration : selectMigrations(source)) { - result = applyUpgrade.apply(migration, result); - } - return (CurrentVersion) result; - } - - public Collection selectMigrations(final Version version) { - final Collection results = migrations.tailMap(version.getMajorVersion()).values(); - if (results.isEmpty()) { - throw new RuntimeException("Unsupported migration version " + version.serialize()); - } - return results; - } - - /** - * Store migration in a sorted map key by the major of the lower version of the migration. - * - * The goal is to be able to retrieve the list of migrations to apply to get to/from a given - * version. We are only keying on the lower version because the right side (most recent version of - * the migration range) is always current version. - */ - private void registerMigration(final T migration) { - final String key = migration.getPreviousVersion().getMajorVersion(); - if (!migrations.containsKey(key)) { - migrations.put(key, migration); - if (migration.getCurrentVersion().getMajorVersion().compareTo(mostRecentMajorVersion) > 0) { - mostRecentMajorVersion = migration.getCurrentVersion().getMajorVersion(); - } - } else { - throw new RuntimeException("Trying to register a duplicated migration " + migration.getClass().getName()); - } - } - - public Set getMigrationKeys() { - return migrations.keySet(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java deleted file mode 100644 index 4c580929f5af..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.util; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ARRAY_TYPE; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ITEMS_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.OBJECT_TYPE; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ONEOF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.PROPERTIES_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.TYPE_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.function.BiFunction; -import java.util.function.Function; - -public class RecordMigrations { - - /** - * Quick and dirty tuple. Used internally by - * {@link #mutateDataNode(JsonSchemaValidator, Function, Transformer, JsonNode, JsonNode)}; callers - * probably only actually need the node. - * - * matchedSchema is useful for mutating using a oneOf schema, where we need to recognize the correct - * subschema. - * - * @param node Our attempt at mutating the node, under the given schema - * @param matchedSchema Whether the original node actually matched the schema - */ - public record MigratedNode(JsonNode node, boolean matchedSchema) {} - - /** - * Extend BiFunction so that we can have named parameters. - */ - @FunctionalInterface - public interface Transformer extends BiFunction { - - @Override - MigratedNode apply(JsonNode schema, JsonNode data); - - } - - /** - * Works on a best-effort basis. If the schema doesn't match the data, we'll do our best to mutate - * anything that we can definitively say matches the criteria. Should _not_ throw an exception if - * bad things happen (e.g. we try to parse a non-numerical string as a number). - * - * @param schemaMatcher Accepts a JsonNode schema and returns whether its corresponding entry in the - * data should be mutated. Doesn't need to handle oneOf cases, i.e. should only care about - * type/$ref. - * @param transformer Performs the modification on the given data node. Should not throw exceptions. - */ - public static MigratedNode mutateDataNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - // If this is a oneOf node, then we need to handle each oneOf case. - if (!schema.hasNonNull(REF_KEY) && !schema.hasNonNull(TYPE_KEY) && schema.hasNonNull(ONEOF_KEY)) { - return mutateOneOfNode(validator, schemaMatcher, transformer, data, schema); - } - - // If we should mutate the data, then mutate it appropriately - if (schemaMatcher.apply(schema)) { - return transformer.apply(schema, data); - } - - // Otherwise, we need to recurse into non-primitive nodes. - if (data.isObject()) { - return mutateObjectNode(validator, schemaMatcher, transformer, data, schema); - } else if (data.isArray()) { - return mutateArrayNode(validator, schemaMatcher, transformer, data, schema); - } else { - // There's nothing to do in the case of a primitive node. - // So we just check whether the schema is correct and return the node as-is. - return new MigratedNode(data, validator.test(schema, data)); - } - } - - /** - * Attempt to mutate using each oneOf option in sequence. Returns the result from mutating using the - * first subschema that matches the data, or if none match, then the result of using the first - * subschema. - */ - private static MigratedNode mutateOneOfNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - final JsonNode schemaOptions = schema.get(ONEOF_KEY); - if (schemaOptions.size() == 0) { - // If the oneOf has no options, then don't do anything interesting. - return new MigratedNode(data, validator.test(schema, data)); - } - - // Attempt to mutate the node against each oneOf schema. - // Return the first schema that matches the data, or the first schema if none matched successfully. - MigratedNode migratedNode = null; - for (final JsonNode maybeSchema : schemaOptions) { - final MigratedNode maybeMigratedNode = mutateDataNode(validator, schemaMatcher, transformer, data, maybeSchema); - if (maybeMigratedNode.matchedSchema()) { - // If we've found a matching schema, then return immediately - return maybeMigratedNode; - } else if (migratedNode == null) { - // Otherwise - if this is the first subschema, then just take it - migratedNode = maybeMigratedNode; - } - } - // None of the schemas matched, so just return whatever we found first - return migratedNode; - } - - /** - * If data is an object, then we need to recursively mutate all of its fields. - */ - private static MigratedNode mutateObjectNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - boolean isObjectSchema; - // First, check whether the schema is supposed to be an object at all. - if (schema.hasNonNull(REF_KEY)) { - // If the schema uses a reference type, then it's not an object schema. - isObjectSchema = false; - } else if (schema.hasNonNull(TYPE_KEY)) { - // If the schema declares {type: object} or {type: [..., object, ...]} - // Then this is an object schema - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - isObjectSchema = false; - for (final JsonNode typeItem : typeNode) { - if (OBJECT_TYPE.equals(typeItem.asText())) { - isObjectSchema = true; - } - } - } else { - isObjectSchema = OBJECT_TYPE.equals(typeNode.asText()); - } - } else { - // If the schema doesn't declare a type at all (which is bad practice, but let's handle it anyway) - // Then check for a properties entry, and assume that this is an object if it's present - isObjectSchema = schema.hasNonNull(PROPERTIES_KEY); - } - - if (!isObjectSchema) { - // If it's not supposed to be an object, then we can't do anything here. - // Return the data without modification. - return new MigratedNode(data, false); - } else { - // If the schema _is_ for an object, then recurse into each field - final ObjectNode mutatedData = (ObjectNode) Jsons.emptyObject(); - final JsonNode propertiesNode = schema.get(PROPERTIES_KEY); - - final Iterator> dataFields = data.fields(); - boolean matchedSchema = true; - while (dataFields.hasNext()) { - final Entry field = dataFields.next(); - final String key = field.getKey(); - final JsonNode value = field.getValue(); - if (propertiesNode != null && propertiesNode.hasNonNull(key)) { - // If we have a schema for this property, mutate the value - final JsonNode subschema = propertiesNode.get(key); - final MigratedNode migratedNode = mutateDataNode(validator, schemaMatcher, transformer, value, subschema); - mutatedData.set(key, migratedNode.node); - if (!migratedNode.matchedSchema) { - matchedSchema = false; - } - } else { - // Else it's an additional property - we _could_ check additionalProperties, - // but that's annoying. We don't actually respect that in destinations/normalization anyway. - mutatedData.set(key, value); - } - } - - return new MigratedNode(mutatedData, matchedSchema); - } - } - - /** - * Much like objects, arrays must be recursively mutated. - */ - private static MigratedNode mutateArrayNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - // Similar to objects, we first check whether this is even supposed to be an array. - boolean isArraySchema; - if (schema.hasNonNull(REF_KEY)) { - // If the schema uses a reference type, then it's not an array schema. - isArraySchema = false; - } else if (schema.hasNonNull(TYPE_KEY)) { - // If the schema declares {type: array} or {type: [..., array, ...]} - // Then this is an array schema - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - isArraySchema = false; - for (final JsonNode typeItem : typeNode) { - if (ARRAY_TYPE.equals(typeItem.asText())) { - isArraySchema = true; - } - } - } else { - isArraySchema = ARRAY_TYPE.equals(typeNode.asText()); - } - } else { - // If the schema doesn't declare a type at all (which is bad practice, but let's handle it anyway) - // Then check for an items entry, and assume that this is an array if it's present - isArraySchema = schema.hasNonNull(ITEMS_KEY); - } - - if (!isArraySchema) { - return new MigratedNode(data, false); - } else { - final ArrayNode mutatedItems = Jsons.arrayNode(); - final JsonNode itemsNode = schema.get(ITEMS_KEY); - if (itemsNode == null) { - // We _could_ check additionalItems, but much like the additionalProperties comment for objects: - // it's a lot of work for no payoff - return new MigratedNode(data, true); - } else if (itemsNode.isArray()) { - // In the case of {items: [schema1, schema2, ...]} - // We need to check schema1 against the first element of the array, - // schema2 against the second element, etc. - boolean allSchemasMatched = true; - for (int i = 0; i < data.size(); i++) { - final JsonNode element = data.get(i); - if (itemsNode.size() > i) { - // If we have a schema for this element, then try mutating the element - final MigratedNode mutatedElement = mutateDataNode(validator, schemaMatcher, transformer, element, itemsNode.get(i)); - if (!mutatedElement.matchedSchema()) { - allSchemasMatched = false; - } - mutatedItems.add(mutatedElement.node()); - } - } - // If there were more elements in `data` than there were schemas in `itemsNode`, - // then just blindly add the rest of those elements. - for (int i = itemsNode.size(); i < data.size(); i++) { - mutatedItems.add(data.get(i)); - } - return new MigratedNode(mutatedItems, allSchemasMatched); - } else { - // IN the case of {items: schema}, we just check every array element against that schema. - boolean matchedSchema = true; - for (final JsonNode item : data) { - final MigratedNode migratedNode = mutateDataNode(validator, schemaMatcher, transformer, item, itemsNode); - mutatedItems.add(migratedNode.node); - if (!migratedNode.matchedSchema) { - matchedSchema = false; - } - } - return new MigratedNode(mutatedItems, matchedSchema); - } - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java deleted file mode 100644 index f689600fdabf..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.function.Consumer; -import java.util.function.Function; - -/** - * Utility class for recursively modifying JsonSchemas. Useful for up/downgrading AirbyteCatalog - * objects. - * - * See {@link io.airbyte.commons.protocol.migrations.v1.SchemaMigrationV1} for example usage. - */ -public class SchemaMigrations { - - /** - * Generic utility method that recurses through all type declarations in the schema. For each type - * declaration that are accepted by matcher, mutate them using transformer. For all other type - * declarations, recurse into their subschemas (if any). - *

- * Note that this modifies the schema in-place. Callers who need a copy of the old schema should - * save schema.deepCopy() before calling this method. - * - * @param schema The JsonSchema node to walk down - * @param matcher A function which returns true on any schema node that needs to be transformed - * @param transformer A function which mutates a schema node - */ - public static void mutateSchemas(final Function matcher, final Consumer transformer, final JsonNode schema) { - if (schema.isBoolean()) { - // We never want to modify a schema of `true` or `false` (e.g. additionalProperties: true) - // so just return immediately - return; - } - if (matcher.apply(schema)) { - // Base case: If this schema should be mutated, then we need to mutate it - transformer.accept(schema); - } else { - // Otherwise, we need to find all the subschemas and mutate them. - // technically, it might be more correct to do something like: - // if schema["type"] == "array": find subschemas for items, additionalItems, contains - // else if schema["type"] == "object": find subschemas for properties, patternProperties, - // additionalProperties - // else if oneof, allof, etc - // but that sounds really verbose for no real benefit - final List subschemas = findSubschemas(schema); - - // recurse into each subschema - for (final JsonNode subschema : subschemas) { - mutateSchemas(matcher, transformer, subschema); - } - } - } - - /** - * Returns a list of all the direct children nodes to consider for subSchemas - * - * @param schema The JsonSchema node to start - * @return a list of the JsonNodes to be considered - */ - public static List findSubschemas(final JsonNode schema) { - final List subschemas = new ArrayList<>(); - - // array schemas - findSubschemas(subschemas, schema, "items"); - findSubschemas(subschemas, schema, "additionalItems"); - findSubschemas(subschemas, schema, "contains"); - - // object schemas - if (schema.hasNonNull("properties")) { - final ObjectNode propertiesNode = (ObjectNode) schema.get("properties"); - final Iterator> propertiesIterator = propertiesNode.fields(); - while (propertiesIterator.hasNext()) { - final Entry property = propertiesIterator.next(); - subschemas.add(property.getValue()); - } - } - if (schema.hasNonNull("patternProperties")) { - final ObjectNode propertiesNode = (ObjectNode) schema.get("patternProperties"); - final Iterator> propertiesIterator = propertiesNode.fields(); - while (propertiesIterator.hasNext()) { - final Entry property = propertiesIterator.next(); - subschemas.add(property.getValue()); - } - } - findSubschemas(subschemas, schema, "additionalProperties"); - - // combining restrictions - destinations have limited support for these, but we should handle the - // schemas correctly anyway - findSubschemas(subschemas, schema, "allOf"); - findSubschemas(subschemas, schema, "oneOf"); - findSubschemas(subschemas, schema, "anyOf"); - findSubschemas(subschemas, schema, "not"); - - return subschemas; - } - - /** - * If schema contains key, then grab the subschema(s) at schema[key] and add them to the subschemas - * list. - *

- * For example: - *

    - *
  • schema = {"items": [{"type": "string}]} - *

    - * key = "items" - *

    - * -> add {"type": "string"} to subschemas

    • - *
  • schema = {"items": {"type": "string"}} - *

    - * key = "items" - *

    - * -> add {"type": "string"} to subschemas

    • - *
  • schema = {"additionalProperties": true} - *

    - * key = "additionalProperties" - *

    - * -> add nothing to subschemas - *

    - * (technically `true` is a valid JsonSchema, but we don't want to modify it)

    • - *
- */ - public static void findSubschemas(final List subschemas, final JsonNode schema, final String key) { - if (schema.hasNonNull(key)) { - final JsonNode subschemaNode = schema.get(key); - if (subschemaNode.isArray()) { - for (final JsonNode subschema : subschemaNode) { - subschemas.add(subschema); - } - } else if (subschemaNode.isObject()) { - subschemas.add(subschemaNode); - } - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java deleted file mode 100644 index 76c14cdc54a0..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.protocol.migrations.util.RecordMigrations; -import io.airbyte.commons.protocol.migrations.util.RecordMigrations.MigratedNode; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.JsonSchemaReferenceTypes; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; - -// Disable V1 Migration, uncomment to re-enable -// @Singleton -public class AirbyteMessageMigrationV1 implements AirbyteMessageMigration { - - private final JsonSchemaValidator validator; - - public AirbyteMessageMigrationV1() { - this(new JsonSchemaValidator()); - } - - @VisibleForTesting - public AirbyteMessageMigrationV1(final JsonSchemaValidator validator) { - this.validator = validator; - } - - @Override - public io.airbyte.protocol.models.v0.AirbyteMessage downgrade(final AirbyteMessage oldMessage, - final Optional configuredAirbyteCatalog) { - final io.airbyte.protocol.models.v0.AirbyteMessage newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - io.airbyte.protocol.models.v0.AirbyteMessage.class); - if (oldMessage.getType() == Type.CATALOG && oldMessage.getCatalog() != null) { - for (final io.airbyte.protocol.models.v0.AirbyteStream stream : newMessage.getCatalog().getStreams()) { - final JsonNode schema = stream.getJsonSchema(); - SchemaMigrationV1.downgradeSchema(schema); - } - } else if (oldMessage.getType() == Type.RECORD && oldMessage.getRecord() != null) { - if (configuredAirbyteCatalog.isPresent()) { - final ConfiguredAirbyteCatalog catalog = configuredAirbyteCatalog.get(); - final io.airbyte.protocol.models.v0.AirbyteRecordMessage record = newMessage.getRecord(); - final Optional maybeStream = catalog.getStreams().stream() - .filter(stream -> Objects.equals(stream.getStream().getName(), record.getStream()) - && Objects.equals(stream.getStream().getNamespace(), record.getNamespace())) - .findFirst(); - // If this record doesn't belong to any configured stream, then there's no point downgrading it - // So only do the downgrade if we can find its stream - if (maybeStream.isPresent()) { - final JsonNode schema = maybeStream.get().getStream().getJsonSchema(); - final JsonNode oldData = record.getData(); - final MigratedNode downgradedNode = downgradeRecord(oldData, schema); - record.setData(downgradedNode.node()); - } - } - } - return newMessage; - } - - @Override - public AirbyteMessage upgrade(final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage, - final Optional configuredAirbyteCatalog) { - // We're not introducing any changes to the structure of the record/catalog - // so just clone a new message object, which we can edit in-place - final AirbyteMessage newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - AirbyteMessage.class); - if (oldMessage.getType() == io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG && oldMessage.getCatalog() != null) { - for (final AirbyteStream stream : newMessage.getCatalog().getStreams()) { - final JsonNode schema = stream.getJsonSchema(); - SchemaMigrationV1.upgradeSchema(schema); - } - } else if (oldMessage.getType() == io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD && oldMessage.getRecord() != null) { - final JsonNode oldData = newMessage.getRecord().getData(); - final JsonNode newData = upgradeRecord(oldData); - newMessage.getRecord().setData(newData); - } - return newMessage; - } - - /** - * Returns a copy of oldData, with numeric values converted to strings. String and boolean values - * are returned as-is for convenience, i.e. this is not a true deep copy. - */ - private static JsonNode upgradeRecord(final JsonNode oldData) { - if (oldData.isNumber()) { - // Base case: convert numbers to strings - return Jsons.convertValue(oldData.asText(), TextNode.class); - } else if (oldData.isObject()) { - // Recurse into each field of the object - final ObjectNode newData = (ObjectNode) Jsons.emptyObject(); - - final Iterator> fieldsIterator = oldData.fields(); - while (fieldsIterator.hasNext()) { - final Entry next = fieldsIterator.next(); - final String key = next.getKey(); - final JsonNode value = next.getValue(); - - final JsonNode newValue = upgradeRecord(value); - newData.set(key, newValue); - } - - return newData; - } else if (oldData.isArray()) { - // Recurse into each element of the array - final ArrayNode newData = Jsons.arrayNode(); - for (final JsonNode element : oldData) { - newData.add(upgradeRecord(element)); - } - return newData; - } else { - // Base case: this is a string or boolean, so we don't need to modify it - return oldData; - } - } - - /** - * We need the schema to recognize which fields are integers, since it would be wrong to just assume - * any numerical string should be parsed out. - * - * Works on a best-effort basis. If the schema doesn't match the data, we'll do our best to - * downgrade anything that we can definitively say is a number. Should _not_ throw an exception if - * bad things happen (e.g. we try to parse a non-numerical string as a number). - */ - private MigratedNode downgradeRecord(final JsonNode data, final JsonNode schema) { - return RecordMigrations.mutateDataNode( - validator, - s -> { - if (s.hasNonNull(REF_KEY)) { - final String type = s.get(REF_KEY).asText(); - return JsonSchemaReferenceTypes.INTEGER_REFERENCE.equals(type) - || JsonSchemaReferenceTypes.NUMBER_REFERENCE.equals(type); - } else { - return false; - } - }, - (s, d) -> { - if (d.asText().matches("-?\\d+(\\.\\d+)?")) { - // If this string is a numeric literal, convert it to a numeric node. - return new MigratedNode(Jsons.deserialize(d.asText()), true); - } else { - // Otherwise, just leave the node unchanged. - return new MigratedNode(d, false); - } - }, - data, schema); - } - - @Override - public Version getPreviousVersion() { - return AirbyteProtocolVersion.V0; - } - - @Override - public Version getCurrentVersion() { - return AirbyteProtocolVersion.V1; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java deleted file mode 100644 index fb627ef422f4..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.protocol.migrations.util.SchemaMigrations; -import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; - -/** - * For the v0 to v1 migration, it appears that we are persisting some protocol objects without - * version. Until this gets addressed more properly, this class contains the helper functions used - * to handle this on the fly migration. - * - * Once persisted objects are versioned, this code should be deleted. - */ -public class CatalogMigrationV1Helper { - - /** - * Performs an in-place migration of the schema from v0 to v1 if v0 data types are detected - * - * @param configuredAirbyteCatalog to migrate - */ - public static void upgradeSchemaIfNeeded(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (containsV0DataTypes(configuredAirbyteCatalog)) { - upgradeSchema(configuredAirbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 if v0 data types are detected - * - * @param airbyteCatalog to migrate - */ - public static void upgradeSchemaIfNeeded(final AirbyteCatalog airbyteCatalog) { - if (containsV0DataTypes(airbyteCatalog)) { - upgradeSchema(airbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 - * - * @param configuredAirbyteCatalog to migrate - */ - private static void upgradeSchema(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - for (final var stream : configuredAirbyteCatalog.getStreams()) { - SchemaMigrationV1.upgradeSchema(stream.getStream().getJsonSchema()); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 - * - * @param airbyteCatalog to migrate - */ - private static void upgradeSchema(final AirbyteCatalog airbyteCatalog) { - for (final var stream : airbyteCatalog.getStreams()) { - SchemaMigrationV1.upgradeSchema(stream.getJsonSchema()); - } - } - - /** - * Returns true if catalog contains v0 data types - */ - private static boolean containsV0DataTypes(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (configuredAirbyteCatalog == null) { - return false; - } - - return configuredAirbyteCatalog - .getStreams() - .stream().findFirst() - .map(ConfiguredAirbyteStream::getStream) - .map(CatalogMigrationV1Helper::streamContainsV0DataTypes) - .orElse(false); - } - - /** - * Returns true if catalog contains v0 data types - */ - private static boolean containsV0DataTypes(final AirbyteCatalog airbyteCatalog) { - if (airbyteCatalog == null) { - return false; - } - - return airbyteCatalog - .getStreams() - .stream().findFirst() - .map(CatalogMigrationV1Helper::streamContainsV0DataTypes) - .orElse(false); - } - - private static boolean streamContainsV0DataTypes(final AirbyteStream airbyteStream) { - if (airbyteStream == null || airbyteStream.getJsonSchema() == null) { - return false; - } - return hasV0DataType(airbyteStream.getJsonSchema()); - } - - /** - * Performs of search of a v0 data type node, returns true at the first node found. - */ - private static boolean hasV0DataType(final JsonNode schema) { - if (SchemaMigrationV1.isPrimitiveTypeDeclaration(schema)) { - return true; - } - - for (final JsonNode subSchema : SchemaMigrations.findSubschemas(schema)) { - if (hasV0DataType(subSchema)) { - return true; - } - } - return false; - } - - /** - * Performs an in-place migration of the schema from v1 to v0 if v1 data types are detected - * - * @param configuredAirbyteCatalog to migrate - */ - public static void downgradeSchemaIfNeeded(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (containsV1DataTypes(configuredAirbyteCatalog)) { - downgradeSchema(configuredAirbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 if v1 data types are detected - * - * @param airbyteCatalog to migrate - */ - public static void downgradeSchemaIfNeeded(final AirbyteCatalog airbyteCatalog) { - if (containsV1DataTypes(airbyteCatalog)) { - downgradeSchema(airbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 - * - * @param configuredAirbyteCatalog to migrate - */ - private static void downgradeSchema(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - for (final var stream : configuredAirbyteCatalog.getStreams()) { - SchemaMigrationV1.downgradeSchema(stream.getStream().getJsonSchema()); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 - * - * @param airbyteCatalog to migrate - */ - private static void downgradeSchema(final AirbyteCatalog airbyteCatalog) { - for (final var stream : airbyteCatalog.getStreams()) { - SchemaMigrationV1.downgradeSchema(stream.getJsonSchema()); - } - } - - /** - * Returns true if catalog contains v1 data types - */ - private static boolean containsV1DataTypes(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (configuredAirbyteCatalog == null) { - return false; - } - - return configuredAirbyteCatalog - .getStreams() - .stream().findFirst() - .map(ConfiguredAirbyteStream::getStream) - .map(CatalogMigrationV1Helper::streamContainsV1DataTypes) - .orElse(false); - } - - /** - * Returns true if catalog contains v1 data types - */ - private static boolean containsV1DataTypes(final AirbyteCatalog airbyteCatalog) { - if (airbyteCatalog == null) { - return false; - } - - return airbyteCatalog - .getStreams() - .stream().findFirst() - .map(CatalogMigrationV1Helper::streamContainsV1DataTypes) - .orElse(false); - } - - private static boolean streamContainsV1DataTypes(final AirbyteStream airbyteStream) { - if (airbyteStream == null || airbyteStream.getJsonSchema() == null) { - return false; - } - return hasV1DataType(airbyteStream.getJsonSchema()); - } - - /** - * Performs of search of a v0 data type node, returns true at the first node found. - */ - private static boolean hasV1DataType(final JsonNode schema) { - if (SchemaMigrationV1.isPrimitiveReferenceTypeDeclaration(schema)) { - return true; - } - - for (final JsonNode subSchema : SchemaMigrations.findSubschemas(schema)) { - if (hasV1DataType(subSchema)) { - return true; - } - } - return false; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java deleted file mode 100644 index 2f56e823b247..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.ConfiguredAirbyteCatalogMigration; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; - -// Disable V1 Migration, uncomment to re-enable -// @Singleton -public class ConfiguredAirbyteCatalogMigrationV1 - implements ConfiguredAirbyteCatalogMigration { - - @Override - public io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgrade(final ConfiguredAirbyteCatalog oldMessage) { - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog.class); - for (final io.airbyte.protocol.models.v0.ConfiguredAirbyteStream stream : newMessage.getStreams()) { - final JsonNode schema = stream.getStream().getJsonSchema(); - SchemaMigrationV1.downgradeSchema(schema); - } - return newMessage; - } - - @Override - public ConfiguredAirbyteCatalog upgrade(final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog oldMessage) { - final ConfiguredAirbyteCatalog newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - ConfiguredAirbyteCatalog.class); - for (final ConfiguredAirbyteStream stream : newMessage.getStreams()) { - final JsonNode schema = stream.getStream().getJsonSchema(); - SchemaMigrationV1.upgradeSchema(schema); - } - return newMessage; - } - - @Override - public Version getPreviousVersion() { - return AirbyteProtocolVersion.V0; - } - - @Override - public Version getCurrentVersion() { - return AirbyteProtocolVersion.V1; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java deleted file mode 100644 index e60fec3a571c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ONEOF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.TYPE_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.util.SchemaMigrations; -import io.airbyte.protocol.models.JsonSchemaReferenceTypes; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.StreamSupport; - -public class SchemaMigrationV1 { - - /** - * Perform the {type: foo} -> {$ref: foo} upgrade. Modifies the schema in-place. - */ - public static void upgradeSchema(final JsonNode schema) { - SchemaMigrations.mutateSchemas( - SchemaMigrationV1::isPrimitiveTypeDeclaration, - SchemaMigrationV1::upgradeTypeDeclaration, - schema); - } - - /** - * Perform the {$ref: foo} -> {type: foo} downgrade. Modifies the schema in-place. - */ - public static void downgradeSchema(final JsonNode schema) { - SchemaMigrations.mutateSchemas( - SchemaMigrationV1::isPrimitiveReferenceTypeDeclaration, - SchemaMigrationV1::downgradeTypeDeclaration, - schema); - } - - /** - * Detects any schema that looks like a primitive type declaration, e.g.: { "type": "string" } or { - * "type": ["string", "object"] } - */ - static boolean isPrimitiveTypeDeclaration(final JsonNode schema) { - if (!schema.isObject() || !schema.hasNonNull(TYPE_KEY)) { - return false; - } - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - return StreamSupport.stream(typeNode.spliterator(), false) - .anyMatch(n -> JsonSchemaReferenceTypes.PRIMITIVE_JSON_TYPES.contains(n.asText())); - } else { - return JsonSchemaReferenceTypes.PRIMITIVE_JSON_TYPES.contains(typeNode.asText()); - } - } - - /** - * Detects any schema that looks like a reference type declaration, e.g.: { "$ref": - * "WellKnownTypes.json...." } or { "oneOf": [{"$ref": "..."}, {"type": "object"}] } - */ - static boolean isPrimitiveReferenceTypeDeclaration(final JsonNode schema) { - if (!schema.isObject()) { - // Non-object schemas (i.e. true/false) never need to be modified - return false; - } else if (schema.hasNonNull(REF_KEY) && schema.get(REF_KEY).asText().startsWith("WellKnownTypes.json")) { - // If this schema has a $ref, then we need to convert it back to type/airbyte_type/format - return true; - } else if (schema.hasNonNull(ONEOF_KEY)) { - // If this is a oneOf with at least one primitive $ref option, then we should consider converting it - // back - final List subschemas = getSubschemas(schema, ONEOF_KEY); - return subschemas.stream().anyMatch( - subschema -> subschema.hasNonNull(REF_KEY) - && subschema.get(REF_KEY).asText().startsWith("WellKnownTypes.json")); - } else { - return false; - } - } - - /** - * Modifies the schema in-place to upgrade from the old-style type declaration to the new-style $ref - * declaration. Assumes that the schema is an ObjectNode containing a primitive declaration, i.e. - * either something like: {"type": "string"} or: {"type": ["string", "object"]} - *

- * In the latter case, the schema may contain subschemas. This method mutually recurses with - * {@link SchemaMigrations#mutateSchemas(Function, Consumer, JsonNode)} to upgrade those subschemas. - * - * @param schema An ObjectNode representing a primitive type declaration - */ - private static void upgradeTypeDeclaration(final JsonNode schema) { - final ObjectNode schemaNode = (ObjectNode) schema; - - if (schemaNode.hasNonNull("airbyte_type")) { - // If airbyte_type is defined, always respect it - final String referenceType = JsonSchemaReferenceTypes.LEGACY_AIRBYTE_PROPERY_TO_REFERENCE.get(schemaNode.get("airbyte_type").asText()); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // Otherwise, fall back to type/format - final JsonNode typeNode = schemaNode.get(TYPE_KEY); - if (typeNode.isTextual()) { - // If the type is a single string, then replace this node with the appropriate reference type - final String type = typeNode.asText(); - final String referenceType = getReferenceType(type, schemaNode); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // If type is an array of strings, then things are more complicated - final List types = StreamSupport.stream(typeNode.spliterator(), false) - .map(JsonNode::asText) - // Everything is implicitly nullable by just not declaring the `required `field - // so filter out any explicit null types - .filter(type -> !"null".equals(type)) - .toList(); - final boolean exactlyOneType = types.size() == 1; - if (exactlyOneType) { - // If there's only one type, e.g. {type: [string]}, just treat that as equivalent to {type: string} - final String type = types.get(0); - final String referenceType = getReferenceType(type, schemaNode); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // If there are multiple types, we'll need to convert this to a oneOf. - // For arrays and objects, we do a mutual recursion back into mutateSchemas to upgrade their - // subschemas. - final ArrayNode oneOfOptions = Jsons.arrayNode(); - for (final String type : types) { - final ObjectNode option = (ObjectNode) Jsons.emptyObject(); - switch (type) { - case "array" -> { - option.put(TYPE_KEY, "array"); - copyKey(schemaNode, option, "items"); - copyKey(schemaNode, option, "additionalItems"); - copyKey(schemaNode, option, "contains"); - upgradeSchema(option); - } - case "object" -> { - option.put(TYPE_KEY, "object"); - copyKey(schemaNode, option, "properties"); - copyKey(schemaNode, option, "patternProperties"); - copyKey(schemaNode, option, "additionalProperties"); - upgradeSchema(option); - } - default -> { - final String referenceType = getReferenceType(type, schemaNode); - option.put(REF_KEY, referenceType); - } - } - oneOfOptions.add(option); - } - schemaNode.removeAll(); - schemaNode.set(ONEOF_KEY, oneOfOptions); - } - } - } - } - - /** - * Modifies the schema in-place to downgrade from the new-style $ref declaration to the old-style - * type declaration. Assumes that the schema is an ObjectNode containing a primitive declaration, - * i.e. either something like: {"$ref": "WellKnownTypes..."} or: {"oneOf": [{"$ref": - * "WellKnownTypes..."}, ...]} - *

- * In the latter case, the schema may contain subschemas. This method mutually recurses with - * {@link SchemaMigrations#mutateSchemas(Function, Consumer, JsonNode)} to downgrade those - * subschemas. - * - * @param schema An ObjectNode representing a primitive type declaration - */ - private static void downgradeTypeDeclaration(final JsonNode schema) { - if (schema.hasNonNull(REF_KEY)) { - // If this is a direct type declaration, then we can just replace it with the old-style declaration - final String referenceType = schema.get(REF_KEY).asText(); - ((ObjectNode) schema).removeAll(); - ((ObjectNode) schema).setAll(JsonSchemaReferenceTypes.REFERENCE_TYPE_TO_OLD_TYPE.get(referenceType)); - } else if (schema.hasNonNull(ONEOF_KEY)) { - // If this is a oneOf, then we need to check whether we can recombine it into a single type - // declaration. - // This means we must do three things: - // 1. Downgrade each subschema - // 2. Build a new `type` array, containing the `type` of each subschema - // 3. Combine all the fields in each subschema (properties, items, etc) - // If any two subschemas have the same `type`, or the same field, then we can't combine them, but we - // should still downgrade them. - // See V0ToV1MigrationTest.CatalogDowngradeTest#testDowngradeMultiTypeFields for some examples. - - // We'll build up a node containing the combined subschemas. - final ObjectNode replacement = (ObjectNode) Jsons.emptyObject(); - // As part of this, we need to build up a list of `type` entries. For ease of access, we'll keep it - // in a List. - final List types = new ArrayList<>(); - - boolean canRecombineSubschemas = true; - for (final JsonNode subschemaNode : schema.get(ONEOF_KEY)) { - // No matter what - we always need to downgrade the subschema node. - downgradeSchema(subschemaNode); - - if (subschemaNode instanceof ObjectNode subschema) { - // If this subschema is an object, then we can attempt to combine it with the other subschemas. - - // First, update our list of types. - final JsonNode subschemaType = subschema.get(TYPE_KEY); - if (subschemaType != null) { - if (types.contains(subschemaType.asText())) { - // If another subschema has the same type, then we can't combine them. - canRecombineSubschemas = false; - } else { - types.add(subschemaType.asText()); - } - } - - // Then, update the combined schema with this subschema's fields. - if (canRecombineSubschemas) { - final Iterator> fields = subschema.fields(); - while (fields.hasNext()) { - final Entry field = fields.next(); - if (TYPE_KEY.equals(field.getKey())) { - // We're handling the `type` field outside this loop, so ignore it here. - continue; - } - if (replacement.has(field.getKey())) { - // A previous subschema is already using this field, so we should stop trying to combine them. - canRecombineSubschemas = false; - break; - } else { - replacement.set(field.getKey(), field.getValue()); - } - } - } - } else { - // If this subschema is a boolean, then the oneOf is doing something funky, and we shouldn't attempt - // to - // combine it into a single type entry - canRecombineSubschemas = false; - } - } - - if (canRecombineSubschemas) { - // Update our replacement node with the full list of types - final ArrayNode typeNode = Jsons.arrayNode(); - types.forEach(typeNode::add); - replacement.set(TYPE_KEY, typeNode); - - // And commit our changes to the actual schema node - ((ObjectNode) schema).removeAll(); - ((ObjectNode) schema).setAll(replacement); - } - } - } - - private static void copyKey(final ObjectNode source, final ObjectNode target, final String key) { - if (source.hasNonNull(key)) { - target.set(key, source.get(key)); - } - } - - /** - * Given a primitive (string/int/num/bool) type declaration _without_ an airbyte_type, get the - * appropriate $ref type. In most cases, this only depends on the "type" key. When type=string, also - * checks the "format" key. - */ - private static String getReferenceType(final String type, final ObjectNode schemaNode) { - return switch (type) { - case "string" -> { - if (schemaNode.hasNonNull("format")) { - yield switch (schemaNode.get("format").asText()) { - case "date" -> JsonSchemaReferenceTypes.DATE_REFERENCE; - // In these two cases, we default to the "with timezone" type, rather than "without timezone". - // This matches existing behavior in normalization. - case "date-time" -> JsonSchemaReferenceTypes.TIMESTAMP_WITH_TIMEZONE_REFERENCE; - case "time" -> JsonSchemaReferenceTypes.TIME_WITH_TIMEZONE_REFERENCE; - // If we don't recognize the format, just use a plain string - default -> JsonSchemaReferenceTypes.STRING_REFERENCE; - }; - } else if (schemaNode.hasNonNull("contentEncoding")) { - if ("base64".equals(schemaNode.get("contentEncoding").asText())) { - yield JsonSchemaReferenceTypes.BINARY_DATA_REFERENCE; - } else { - yield JsonSchemaReferenceTypes.STRING_REFERENCE; - } - } else { - yield JsonSchemaReferenceTypes.STRING_REFERENCE; - } - } - case "integer" -> JsonSchemaReferenceTypes.INTEGER_REFERENCE; - case "number" -> JsonSchemaReferenceTypes.NUMBER_REFERENCE; - case "boolean" -> JsonSchemaReferenceTypes.BOOLEAN_REFERENCE; - // This is impossible, because we'll only call this method on string/integer/number/boolean - default -> throw new IllegalStateException("Somehow got non-primitive type: " + type + " for schema: " + schemaNode); - }; - } - - private static List getSubschemas(final JsonNode schema, final String key) { - final List subschemas = new ArrayList<>(); - SchemaMigrations.findSubschemas(subschemas, schema, key); - return subschemas; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java deleted file mode 100644 index e641a172e4c7..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.version.Version; - -public interface AirbyteMessageDeserializer { - - T deserialize(final JsonNode json); - - Version getTargetVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java deleted file mode 100644 index 8be50450793b..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import lombok.Getter; - -public class AirbyteMessageGenericDeserializer implements AirbyteMessageDeserializer { - - @Getter - final Version targetVersion; - final Class typeClass; - - public AirbyteMessageGenericDeserializer(final Version targetVersion, final Class typeClass) { - this.targetVersion = targetVersion; - this.typeClass = typeClass; - } - - @Override - public T deserialize(JsonNode json) { - return Jsons.object(json, typeClass); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java deleted file mode 100644 index ad43e35c1d14..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@AllArgsConstructor -public class AirbyteMessageGenericSerializer implements AirbyteMessageSerializer { - - @Getter - private final Version targetVersion; - - @Override - public String serialize(T message) { - return Jsons.serialize(message); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java deleted file mode 100644 index 05b2b4e40834..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.Version; - -public interface AirbyteMessageSerializer { - - String serialize(final T message); - - Version getTargetVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java deleted file mode 100644 index 4f91e74aa4f0..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; - -@Singleton -public class AirbyteMessageV0Deserializer extends AirbyteMessageGenericDeserializer { - - public AirbyteMessageV0Deserializer() { - super(AirbyteProtocolVersion.V0, AirbyteMessage.class); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java deleted file mode 100644 index e50fa17a67fd..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; - -@Singleton -public class AirbyteMessageV0Serializer extends AirbyteMessageGenericSerializer { - - public AirbyteMessageV0Serializer() { - super(AirbyteProtocolVersion.V0); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java deleted file mode 100644 index e0708530f17e..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; - -@Singleton -public class AirbyteMessageV1Deserializer extends AirbyteMessageGenericDeserializer { - - public AirbyteMessageV1Deserializer() { - super(AirbyteProtocolVersion.V1, AirbyteMessage.class); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java deleted file mode 100644 index 2664042d2998..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; -import jakarta.inject.Singleton; - -@Singleton -public class AirbyteMessageV1Serializer extends AirbyteMessageGenericSerializer { - - public AirbyteMessageV1Serializer() { - super(AirbyteProtocolVersion.V1); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java deleted file mode 100644 index 25ea40bd8b03..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AirbyteMessageMigratorTest { - - static final Version v0 = new Version("0.0.0"); - static final Version v1 = new Version("1.0.0"); - static final Version v2 = new Version("2.0.0"); - - record ObjectV0(String name0) {} - - record ObjectV1(String name1) {} - - record ObjectV2(String name2) {} - - static class Migrate0to1 implements AirbyteMessageMigration { - - @Override - public ObjectV0 downgrade(ObjectV1 message, Optional configuredAirbyteCatalog) { - return new ObjectV0(message.name1); - } - - @Override - public ObjectV1 upgrade(ObjectV0 message, Optional configuredAirbyteCatalog) { - return new ObjectV1(message.name0); - } - - @Override - public Version getPreviousVersion() { - return v0; - } - - @Override - public Version getCurrentVersion() { - return v1; - } - - } - - static class Migrate1to2 implements AirbyteMessageMigration { - - @Override - public ObjectV1 downgrade(ObjectV2 message, Optional configuredAirbyteCatalog) { - return new ObjectV1(message.name2); - } - - @Override - public ObjectV2 upgrade(ObjectV1 message, Optional configuredAirbyteCatalog) { - return new ObjectV2(message.name1); - } - - @Override - public Version getPreviousVersion() { - return v1; - } - - @Override - public Version getCurrentVersion() { - return v2; - } - - } - - AirbyteMessageMigrator migrator; - - @BeforeEach - void beforeEach() { - migrator = new AirbyteMessageMigrator( - List.of(new Migrate0to1(), new Migrate1to2())); - migrator.initialize(); - } - - @Test - void testDowngrade() { - final ObjectV2 obj = new ObjectV2("my name"); - - final ObjectV0 objDowngradedTo0 = migrator.downgrade(obj, v0, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo0.name0); - - final ObjectV1 objDowngradedTo1 = migrator.downgrade(obj, v1, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo1.name1); - - final ObjectV2 objDowngradedTo2 = migrator.downgrade(obj, v2, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo2.name2); - } - - @Test - void testUpgrade() { - final ObjectV0 obj0 = new ObjectV0("my name 0"); - final ObjectV2 objUpgradedFrom0 = migrator.upgrade(obj0, v0, Optional.empty()); - assertEquals(obj0.name0, objUpgradedFrom0.name2); - - final ObjectV1 obj1 = new ObjectV1("my name 1"); - final ObjectV2 objUpgradedFrom1 = migrator.upgrade(obj1, v1, Optional.empty()); - assertEquals(obj1.name1, objUpgradedFrom1.name2); - - final ObjectV2 obj2 = new ObjectV2("my name 2"); - final ObjectV2 objUpgradedFrom2 = migrator.upgrade(obj2, v2, Optional.empty()); - assertEquals(obj2.name2, objUpgradedFrom2.name2); - } - - @Test - void testUnsupportedDowngradeShouldFailExplicitly() { - assertThrows(RuntimeException.class, () -> { - migrator.downgrade(new ObjectV2("woot"), new Version("5.0.0"), Optional.empty()); - }); - } - - @Test - void testUnsupportedUpgradeShouldFailExplicitly() { - assertThrows(RuntimeException.class, () -> { - migrator.upgrade(new ObjectV0("woot"), new Version("4.0.0"), Optional.empty()); - }); - } - - @Test - void testRegisterCollisionsShouldFail() { - assertThrows(RuntimeException.class, () -> { - migrator = new AirbyteMessageMigrator( - List.of(new Migrate0to1(), new Migrate1to2(), new Migrate0to1())); - migrator.initialize(); - }); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java deleted file mode 100644 index 8964e36b6dde..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import jakarta.inject.Inject; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.junit.jupiter.api.Test; - -@MicronautTest -class AirbyteMessageSerDeProviderMicronautTest { - - @Inject - AirbyteMessageSerDeProvider serDeProvider; - - @Test - void testSerDeInjection() { - // This should contain the list of all the supported majors of the airbyte protocol - final Set expectedVersions = new HashSet<>(List.of("0", "1")); - - assertEquals(expectedVersions, serDeProvider.getDeserializerKeys()); - assertEquals(expectedVersions, serDeProvider.getSerializerKeys()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java deleted file mode 100644 index 2e0717a0176d..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.commons.version.Version; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AirbyteMessageSerDeProviderTest { - - AirbyteMessageSerDeProvider serDeProvider; - AirbyteMessageDeserializer deserV0; - AirbyteMessageDeserializer deserV1; - - AirbyteMessageSerializer serV0; - AirbyteMessageSerializer serV1; - - @BeforeEach - void beforeEach() { - serDeProvider = new AirbyteMessageSerDeProvider(); - - deserV0 = buildDeserializer(new Version("0.1.0")); - deserV1 = buildDeserializer(new Version("1.1.0")); - serDeProvider.registerDeserializer(deserV0); - serDeProvider.registerDeserializer(deserV1); - - serV0 = buildSerializer(new Version("0.2.0")); - serV1 = buildSerializer(new Version("1.0.0")); - serDeProvider.registerSerializer(serV0); - serDeProvider.registerSerializer(serV1); - } - - @Test - void testGetDeserializer() { - assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new Version("0.1.0"))); - assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new Version("0.2.0"))); - assertEquals(Optional.of(deserV1), serDeProvider.getDeserializer(new Version("1.1.0"))); - assertEquals(Optional.empty(), serDeProvider.getDeserializer(new Version("2.0.0"))); - } - - @Test - void testGetSerializer() { - assertEquals(Optional.of(serV0), serDeProvider.getSerializer(new Version("0.1.0"))); - assertEquals(Optional.of(serV1), serDeProvider.getSerializer(new Version("1.0.0"))); - assertEquals(Optional.empty(), serDeProvider.getSerializer(new Version("3.2.0"))); - } - - @Test - void testRegisterDeserializerShouldFailOnVersionCollision() { - AirbyteMessageDeserializer deser = buildDeserializer(new Version("0.2.0")); - assertThrows(RuntimeException.class, () -> { - serDeProvider.registerDeserializer(deser); - }); - } - - @Test - void testRegisterSerializerShouldFailOnVersionCollision() { - AirbyteMessageSerializer ser = buildSerializer(new Version("0.5.0")); - assertThrows(RuntimeException.class, () -> { - serDeProvider.registerSerializer(ser); - }); - } - - private AirbyteMessageDeserializer buildDeserializer(Version version) { - final AirbyteMessageDeserializer deser = mock(AirbyteMessageDeserializer.class); - when(deser.getTargetVersion()).thenReturn(version); - return deser; - } - - private AirbyteMessageSerializer buildSerializer(Version version) { - final AirbyteMessageSerializer ser = mock(AirbyteMessageSerializer.class); - when(ser.getTargetVersion()).thenReturn(version); - return ser; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java deleted file mode 100644 index 066828386902..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import jakarta.inject.Inject; -import java.util.Set; -import org.junit.jupiter.api.Test; - -@MicronautTest -class MigratorsMicronautTest { - - @Inject - AirbyteMessageMigrator messageMigrator; - - @Inject - ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - - // This should contain the list of all the supported majors of the airbyte protocol except the most - // recent one since the migrations themselves are keyed on the lower version. - final Set SUPPORTED_VERSIONS = Set.of(); - - @Test - void testAirbyteMessageMigrationInjection() { - assertEquals(SUPPORTED_VERSIONS, messageMigrator.getMigrationKeys()); - } - - @Test - void testConfiguredAirbyteCatalogMigrationInjection() { - assertEquals(SUPPORTED_VERSIONS, configuredAirbyteCatalogMigrator.getMigrationKeys()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java deleted file mode 100644 index 4d6905da984a..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java +++ /dev/null @@ -1,1633 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -// most of these tests rely on a doTest utility method for brevity, which hides the assertion. -@SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") -class AirbyteMessageMigrationV1Test { - - JsonSchemaValidator validator; - private AirbyteMessageMigrationV1 migration; - - @BeforeEach - void setup() throws URISyntaxException { - // TODO this should probably just get generated as part of the airbyte-protocol build, and - // airbyte-workers / airbyte-commons-protocol would reference it directly - final URI parentUri = MoreResources.readResourceAsFile("WellKnownTypes.json").getAbsoluteFile().toURI(); - validator = new JsonSchemaValidator(parentUri); - migration = new AirbyteMessageMigrationV1(validator); - } - - @Test - void testVersionMetadata() { - assertEquals("0.3.0", migration.getPreviousVersion().serialize()); - assertEquals("1.0.0", migration.getCurrentVersion().serialize()); - } - - @Nested - class CatalogUpgradeTest { - - @Test - void testBasicUpgrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final JsonNode oldSchema = Jsons.deserialize( - """ - { - "type": "string" - } - """); - - final AirbyteMessage upgradedMessage = migration.upgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "CATALOG", - "catalog": { - "streams": [ - { - "json_schema": { - "$ref": "WellKnownTypes.json#/definitions/String" - } - } - ] - } - } - """, - AirbyteMessage.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testNullUpgrade() { - final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG); - final AirbyteMessage upgradedMessage = migration.upgrade(oldMessage, Optional.empty()); - final AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.CATALOG); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to upgrade the oldSchema, and assert that the result is equal to expectedSchema - * - * @param oldSchemaString The schema to be upgraded - * @param expectedSchemaString The expected schema after upgrading - */ - private void doTest(final String oldSchemaString, final String expectedSchemaString) { - final JsonNode oldSchema = Jsons.deserialize(oldSchemaString); - - final AirbyteMessage upgradedMessage = migration.upgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final JsonNode expectedSchema = Jsons.deserialize(expectedSchemaString); - assertEquals(expectedSchema, upgradedMessage.getCatalog().getStreams().get(0).getJsonSchema()); - } - - @Test - void testUpgradeAllPrimitives() { - doTest( - """ - { - "type": "object", - "properties": { - "example_string": { - "type": "string" - }, - "example_number": { - "type": "number" - }, - "example_integer": { - "type": "integer" - }, - "example_airbyte_integer": { - "type": "number", - "airbyte_type": "integer" - }, - "example_boolean": { - "type": "boolean" - }, - "example_timestamptz": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "example_timestamptz_implicit": { - "type": "string", - "format": "date-time" - }, - "example_timestamp_without_tz": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "example_timez": { - "type": "string", - "format": "time", - "airbyte_type": "time_with_timezone" - }, - "example_timetz_implicit": { - "type": "string", - "format": "time" - }, - "example_time_without_tz": { - "type": "string", - "format": "time", - "airbyte_type": "time_without_timezone" - }, - "example_date": { - "type": "string", - "format": "date" - }, - "example_binary": { - "type": "string", - "contentEncoding": "base64" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "example_string": { - "$ref": "WellKnownTypes.json#/definitions/String" - }, - "example_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" - }, - "example_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_airbyte_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_boolean": { - "$ref": "WellKnownTypes.json#/definitions/Boolean" - }, - "example_timestamptz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamptz_implicit": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamp_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" - }, - "example_timez": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_timetz_implicit": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_time_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" - }, - "example_date": { - "$ref": "WellKnownTypes.json#/definitions/Date" - }, - "example_binary": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" - } - } - } - """); - } - - @Test - void testUpgradeNestedFields() { - doTest( - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"type": "string"} - }, - "tuple_array": { - "items": [ - {"type": "string"}, - {"type": "integer"} - ], - "additionalItems": {"type": "string"}, - "contains": {"type": "integer"} - }, - "nested_object": { - "properties": { - "id": {"type": "integer"}, - "nested_oneof": { - "oneOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_not": { - "not": [ - {"type": "string"}, - {"type": "integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"type": "integer"} - }, - "additionalProperties": {"type": "string"} - } - } - } - """, - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "tuple_array": { - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ], - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "nested_object": { - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "nested_oneof": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_not": { - "not": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """); - } - - @Test - void testUpgradeBooleanSchemas() { - // Most of these should never happen in reality, but let's handle them just in case - // The only ones that we're _really_ expecting are additionalItems and additionalProperties - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": true - }, - "tuple_array": { - "items": [true], - "additionalItems": true, - "contains": true - }, - "nested_object": { - "properties": { - "id": true, - "nested_oneof": { - "oneOf": [true] - }, - "nested_anyof": { - "anyOf": [true] - }, - "nested_allof": { - "allOf": [true] - }, - "nested_not": { - "not": [true] - } - }, - "patternProperties": { - "integer_.*": true - }, - "additionalProperties": true - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeEmptySchema() { - // Sources shouldn't do this, but we should have handling for it anyway, since it's not currently - // enforced by SATs - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {} - }, - "tuple_array": { - "items": [{}], - "additionalItems": {}, - "contains": {} - }, - "nested_object": { - "properties": { - "id": {}, - "nested_oneof": { - "oneOf": [{}] - }, - "nested_anyof": { - "anyOf": [{}] - }, - "nested_allof": { - "allOf": [{}] - }, - "nested_not": { - "not": [{}] - } - }, - "patternProperties": { - "integer_.*": {} - }, - "additionalProperties": {} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeLiteralSchema() { - // Verify that we do _not_ recurse into places we shouldn't - final String schemaString = """ - { - "type": "object", - "properties": { - "example_schema": { - "type": "object", - "default": {"type": "string"}, - "enum": [{"type": "string"}], - "const": {"type": "string"} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeMalformedSchemas() { - // These schemas are "wrong" in some way. For example, normalization will currently treat - // bad_timestamptz as a string timestamp_with_timezone, - // i.e. it will disregard the option for a boolean. - // Generating this sort of schema is just wrong; sources shouldn't do this to begin with. But let's - // verify that we behave mostly correctly here. - doTest( - """ - { - "type": "object", - "properties": { - "bad_timestamptz": { - "type": ["boolean", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "bad_integer": { - "type": "string", - "format": "date-time", - "airbyte_type": "integer" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "bad_timestamptz": {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"}, - "bad_integer": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - """); - } - - @Test - void testUpgradeMultiTypeFields() { - doTest( - """ - { - "type": "object", - "properties": { - "multityped_field": { - "type": ["string", "object", "array"], - "properties": { - "id": {"type": "string"} - }, - "patternProperties": { - "integer_.*": {"type": "integer"} - }, - "additionalProperties": {"type": "string"}, - "items": {"type": "string"}, - "additionalItems": {"type": "string"}, - "contains": {"type": "string"} - }, - "nullable_multityped_field": { - "type": ["null", "string", "array", "object"], - "items": [{"type": "string"}, {"type": "integer"}], - "properties": { - "id": {"type": "integer"} - } - }, - "multityped_date_field": { - "type": ["string", "integer"], - "format": "date" - }, - "sneaky_singletype_field": { - "type": ["string", "null"], - "format": "date-time" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - ] - }, - "nullable_multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "array", - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "multityped_date_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Date"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "sneaky_singletype_field": {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - } - } - """); - } - - private io.airbyte.protocol.models.v0.AirbyteMessage createCatalogMessage(final JsonNode schema) { - return new io.airbyte.protocol.models.v0.AirbyteMessage().withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG) - .withCatalog( - new io.airbyte.protocol.models.v0.AirbyteCatalog().withStreams(List.of(new io.airbyte.protocol.models.v0.AirbyteStream().withJsonSchema( - schema)))); - } - - } - - @Nested - class RecordUpgradeTest { - - @Test - void testBasicUpgrade() { - final JsonNode oldData = Jsons.deserialize( - """ - { - "id": 42 - } - """); - - final AirbyteMessage upgradedMessage = migration.upgrade(createRecordMessage(oldData), Optional.empty()); - - final AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "RECORD", - "record": { - "data": { - "id": "42" - } - } - } - """, - AirbyteMessage.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testNullUpgrade() { - final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD); - final AirbyteMessage upgradedMessage = migration.upgrade(oldMessage, Optional.empty()); - final AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.RECORD); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to upgrade the oldData, and assert that the result is equal to expectedData - * - * @param oldDataString The data of the record to be upgraded - * @param expectedDataString The expected data after upgrading - */ - private void doTest(final String oldDataString, final String expectedDataString) { - final JsonNode oldData = Jsons.deserialize(oldDataString); - - final AirbyteMessage upgradedMessage = migration.upgrade(createRecordMessage(oldData), Optional.empty()); - - final JsonNode expectedData = Jsons.deserialize(expectedDataString); - assertEquals(expectedData, upgradedMessage.getRecord().getData()); - } - - @Test - void testNestedUpgrade() { - doTest( - """ - { - "int": 42, - "float": 42.0, - "float2": 42.2, - "sub_object": { - "sub_int": 42, - "sub_float": 42.0, - "sub_float2": 42.2 - }, - "sub_array": [42, 42.0, 42.2] - } - """, - """ - { - "int": "42", - "float": "42.0", - "float2": "42.2", - "sub_object": { - "sub_int": "42", - "sub_float": "42.0", - "sub_float2": "42.2" - }, - "sub_array": ["42", "42.0", "42.2"] - } - """); - } - - @Test - void testNonUpgradableValues() { - doTest( - """ - { - "boolean": true, - "string": "arst", - "sub_object": { - "boolean": true, - "string": "arst" - }, - "sub_array": [true, "arst"] - } - """, - """ - { - "boolean": true, - "string": "arst", - "sub_object": { - "boolean": true, - "string": "arst" - }, - "sub_array": [true, "arst"] - } - """); - } - - private io.airbyte.protocol.models.v0.AirbyteMessage createRecordMessage(final JsonNode data) { - return new io.airbyte.protocol.models.v0.AirbyteMessage().withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD) - .withRecord(new io.airbyte.protocol.models.v0.AirbyteRecordMessage().withData(data)); - } - - } - - @Nested - class CatalogDowngradeTest { - - @Test - void testBasicDowngrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final JsonNode newSchema = Jsons.deserialize( - """ - { - "$ref": "WellKnownTypes.json#/definitions/String" - } - """); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = migration.downgrade(createCatalogMessage(newSchema), Optional.empty()); - - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "CATALOG", - "catalog": { - "streams": [ - { - "json_schema": { - "type": "string" - } - } - ] - } - } - """, - io.airbyte.protocol.models.v0.AirbyteMessage.class); - assertEquals(expectedMessage, downgradedMessage); - } - - @Test - void testNullDowngrade() { - final AirbyteMessage oldMessage = new AirbyteMessage().withType(Type.CATALOG); - final io.airbyte.protocol.models.v0.AirbyteMessage upgradedMessage = migration.downgrade(oldMessage, Optional.empty()); - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to downgrade the oldSchema, and assert that the result is equal to expectedSchema - * - * @param oldSchemaString The schema to be downgraded - * @param expectedSchemaString The expected schema after downgrading - */ - private void doTest(final String oldSchemaString, final String expectedSchemaString) { - final JsonNode oldSchema = Jsons.deserialize(oldSchemaString); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = migration.downgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final JsonNode expectedSchema = Jsons.deserialize(expectedSchemaString); - assertEquals(expectedSchema, downgradedMessage.getCatalog().getStreams().get(0).getJsonSchema()); - } - - @Test - void testDowngradeAllPrimitives() { - doTest( - """ - { - "type": "object", - "properties": { - "example_string": { - "$ref": "WellKnownTypes.json#/definitions/String" - }, - "example_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" - }, - "example_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_boolean": { - "$ref": "WellKnownTypes.json#/definitions/Boolean" - }, - "example_timestamptz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamp_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" - }, - "example_timez": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_time_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" - }, - "example_date": { - "$ref": "WellKnownTypes.json#/definitions/Date" - }, - "example_binary": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "example_string": { - "type": "string" - }, - "example_number": { - "type": "number" - }, - "example_integer": { - "type": "number", - "airbyte_type": "integer" - }, - "example_boolean": { - "type": "boolean" - }, - "example_timestamptz": { - "type": "string", - "airbyte_type": "timestamp_with_timezone", - "format": "date-time" - }, - "example_timestamp_without_tz": { - "type": "string", - "airbyte_type": "timestamp_without_timezone", - "format": "date-time" - }, - "example_timez": { - "type": "string", - "airbyte_type": "time_with_timezone", - "format": "time" - }, - "example_time_without_tz": { - "type": "string", - "airbyte_type": "time_without_timezone", - "format": "time" - }, - "example_date": { - "type": "string", - "format": "date" - }, - "example_binary": { - "type": "string", - "contentEncoding": "base64" - } - } - } - """); - } - - @Test - void testDowngradeNestedFields() { - doTest( - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "tuple_array": { - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ], - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "nested_object": { - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "nested_oneof": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_not": { - "not": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """, - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"type": "string"} - }, - "tuple_array": { - "items": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ], - "additionalItems": {"type": "string"}, - "contains": {"type": "number", "airbyte_type": "integer"} - }, - "nested_object": { - "properties": { - "id": {"type": "number", "airbyte_type": "integer"}, - "nested_oneof": { - "oneOf": [ - {"type": "string"}, - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - }, - "nested_not": { - "not": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"type": "number", "airbyte_type": "integer"} - }, - "additionalProperties": {"type": "string"} - } - } - } - """); - } - - @Test - void testDowngradeBooleanSchemas() { - // Most of these should never happen in reality, but let's handle them just in case - // The only ones that we're _really_ expecting are additionalItems and additionalProperties - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": true - }, - "tuple_array": { - "items": [true], - "additionalItems": true, - "contains": true - }, - "nested_object": { - "properties": { - "id": true, - "nested_oneof": { - "oneOf": [true] - }, - "nested_anyof": { - "anyOf": [true] - }, - "nested_allof": { - "allOf": [true] - }, - "nested_not": { - "not": [true] - } - }, - "patternProperties": { - "integer_.*": true - }, - "additionalProperties": true - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeEmptySchema() { - // Sources shouldn't do this, but we should have handling for it anyway, since it's not currently - // enforced by SATs - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {} - }, - "tuple_array": { - "items": [{}], - "additionalItems": {}, - "contains": {} - }, - "nested_object": { - "properties": { - "id": {}, - "nested_oneof": { - "oneOf": [{}] - }, - "nested_anyof": { - "anyOf": [{}] - }, - "nested_allof": { - "allOf": [{}] - }, - "nested_not": { - "not": [{}] - } - }, - "patternProperties": { - "integer_.*": {} - }, - "additionalProperties": {} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeLiteralSchema() { - // Verify that we do _not_ recurse into places we shouldn't - final String schemaString = """ - { - "type": "object", - "properties": { - "example_schema": { - "type": "object", - "default": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "enum": [{"$ref": "WellKnownTypes.json#/definitions/String"}], - "const": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeMultiTypeFields() { - doTest( - """ - { - "type": "object", - "properties": { - "multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - ] - }, - "multityped_date_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Date"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "boolean_field": { - "oneOf": [ - true, - {"$ref": "WellKnownTypes.json#/definitions/String"}, - false - ] - }, - "conflicting_field": { - "oneOf": [ - {"type": "object", "properties": {"id": {"$ref": "WellKnownTypes.json#/definitions/String"}}}, - {"type": "object", "properties": {"name": {"$ref": "WellKnownTypes.json#/definitions/String"}}}, - {"$ref": "WellKnownTypes.json#/definitions/String"} - ] - }, - "conflicting_primitives": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone"}, - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - ] - } - } - } - """, - """ - { - "type": "object", - "properties": { - "multityped_field": { - "type": ["string", "object", "array"], - "properties": { - "id": {"type": "string"} - }, - "patternProperties": { - "integer_.*": {"type": "number", "airbyte_type": "integer"} - }, - "additionalProperties": {"type": "string"}, - "items": {"type": "string"}, - "additionalItems": {"type": "string"}, - "contains": {"type": "string"} - }, - "multityped_date_field": { - "type": ["string", "number"], - "format": "date", - "airbyte_type": "integer" - }, - "boolean_field": { - "oneOf": [ - true, - {"type": "string"}, - false - ] - }, - "conflicting_field": { - "oneOf": [ - {"type": "object", "properties": {"id": {"type": "string"}}}, - {"type": "object", "properties": {"name": {"type": "string"}}}, - {"type": "string"} - ] - }, - "conflicting_primitives": { - "oneOf": [ - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"} - ] - } - } - } - """); - } - - @Test - void testDowngradeWeirdSchemas() { - // old_style_schema isn't actually valid (i.e. v1 schemas should always be using $ref) - // but we should check that it behaves well anyway - doTest( - """ - { - "type": "object", - "properties": { - "old_style_schema": {"type": "string"} - } - } - """, - """ - { - "type": "object", - "properties": { - "old_style_schema": {"type": "string"} - } - } - """); - } - - private AirbyteMessage createCatalogMessage(final JsonNode schema) { - return new AirbyteMessage().withType(AirbyteMessage.Type.CATALOG) - .withCatalog( - new AirbyteCatalog().withStreams(List.of(new AirbyteStream().withJsonSchema( - schema)))); - } - - } - - @Nested - class RecordDowngradeTest { - - private static final String STREAM_NAME = "foo_stream"; - private static final String NAMESPACE_NAME = "foo_namespace"; - - @Test - void testBasicDowngrade() { - final ConfiguredAirbyteCatalog catalog = createConfiguredAirbyteCatalog( - """ - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - """); - final JsonNode oldData = Jsons.deserialize( - """ - "42" - """); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = new AirbyteMessageMigrationV1(validator) - .downgrade(createRecordMessage(oldData), Optional.of(catalog)); - - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "RECORD", - "record": { - "stream": "foo_stream", - "namespace": "foo_namespace", - "data": 42 - } - } - """, - io.airbyte.protocol.models.v0.AirbyteMessage.class); - assertEquals(expectedMessage, downgradedMessage); - } - - @Test - void testNullDowngrade() { - final AirbyteMessage oldMessage = new AirbyteMessage().withType(Type.RECORD); - final io.airbyte.protocol.models.v0.AirbyteMessage upgradedMessage = migration.downgrade(oldMessage, Optional.empty()); - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to use the given catalog to downgrade the oldData, and assert that the result is - * equal to expectedDataString - * - * @param schemaString The JSON schema of the record - * @param oldDataString The data of the record to be downgraded - * @param expectedDataString The expected data after downgrading - */ - private void doTest(final String schemaString, final String oldDataString, final String expectedDataString) { - final ConfiguredAirbyteCatalog catalog = createConfiguredAirbyteCatalog(schemaString); - final JsonNode oldData = Jsons.deserialize(oldDataString); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = new AirbyteMessageMigrationV1(validator) - .downgrade(createRecordMessage(oldData), Optional.of(catalog)); - - final JsonNode expectedDowngradedRecord = Jsons.deserialize(expectedDataString); - assertEquals(expectedDowngradedRecord, downgradedMessage.getRecord().getData()); - } - - @Test - void testNestedDowngrade() { - doTest( - """ - { - "type": "object", - "properties": { - "int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "num": {"$ref": "WellKnownTypes.json#/definitions/Number"}, - "binary": {"$ref": "WellKnownTypes.json#/definitions/BinaryData"}, - "bool": {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - "object": { - "type": "object", - "properties": { - "int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "arr": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - }, - "array": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "array_multitype": { - "type": "array", - "items": [{"$ref": "WellKnownTypes.json#/definitions/Integer"}, {"$ref": "WellKnownTypes.json#/definitions/String"}] - }, - "oneof": { - "type": "array", - "items": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - {"$ref": "WellKnownTypes.json#/definitions/Boolean"} - ] - } - } - } - } - """, - """ - { - "int": "42", - "num": "43.2", - "string": "42", - "bool": true, - "object": { - "int": "42" - }, - "array": ["42"], - "array_multitype": ["42", "42"], - "oneof": ["42", true], - "additionalProperty": "42" - } - """, - """ - { - "int": 42, - "num": 43.2, - "string": "42", - "bool": true, - "object": { - "int": 42 - }, - "array": [42], - "array_multitype": [42, "42"], - "oneof": [42, true], - "additionalProperty": "42" - } - """); - } - - @Test - void testWeirdDowngrade() { - doTest( - """ - { - "type": "object", - "properties": { - "raw_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "raw_num": {"$ref": "WellKnownTypes.json#/definitions/Number"}, - "bad_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "typeless_object": { - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - }, - "typeless_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "arr_obj_union1": { - "type": ["array", "object"], - "items": { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "arr_obj_union2": { - "type": ["array", "object"], - "items": { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "empty_oneof": { - "oneOf": [] - } - } - } - """, - """ - { - "raw_int": 42, - "raw_num": 43.2, - "bad_int": "foo", - "typeless_object": { - "foo": "42" - }, - "typeless_array": ["42"], - "arr_obj_union1": [{"id": "42", "name": "arst"}, {"id": "43", "name": "qwfp"}], - "arr_obj_union2": {"id": "42", "name": "arst"}, - "empty_oneof": "42" - } - """, - """ - { - "raw_int": 42, - "raw_num": 43.2, - "bad_int": "foo", - "typeless_object": { - "foo": 42 - }, - "typeless_array": [42], - "arr_obj_union1": [{"id": 42, "name": "arst"}, {"id": 43, "name": "qwfp"}], - "arr_obj_union2": {"id": 42, "name": "arst"}, - "empty_oneof": "42" - } - """); - } - - @Test - void testEmptySchema() { - doTest( - """ - { - "type": "object", - "properties": { - "empty_schema_primitive": {}, - "empty_schema_array": {}, - "empty_schema_object": {}, - "implicit_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "implicit_object": { - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - } - } - """, - """ - { - "empty_schema_primitive": "42", - "empty_schema_array": ["42", false], - "empty_schema_object": {"foo": "42"}, - "implicit_array": ["42"], - "implicit_object": {"foo": "42"} - } - """, - """ - { - "empty_schema_primitive": "42", - "empty_schema_array": ["42", false], - "empty_schema_object": {"foo": "42"}, - "implicit_array": [42], - "implicit_object": {"foo": 42} - } - """); - } - - @Test - void testBacktracking() { - // These test cases verify that we correctly choose the most-correct oneOf option. - doTest( - """ - { - "type": "object", - "properties": { - "valid_option": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - {"$ref": "WellKnownTypes.json#/definitions/String"} - ] - }, - "all_invalid": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Boolean"} - } - ] - }, - "nested_oneof": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "foo": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - } - } - } - ] - }, - "mismatched_primitive": { - "oneOf": [ - { - "type": "object", - "properties": { - "foo": {"type": "object"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "mismatched_text": { - "oneOf": [ - { - "type": "object", - "properties": { - "foo": {"type": "object"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "mismatch_array": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - ] - } - } - } - """, - """ - { - "valid_option": "42", - "all_invalid": ["42", "arst"], - "nested_oneof": [{"foo": "42"}], - "mismatched_primitive": { - "foo": true, - "bar": "42" - }, - "mismatched_text": { - "foo": "bar", - "bar": "42" - }, - "mismatch_array": ["arst", "41", "42"] - } - """, - """ - { - "valid_option": 42, - "all_invalid": [42, "arst"], - "nested_oneof": [{"foo": 42}], - "mismatched_primitive": { - "foo": true, - "bar": 42 - }, - "mismatched_text": { - "foo": "bar", - "bar": 42 - }, - "mismatch_array": ["arst", "41", 42] - } - """); - } - - @Test - void testIncorrectSchema() { - doTest( - """ - { - "type": "object", - "properties": { - "bad_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "bad_int_array": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "bad_int_obj": { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - } - } - """, - """ - { - "bad_int": "arst", - "bad_int_array": ["arst"], - "bad_int_obj": {"foo": "arst"} - } - """, - """ - { - "bad_int": "arst", - "bad_int_array": ["arst"], - "bad_int_obj": {"foo": "arst"} - } - """); - } - - private ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(final String schema) { - return new ConfiguredAirbyteCatalog() - .withStreams(List.of(new ConfiguredAirbyteStream().withStream(new io.airbyte.protocol.models.AirbyteStream() - .withName(STREAM_NAME) - .withNamespace(NAMESPACE_NAME) - .withJsonSchema(Jsons.deserialize(schema))))); - } - - private AirbyteMessage createRecordMessage(final JsonNode data) { - return new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(STREAM_NAME).withNamespace(NAMESPACE_NAME).withData(data)); - } - - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java deleted file mode 100644 index 1a663ec2cffe..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * These depend on the same {@link SchemaMigrationV1} class as - * {@link io.airbyte.commons.protocol.migrations.v1.AirbyteMessageMigrationV1}. So, uh, I didn't - * bother writing a ton of tests for it. - * - * Check out {@link AirbyteMessageMigrationV1} for more comprehensive tests. Theoretically - * SchemaMigrationV1 should have its own set of tests, but for various (development history-related) - * reasons, that would be a lot of work. - */ -class ConfiguredAirbyteCatalogMigrationV1Test { - - private ConfiguredAirbyteCatalogMigrationV1 migration; - - @BeforeEach - void setup() { - migration = new ConfiguredAirbyteCatalogMigrationV1(); - } - - @Test - void testVersionMetadata() { - assertEquals("0.3.0", migration.getPreviousVersion().serialize()); - assertEquals("1.0.0", migration.getCurrentVersion().serialize()); - } - - @Test - void testBasicUpgrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgradedCatalog = new io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog() - .withStreams(List.of( - new io.airbyte.protocol.models.v0.ConfiguredAirbyteStream().withStream(new io.airbyte.protocol.models.v0.AirbyteStream().withJsonSchema( - Jsons.deserialize( - """ - { - "type": "string" - } - """))))); - - final ConfiguredAirbyteCatalog upgradedMessage = migration.upgrade(downgradedCatalog); - - final ConfiguredAirbyteCatalog expectedMessage = Jsons.deserialize( - """ - { - "streams": [ - { - "stream": { - "json_schema": { - "$ref": "WellKnownTypes.json#/definitions/String" - } - } - } - ] - } - """, - ConfiguredAirbyteCatalog.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testBasicDowngrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final ConfiguredAirbyteCatalog upgradedCatalog = new ConfiguredAirbyteCatalog() - .withStreams(List.of( - new ConfiguredAirbyteStream().withStream(new AirbyteStream().withJsonSchema( - Jsons.deserialize(""" - { - "$ref": "WellKnownTypes.json#/definitions/String" - } - """))))); - - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgradedMessage = migration.downgrade(upgradedCatalog); - - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog expectedMessage = Jsons.deserialize( - """ - { - "streams": [ - { - "stream": { - "json_schema": { - "type": "string" - } - } - } - ] - } - """, - io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog.class); - assertEquals(expectedMessage, downgradedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java deleted file mode 100644 index 80a398497ded..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.jupiter.api.Test; - -class AirbyteMessageV0SerDeTest { - - @Test - void v0SerDeRoundTripTest() throws URISyntaxException { - final AirbyteMessageV0Deserializer deser = new AirbyteMessageV0Deserializer(); - final AirbyteMessageV0Serializer ser = new AirbyteMessageV0Serializer(); - - final AirbyteMessage message = new AirbyteMessage() - .withType(Type.SPEC) - .withSpec( - new ConnectorSpecification() - .withProtocolVersion("0.3.0") - .withDocumentationUrl(new URI("file:///tmp/doc"))); - - final String serializedMessage = ser.serialize(message); - final AirbyteMessage deserializedMessage = deser.deserialize(Jsons.deserialize(serializedMessage)); - - assertEquals(message, deserializedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java deleted file mode 100644 index 1a1e186c515f..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.jupiter.api.Test; - -class AirbyteMessageV1SerDeTest { - - @Test - void v1SerDeRoundTripTest() throws URISyntaxException { - final AirbyteMessageV1Deserializer deser = new AirbyteMessageV1Deserializer(); - final AirbyteMessageV1Serializer ser = new AirbyteMessageV1Serializer(); - - final AirbyteMessage message = new AirbyteMessage() - .withType(Type.SPEC) - .withSpec( - new ConnectorSpecification() - .withProtocolVersion("1.0.0") - .withDocumentationUrl(new URI("file:///tmp/doc"))); - - final String serializedMessage = ser.serialize(message); - final AirbyteMessage deserializedMessage = deser.deserialize(Jsons.deserialize(serializedMessage)); - - assertEquals(message, deserializedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json deleted file mode 100644 index 9e4d6656deae..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "definitions": { - "String": { - "type": "string", - "description": "Arbitrary text" - }, - "BinaryData": { - "type": "string", - "description": "Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", - "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" - }, - "Date": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support and (-)Infinity" - }, - "TimestampWithTimezone": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support and (-)Infinity. Note that we do _not_ accept Unix epochs here.\n" - }, - "TimestampWithoutTimezone": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support and (-)Infinity.\n" - }, - "TimeWithTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})$", - "description": "An RFC 3339\u00a75.6 full-time" - }, - "TimeWithoutTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$", - "description": "An RFC 3339\u00a75.6 partial-time" - }, - "Number": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)(\\.\\d+)?" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ], - "description": "Note the mix of regex validation for normal numbers, and enum validation for special values." - }, - "Integer": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ] - }, - "Boolean": { - "type": "boolean", - "description": "Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here." - } - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE deleted file mode 100644 index ec45d182fcb9..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle deleted file mode 100644 index 9073e823a0fe..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -plugins { - id 'java-library' - id 'de.undercouch.download' version "5.4.0" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-varargs,-try,-deprecation" - } - compileTestJava { - options.compilerArgs += "-Xlint:-try" - } -} - -dependencies { - // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation. - - // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. - implementation 'com.jayway.jsonpath:json-path:2.7.0' -} - -def downloadSpecSecretMask = tasks.register('downloadSpecSecretMask', Download) { - src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' - dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') - overwrite true -} -tasks.named('processResources').configure { dependsOn downloadSpecSecretMask } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md deleted file mode 100644 index 1f953216d630..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# airbyte-commons - -Common java helpers. - -This submodule is inherited by all other java modules in the monorepo! It is therefore important that we do not add dependencies to it, as those -dependencies will also be added to every java module. The only dependencies that this module uses are the ones declared in the `build.gradle` at the -root of the Airbyte monorepo. In other words it only uses dependencies that are already shared across all modules. The `dependencies` section of -the `build.gradle` of `airbyte-commons` should always be empty. - -For other common java code that needs to be shared across modules that requires additional dependencies, we follow this -convention: `airbyte-commons-`. See for example `airbyte-commons-cli`. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java deleted file mode 100644 index 624783f2104a..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.features; - -public class FeatureFlagsWrapper implements FeatureFlags { - - /** - * Overrides the {@link FeatureFlags#useStreamCapableState} method in the feature flags. - */ - static public FeatureFlags overridingUseStreamCapableState( - final FeatureFlags wrapped, - final boolean useStreamCapableState) { - return new FeatureFlagsWrapper(wrapped) { - - @Override - public boolean useStreamCapableState() { - return useStreamCapableState; - } - - }; - } - - /** - * Overrides the {@link FeatureFlags#deploymentMode} method in the feature flags. - */ - static public FeatureFlags overridingDeploymentMode( - final FeatureFlags wrapped, - final String deploymentMode) { - return new FeatureFlagsWrapper(wrapped) { - - @Override - public String deploymentMode() { - return deploymentMode; - } - - }; - } - - private final FeatureFlags wrapped; - - public FeatureFlagsWrapper(FeatureFlags wrapped) { - this.wrapped = wrapped; - } - - @Override - public boolean useStreamCapableState() { - return wrapped.useStreamCapableState(); - } - - @Override - public boolean autoDetectSchema() { - return wrapped.autoDetectSchema(); - } - - @Override - public boolean logConnectorMessages() { - return wrapped.logConnectorMessages(); - } - - @Override - public boolean concurrentSourceStreamRead() { - return wrapped.concurrentSourceStreamRead(); - } - - @Override - public boolean applyFieldSelection() { - return wrapped.applyFieldSelection(); - } - - @Override - public String fieldSelectionWorkspaces() { - return wrapped.fieldSelectionWorkspaces(); - } - - @Override - public String strictComparisonNormalizationWorkspaces() { - return wrapped.strictComparisonNormalizationWorkspaces(); - } - - @Override - public String strictComparisonNormalizationTag() { - return wrapped.strictComparisonNormalizationTag(); - } - - @Override - public String deploymentMode() { - return wrapped.deploymentMode(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java deleted file mode 100644 index 072884392adc..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.logging; - -import com.fasterxml.jackson.core.type.TypeReference; -import io.airbyte.commons.constants.AirbyteSecretConstants; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import java.nio.charset.Charset; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.impl.Log4jLogEvent; -import org.apache.logging.log4j.message.SimpleMessage; -import org.apache.logging.log4j.status.StatusLogger; - -/** - * Custom Log4j2 {@link RewritePolicy} used to intercept all log messages and mask any JSON - * properties in the message that match the list of maskable properties. - *

- * The maskable properties file is generated by a Gradle task in the {@code :airbyte-config:specs} - * project. The file is named {@code specs_secrets_mask.yaml} and is located in the - * {@code src/main/resources/seed} directory of the {@link :airbyte-config:init} project. - */ -@Plugin(name = "MaskedDataInterceptor", - category = "Core", - elementType = "rewritePolicy", - printObject = true) -public class MaskedDataInterceptor implements RewritePolicy { - - protected static final Logger logger = StatusLogger.getLogger(); - - /** - * The pattern used to determine if a message contains sensitive data. - */ - private final Optional pattern; - - @PluginFactory - public static MaskedDataInterceptor createPolicy( - @PluginAttribute(value = "specMaskFile", - defaultString = "/seed/specs_secrets_mask.yaml") final String specMaskFile) { - return new MaskedDataInterceptor(specMaskFile); - } - - private MaskedDataInterceptor(final String specMaskFile) { - this.pattern = buildPattern(specMaskFile); - } - - @Override - public LogEvent rewrite(final LogEvent source) { - return Log4jLogEvent.newBuilder() - .setLoggerName(source.getLoggerName()) - .setMarker(source.getMarker()) - .setLoggerFqcn(source.getLoggerFqcn()) - .setLevel(source.getLevel()) - .setMessage(new SimpleMessage(applyMask(source.getMessage().getFormattedMessage()))) - .setThrown(source.getThrown()) - .setContextMap(source.getContextMap()) - .setContextStack(source.getContextStack()) - .setThreadName(source.getThreadName()) - .setSource(source.getSource()) - .setTimeMillis(source.getTimeMillis()) - .build(); - } - - /** - * Applies the mask to the message, if necessary. - * - * @param message The log message. - * @return The possibly masked log message. - */ - private String applyMask(final String message) { - if (pattern.isPresent()) { - return message.replaceAll(pattern.get(), "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\""); - } else { - return message; - } - } - - /** - * Loads the maskable properties from the provided file. - * - * @param specMaskFile The spec mask file. - * @return The set of maskable properties. - */ - private Set getMaskableProperties(final String specMaskFile) { - - try { - final String maskFileContents = IOUtils.toString(getClass().getResourceAsStream(specMaskFile), Charset.defaultCharset()); - final Map> properties = Jsons.object(Yamls.deserialize(maskFileContents), new TypeReference<>() {}); - return properties.getOrDefault("properties", Set.of()); - } catch (final Exception e) { - logger.error("Unable to load mask data from '{}': {}.", specMaskFile, e.getMessage()); - return Set.of(); - } - } - - /** - * Builds the maskable property matching pattern. - * - * @param specMaskFile The spec mask file. - * @return The regular expression pattern used to find maskable properties. - */ - private Optional buildPattern(final String specMaskFile) { - final Set maskableProperties = getMaskableProperties(specMaskFile); - return !maskableProperties.isEmpty() ? Optional.of(generatePattern(maskableProperties)) : Optional.empty(); - } - - /** - * Generates the property matching pattern string from the provided set of properties. - * - * @param properties The set of properties to match. - * @return The generated regular expression pattern used to match the maskable properties. - */ - private String generatePattern(final Set properties) { - final StringBuilder builder = new StringBuilder(); - builder.append("(?i)"); // case insensitive - builder.append("\"("); - builder.append(properties.stream().collect(Collectors.joining("|"))); - builder.append(")\"\\s*:\\s*(\"(?:[^\"\\\\]|\\\\.)*\"|\\[[^]\\[]*]|\\d+)"); - return builder.toString(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml deleted file mode 100644 index 22d52667d0ee..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml +++ /dev/null @@ -1,219 +0,0 @@ - - - - - - %d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n - - %d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n - - - ${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}} - - ${sys:S3_LOG_BUCKET:-${env:S3_LOG_BUCKET}} - ${sys:S3_LOG_BUCKET_REGION:-${env:S3_LOG_BUCKET_REGION}} - ${sys:AWS_ACCESS_KEY_ID:-${env:AWS_ACCESS_KEY_ID}} - ${sys:AWS_SECRET_ACCESS_KEY:-${env:AWS_SECRET_ACCESS_KEY}} - ${sys:S3_MINIO_ENDPOINT:-${env:S3_MINIO_ENDPOINT}} - ${sys:S3_PATH_STYLE_ACCESS:-${env:S3_PATH_STYLE_ACCESS}} - - ${sys:GCS_LOG_BUCKET:-${env:GCS_LOG_BUCKET}} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java deleted file mode 100644 index fc4e8120072e..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.logging; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.commons.io.IOs; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -class Log4j2ConfigTest { - - private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); - private static final String LOG_FILENAME = "logs.log"; - private Path root; - - @BeforeEach - void setUp() throws IOException { - root = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "test"); - MDC.clear(); - } - - @Test - void testWorkerDispatch() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - MDC.put("context", "worker"); - MDC.put("job_log_path", root + "/" + LOG_FILENAME); - logger.error("random message testWorkerDispatch"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testWorkerDispatch")); - } - - @Test - void testLogSeparateFiles() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testLogSeparateFiles"); - - final Path root1 = root.resolve("1"); - final Path root2 = root.resolve("2"); - - final ExecutorService executor = Executors.newFixedThreadPool(2); - executor.submit(() -> { - MDC.put("job_log_path", root1 + "/" + LOG_FILENAME); - logger.error("random message 1"); - }); - - executor.submit(() -> { - MDC.put("job_log_path", root2 + "/" + LOG_FILENAME); - logger.error("random message 2"); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root1, LOG_FILENAME).contains("random message 1")); - assertTrue(IOs.readFile(root2, LOG_FILENAME).contains("random message 2")); - } - - @Test - void testLogNoJobRoot() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - logger.error("random message testLogNoJobRoot"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertFalse(Files.exists(root.resolve(LOG_FILENAME))); - } - - @Test - void testAppDispatch() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - MDC.put("workspace_app_root", root.toString()); - logger.error("random message testAppDispatch"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testAppDispatch")); - } - - @Test - void testLogNoAppRoot() throws InterruptedException { - final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - - final ExecutorService executor = Executors.newFixedThreadPool(1); - executor.submit(() -> { - logger.error("random message testLogNoAppRoot"); - MDC.clear(); - }); - - executor.shutdown(); - executor.awaitTermination(10, TimeUnit.SECONDS); - - assertFalse(Files.exists(root.resolve(LOG_FILENAME))); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE deleted file mode 100644 index ec45d182fcb9..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle deleted file mode 100644 index 84674554e773..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation 'com.networknt:json-schema-validator:1.0.72' - // needed so that we can follow $ref when parsing json. jackson does not support this natively. - implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md deleted file mode 100644 index 57eff20af41c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# airbyte-json-validation - -This module contains shared Java code for validating JSON objects. - -## Key Files -* `JsonSchemaValidator.java` is the main entrypoint into this library, defining convenience methods for validation. -* `ConfigSchemaValidator.java` is additional sugar to make it easy to validate objects whose schemas are defined in `ConfigSchema`. diff --git a/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle new file mode 100644 index 000000000000..9aa48c2623ea --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle @@ -0,0 +1,6 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + implementation 'com.azure:azure-storage-blob:12.12.0' +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java b/airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java rename to airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/azure-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 1523d78df648..089b72dbc5cd 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -1,31 +1,43 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.9.22' +} + +final var cdkVersion = { + var props = new Properties() + file("core/src/main/resources/version.properties").withInputStream(props::load) + return props.getProperty('version', 'undefined') +}() + + + allprojects { apply plugin: 'java-library' apply plugin: 'maven-publish' - apply plugin: 'airbyte-java-cdk' - apply plugin: 'airbyte-integration-test-java' - apply plugin: 'airbyte-performance-test-java' apply plugin: 'java-test-fixtures' + apply plugin: 'org.jetbrains.kotlin.jvm' group 'io.airbyte.cdk' - version = getCdkTargetVersion() -} -subprojects { subproject -> - def artifactBaseName = 'airbyte-cdk-' + subproject.name + def artifactBaseName = 'airbyte-cdk-' + project.name // E.g. airbyte-cdk-core, airbyte-cdk-db-sources, airbyte-cdk-db-destinations, etc. + project.version = cdkVersion + publishing { publications { main(MavenPublication) { groupId = 'io.airbyte.cdk' artifactId = artifactBaseName + version = project.version from components.java } testFixtures(MavenPublication) { groupId = 'io.airbyte.cdk' artifactId = artifactBaseName + '-test-fixtures' version = project.version - artifact subproject.tasks.testFixturesJar + artifact project.tasks.testFixturesJar } } // This repository is only defined and used in the context of an artifact publishing @@ -41,6 +53,19 @@ subprojects { subproject -> } } } + + compileKotlin { + compilerOptions { + jvmTarget = JvmTarget.JVM_21 + languageVersion = KotlinVersion.KOTLIN_1_9 + } + } + compileTestKotlin { + compilerOptions { + jvmTarget = JvmTarget.JVM_21 + languageVersion = KotlinVersion.KOTLIN_1_9 + } + } } project.configurations { @@ -52,69 +77,31 @@ subprojects { subproject -> description = "Airbyte Connector Development Kit (CDK) for Java." -def recursiveTasks = [ - 'assemble', - 'build', - 'integrationTestJava', - 'publish', - 'publishToMavenLocal', - 'test', -] -recursiveTasks.each { taskName -> - tasks.named(taskName).configure { - dependsOn subprojects.collect { it.tasks.named(taskName) } - } +tasks.register('cdkBuild').configure { + dependsOn subprojects.collect { it.tasks.named('build') } } - -// The `publishSnapshotIfNeeded` task will be a no-op if CDK version does not end with '-SNAPSHOT'. -task publishSnapshotIfNeeded {} - -if (version.endsWith("-SNAPSHOT")) { - logger.lifecycle("Version ${version} ends with '-SNAPSHOT'. Enqueing 'publishToMavenLocal'...") - publishSnapshotIfNeeded.dependsOn publishToMavenLocal -} else { - // Uncomment as needed for debugging: - // logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.") +tasks.register('cdkPublish').configure { + dependsOn subprojects.collect { it.tasks.named('publish') } } - -task assertCdkVersionNotPublished { +tasks.register('assertCdkVersionNotPublished') { doLast { - def checkGroupId = "io.airbyte.cdk" - def checkArtifactId = "airbyte-cdk-core" - def checkVersion = getCdkTargetVersion() - def repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars" - def groupIdUrl = checkGroupId.replace('.', '/') - def artifactUrl = "${repoUrl}/${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom" - - def connection = artifactUrl.toURL().openConnection() as HttpURLConnection + + var repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars" + var groupIdUrl = "${repoUrl}/io/airbyte/cdk" + var artifactUrl = "${groupIdUrl}/airbyte-cdk-core/${project.version}/airbyte-cdk-core-${project.version}.pom" + + var connection = artifactUrl.toURL().openConnection() as HttpURLConnection connection.setRequestMethod("HEAD") connection.connect() - def responseCode = connection.getResponseCode() + var responseCode = connection.getResponseCode() if (responseCode == 200) { - throw new GradleException("Assert failed. Java CDK '${checkVersion}' already published at: ${artifactUrl}") + throw new GradleException("Java CDK '${project.version}' already published at ${groupIdUrl}") } else if (responseCode == 404) { - logger.lifecycle( - "Assert succeeded. Version ${checkVersion} of ${checkArtifactId} has not been published. " + - "Checked: ${artifactUrl}" - ) + logger.lifecycle("Java CDK '${project.version}' not yet published at ${groupIdUrl}") } else { - logger.error("Received unexpected HTTP response code ${responseCode}. Ensure the repository is accessible.") - throw new GradleException("Error during assertion. Received unexpected HTTP response code ${responseCode}.") + throw new GradleException("Unexpected HTTP response code ${responseCode} from ${artifactUrl} : expected either 200 or 404.") } } } - -def cleanLocalCache = tasks.register('cleanLocalCache') { - def userHome = System.getProperty("user.home") - doLast { - delete '.gradle' - delete '${userHome}/.m2/repository/io/airbyte/' - delete '${userHome}/.gradle/caches/modules-2/files-2.1/io.airbyte.cdk/' - } -} -cleanLocalCache.configure { - dependsOn tasks.named('clean') - dependsOn subprojects.collect { it.tasks.named('clean') } -} diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md b/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md deleted file mode 100644 index 996bcf073ec8..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Config Models - -This module uses `jsonschema2pojo` to generate Java config objects from [json schema](https://json-schema.org/) definitions. See [build.gradle](./build.gradle) for details. - -## How to use -- Update json schema under: - ``` - src/main/resources/types/ - ``` -- Run the following command under the project root: - ```sh - ./gradlew airbyte-cdk:java:airbyte-cdk:config-models-oss:generateJsonSchema2Pojo - ``` - The generated file is under: - ``` - build/generated/src/gen/java/io/airbyte/config/ - ``` - -## Reference -- [`jsonschema2pojo` plugin](https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin). diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle deleted file mode 100644 index b64c0a8c7b40..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -import org.jsonschema2pojo.SourceType - -plugins { - id "java-library" - id "com.github.eirnym.js2p" version "1.0" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-unchecked" - } -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') -} - -jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA - source = files("${sourceSets.main.output.resourcesDir}/types") - targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - - targetPackage = 'io.airbyte.configoss' - useLongIntegers = true - - removeOldOutput = true - - generateBuilders = true - includeConstructors = false - includeSetters = true - serializable = true -} -tasks.register('generate').configure { - dependsOn tasks.named('generateJsonSchema2Pojo') -} diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index 816f72283732..50764c9c9fac 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -1,108 +1,48 @@ - java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked" + options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-this-escape" } compileTestJava { options.compilerArgs += "-Xlint:-try,-divzero,-cast" } -} - -configurations.all { - resolutionStrategy { - // TODO: Diagnose conflicting dependencies and remove these force overrides: - force 'org.mockito:mockito-core:4.6.1' + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-cast,-deprecation" } } dependencies { - // Exported dependencies from upstream projects - api libs.airbyte.protocol - api libs.hikaricp - api libs.jooq - api libs.jooq.meta - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - testCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - - // SSH dependencies - implementation 'net.i2p.crypto:eddsa:0.3.0' - - // First party test dependencies - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-sources')) + api 'com.datadoghq:dd-trace-api:1.28.0' + api 'com.datadoghq:dd-trace-ot:1.28.0' + api 'com.zaxxer:HikariCP:5.1.0' + api 'org.jooq:jooq:3.16.23' + api 'org.apache.commons:commons-csv:1.10.0' - testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') - testImplementation libs.bundles.junit - testImplementation libs.junit.jupiter.api - testImplementation libs.junit.jupiter.params - testImplementation 'org.junit.platform:junit-platform-launcher:1.7.0' - testImplementation libs.junit.jupiter.engine - implementation libs.jooq - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' - implementation "io.aesy:datasize:1.0.0" - implementation libs.apache.commons - implementation libs.apache.commons.lang - testImplementation 'commons-lang:commons-lang:2.6' implementation 'commons-cli:commons-cli:1.4' - implementation 'org.apache.commons:commons-csv:1.4' - - // Optional dependencies - // TODO: Change these to 'compileOnly' or 'testCompileOnly' - implementation 'com.azure:azure-storage-blob:12.12.0' - implementation('com.google.cloud:google-cloud-bigquery:1.133.1') - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - implementation libs.postgresql - - // testImplementation libs.junit.jupiter.api - implementation libs.hikaricp - implementation libs.debezium.api - implementation libs.debezium.embedded - implementation libs.debezium.sqlserver - implementation libs.debezium.mysql - implementation libs.debezium.postgres - implementation libs.debezium.mongodb - - api libs.bundles.datadog - implementation 'org.apache.sshd:sshd-mina:2.8.0' - - implementation libs.testcontainers - implementation libs.testcontainers.mysql - implementation libs.testcontainers.jdbc - implementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.jdbc - testImplementation libs.testcontainers.mysql - testImplementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.mssqlserver - implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - + implementation 'io.aesy:datasize:1.0.0' + implementation 'net.i2p.crypto:eddsa:0.3.0' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.17.2' + implementation 'org.apache.sshd:sshd-mina:2.11.0' // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' implementation 'org.bouncycastle:bcprov-jdk15on:1.66' implementation 'org.bouncycastle:bctls-jdk15on:1.66' - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesApi 'org.testcontainers:testcontainers:1.19.0' + testFixturesApi 'org.testcontainers:jdbc:1.19.0' - testImplementation libs.junit.jupiter.system.stubs - - implementation libs.jackson.annotations - implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' + testImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-sources')) + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres')) - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' - testImplementation 'org.mockito:mockito-core:4.6.1' + testImplementation 'mysql:mysql-connector-java:8.0.33' + testImplementation 'org.postgresql:postgresql:42.6.0' + testImplementation 'org.testcontainers:mysql:1.19.0' + testImplementation 'org.testcontainers:postgresql:1.19.0' + testImplementation 'org.xbib.elasticsearch:joptsimple:6.3.2.1' } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DbAnalyticsUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DbAnalyticsUtils.java new file mode 100644 index 000000000000..c10cffa61fe2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DbAnalyticsUtils.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db; + +import io.airbyte.protocol.models.v0.AirbyteAnalyticsTraceMessage; + +/** + * Utility class to define constants associated with database source connector analytics events. + * Make sure to add the analytics event to + * https://www.notion.so/Connector-Analytics-Events-892a79a49852465f8d59a18bd84c36de + */ +public class DbAnalyticsUtils { + + public static final String CDC_CURSOR_INVALID_KEY = "db-sources-cdc-cursor-invalid"; + + public static AirbyteAnalyticsTraceMessage cdcCursorInvalidMessage() { + return new AirbyteAnalyticsTraceMessage().withType(CDC_CURSOR_INVALID_KEY).withValue("1"); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java index 79c4f26e32d8..b70888255e1c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java @@ -4,6 +4,7 @@ package io.airbyte.cdk.db.factory; +import java.time.Duration; import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; @@ -62,8 +63,10 @@ public static DSLContext create(final String username, final String driverClassName, final String jdbcConnectionString, final SQLDialect dialect, - final Map connectionProperties) { - return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties), dialect); + final Map connectionProperties, + final Duration connectionTimeout) { + return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties, + connectionTimeout), dialect); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java index c03b6fb7a89b..a4324a30ebf7 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java @@ -4,17 +4,12 @@ package io.airbyte.cdk.db.factory; -import static org.postgresql.PGProperty.CONNECT_TIMEOUT; - import com.google.common.base.Preconditions; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.io.Closeable; import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; import java.util.Map; -import java.util.Optional; import javax.sql.DataSource; /** @@ -37,11 +32,7 @@ public static DataSource create(final String username, final String password, final String driverClassName, final String jdbcConnectionString) { - return new DataSourceBuilder() - .withDriverClassName(driverClassName) - .withJdbcUrl(jdbcConnectionString) - .withPassword(password) - .withUsername(username) + return new DataSourceBuilder(username, password, driverClassName, jdbcConnectionString) .build(); } @@ -59,14 +50,11 @@ public static DataSource create(final String username, final String password, final String driverClassName, final String jdbcConnectionString, - final Map connectionProperties) { - return new DataSourceBuilder() + final Map connectionProperties, + final Duration connectionTimeout) { + return new DataSourceBuilder(username, password, driverClassName, jdbcConnectionString) .withConnectionProperties(connectionProperties) - .withDriverClassName(driverClassName) - .withJdbcUrl(jdbcConnectionString) - .withPassword(password) - .withUsername(username) - .withConnectionTimeoutMs(DataSourceBuilder.getConnectionTimeoutMs(connectionProperties, driverClassName)) + .withConnectionTimeout(connectionTimeout) .build(); } @@ -87,13 +75,7 @@ public static DataSource create(final String username, final int port, final String database, final String driverClassName) { - return new DataSourceBuilder() - .withDatabase(database) - .withDriverClassName(driverClassName) - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) + return new DataSourceBuilder(username, password, driverClassName, host, port, database) .build(); } @@ -116,14 +98,8 @@ public static DataSource create(final String username, final String database, final String driverClassName, final Map connectionProperties) { - return new DataSourceBuilder() + return new DataSourceBuilder(username, password, driverClassName, host, port, database) .withConnectionProperties(connectionProperties) - .withDatabase(database) - .withDriverClassName(driverClassName) - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) .build(); } @@ -143,13 +119,7 @@ public static DataSource createPostgres(final String username, final String host, final int port, final String database) { - return new DataSourceBuilder() - .withDatabase(database) - .withDriverClassName("org.postgresql.Driver") - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) + return new DataSourceBuilder(username, password, "org.postgresql.Driver", host, port, database) .build(); } @@ -162,7 +132,7 @@ public static DataSource createPostgres(final String username, */ public static void close(final DataSource dataSource) throws Exception { if (dataSource != null) { - if (dataSource instanceof AutoCloseable closeable) { + if (dataSource instanceof final AutoCloseable closeable) { closeable.close(); } } @@ -171,7 +141,7 @@ public static void close(final DataSource dataSource) throws Exception { /** * Builder class used to configure and construct {@link DataSource} instances. */ - private static class DataSourceBuilder { + public static class DataSourceBuilder { private Map connectionProperties = Map.of(); private String database; @@ -180,57 +150,38 @@ private static class DataSourceBuilder { private String jdbcUrl; private int maximumPoolSize = 10; private int minimumPoolSize = 0; - private long connectionTimeoutMs; + private Duration connectionTimeout = Duration.ZERO; private String password; private int port = 5432; private String username; - private static final String CONNECT_TIMEOUT_KEY = "connectTimeout"; - private static final Duration CONNECT_TIMEOUT_DEFAULT = Duration.ofSeconds(60); - - private DataSourceBuilder() {} - - /** - * Retrieves connectionTimeout value from connection properties in millis, default minimum timeout - * is 60 seconds since Hikari default of 30 seconds is not enough for acceptance tests. In the case - * the value is 0, pass the value along as Hikari and Postgres use default max value for 0 timeout - * value. - * - * NOTE: HikariCP uses milliseconds for all time values: - * https://github.com/brettwooldridge/HikariCP#gear-configuration-knobs-baby whereas Postgres is - * measured in seconds: https://jdbc.postgresql.org/documentation/head/connect.html - * - * @param connectionProperties custom jdbc_url_parameters containing information on connection - * properties - * @param driverClassName name of the JDBC driver - * @return DataSourceBuilder class used to create dynamic fields for DataSource - */ - private static long getConnectionTimeoutMs(final Map connectionProperties, String driverClassName) { - final Optional parsedConnectionTimeout = switch (DatabaseDriver.findByDriverClassName(driverClassName)) { - case POSTGRESQL -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT.getName()), ChronoUnit.SECONDS) - .or(() -> maybeParseDuration(CONNECT_TIMEOUT.getDefaultValue(), ChronoUnit.SECONDS)); - case MYSQL -> maybeParseDuration(connectionProperties.get("connectTimeout"), ChronoUnit.MILLIS); - case MSSQLSERVER -> maybeParseDuration(connectionProperties.get("loginTimeout"), ChronoUnit.SECONDS); - default -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) - // Enforce minimum timeout duration for unspecified data sources. - .filter(d -> d.compareTo(CONNECT_TIMEOUT_DEFAULT) >= 0); - }; - return parsedConnectionTimeout.orElse(CONNECT_TIMEOUT_DEFAULT).toMillis(); + private String connectionInitSql; + + private DataSourceBuilder(final String username, + final String password, + final String driverClassName) { + this.username = username; + this.password = password; + this.driverClassName = driverClassName; } - private static Optional maybeParseDuration(final String stringValue, TemporalUnit unit) { - if (stringValue == null) { - return Optional.empty(); - } - final long number; - try { - number = Long.parseLong(stringValue); - } catch (NumberFormatException __) { - return Optional.empty(); - } - if (number < 0) { - return Optional.empty(); - } - return Optional.of(Duration.of(number, unit)); + public DataSourceBuilder(final String username, + final String password, + final String driverClassName, + final String jdbcUrl) { + this(username, password, driverClassName); + this.jdbcUrl = jdbcUrl; + } + + public DataSourceBuilder(final String username, + final String password, + final String driverClassName, + final String host, + final int port, + final String database) { + this(username, password, driverClassName); + this.host = host; + this.port = port; + this.database = database; } public DataSourceBuilder withConnectionProperties(final Map connectionProperties) { @@ -274,9 +225,9 @@ public DataSourceBuilder withMinimumPoolSize(final Integer minimumPoolSize) { return this; } - public DataSourceBuilder withConnectionTimeoutMs(final Long connectionTimeoutMs) { - if (connectionTimeoutMs != null) { - this.connectionTimeoutMs = connectionTimeoutMs; + public DataSourceBuilder withConnectionTimeout(final Duration connectionTimeout) { + if (connectionTimeout != null) { + this.connectionTimeout = connectionTimeout; } return this; } @@ -298,6 +249,11 @@ public DataSourceBuilder withUsername(final String username) { return this; } + public DataSourceBuilder withConnectionInitSql(final String sql) { + this.connectionInitSql = sql; + return this; + } + public DataSource build() { final DatabaseDriver databaseDriver = DatabaseDriver.findByDriverClassName(driverClassName); @@ -309,7 +265,9 @@ public DataSource build() { config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(databaseDriver.getUrlFormatString(), host, port, database)); config.setMaximumPoolSize(maximumPoolSize); config.setMinimumIdle(minimumPoolSize); - config.setConnectionTimeout(connectionTimeoutMs); + // HikariCP uses milliseconds for all time values: + // https://github.com/brettwooldridge/HikariCP#gear-configuration-knobs-baby + config.setConnectionTimeout(connectionTimeout.toMillis()); config.setPassword(password); config.setUsername(username); @@ -320,6 +278,8 @@ public DataSource build() { */ config.setInitializationFailTimeout(Integer.MIN_VALUE); + config.setConnectionInitSql(connectionInitSql); + connectionProperties.forEach(config::addDataSourceProperty); return new HikariDataSource(config); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index e7b8514cb7f5..c87c8047501c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -28,8 +28,8 @@ import java.time.OffsetTime; import java.time.chrono.IsoEra; import java.time.format.DateTimeParseException; +import java.util.Base64; import java.util.Collections; -import javax.xml.bind.DatatypeConverter; /** * Source operation skeleton for JDBC compatible databases. @@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa } protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value)); + preparedStatement.setBytes(parameterIndex, Base64.getDecoder().decode(value)); } protected ObjectType getObject(final ResultSet resultSet, final int index, final Class clazz) throws SQLException { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java index 9b3affc6dd33..183073715bc7 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java @@ -16,6 +16,7 @@ import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.sql.DataSource; @@ -89,6 +90,22 @@ public DatabaseMetaData getMetaData() throws SQLException { } } + @Override + public T executeMetadataQuery(Function query) { + try (final Connection connection = dataSource.getConnection()) { + final DatabaseMetaData metaData = connection.getMetaData(); + return query.apply(metaData); + } catch (final SQLException e) { + // Some databases like Redshift will have null cause + if (Objects.isNull(e.getCause()) || !(e.getCause() instanceof SQLException)) { + throw new ConnectionErrorException(e.getSQLState(), e.getErrorCode(), e.getMessage(), e); + } else { + final SQLException cause = (SQLException) e.getCause(); + throw new ConnectionErrorException(e.getSQLState(), cause.getErrorCode(), cause.getMessage(), e); + } + } + } + /** * You CANNOT assume that data will be returned from this method before the entire {@link ResultSet} * is buffered in memory. Review the implementation of the database's JDBC driver or use the diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java index 790e5ac37a28..21777a0e3a70 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java @@ -13,6 +13,8 @@ public final class JdbcConstants { public static final String JDBC_COLUMN_TABLE_NAME = "TABLE_NAME"; public static final String JDBC_COLUMN_COLUMN_NAME = "COLUMN_NAME"; public static final String JDBC_COLUMN_DATA_TYPE = "DATA_TYPE"; + public static final String JDBC_COLUMN_TYPE = "TYPE"; + public static final String JDBC_COLUMN_TYPE_NAME = "TYPE_NAME"; public static final String JDBC_COLUMN_SIZE = "COLUMN_SIZE"; public static final String JDBC_INDEX_NAME = "INDEX_NAME"; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java index ff7db2e6a5ff..4fb40b74e96f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java @@ -20,6 +20,7 @@ import java.util.Spliterator; import java.util.Spliterators; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -174,15 +175,17 @@ public List queryJsons(final CheckedFunction stream = unsafeQuery(c -> { - PreparedStatement statement = c.prepareStatement(sql); - int i = 1; - for (String param : params) { - statement.setString(i, param); - ++i; - } - return statement; - }, rs -> rs.getInt(1))) { + try (final Stream stream = unsafeQuery( + c -> getPreparedStatement(sql, params, c), + rs -> rs.getInt(1))) { + return stream.findFirst().get(); + } + } + + public boolean queryBoolean(final String sql, final String... params) throws SQLException { + try (final Stream stream = unsafeQuery( + c -> getPreparedStatement(sql, params, c), + rs -> rs.getBoolean(1))) { return stream.findFirst().get(); } } @@ -216,20 +219,38 @@ public List queryJsons(final String sql, final String... params) throw } public ResultSetMetaData queryMetadata(final String sql, final String... params) throws SQLException { - try (final Stream q = unsafeQuery(c -> { - PreparedStatement statement = c.prepareStatement(sql); - int i = 1; - for (String param : params) { - statement.setString(i, param); - ++i; - } - return statement; - }, + try (final Stream q = unsafeQuery( + c -> getPreparedStatement(sql, params, c), ResultSet::getMetaData)) { return q.findFirst().orElse(null); } } + /** + * Implementations of DatabaseMetadata hold a reference of the Connection object. It is safe to use + * this to retrieve static information like getIndentifierQuoteString() etc but calling methods + * which return a ResultSet needs the connection to be still open. This may or may not work + * depending on how the underlying Connection object is handled eg. Hikari's ProxyConnection is not + * actually closed, rather recycled into Pool. See {@link #executeMetadataQuery(Function)} which + * gives the caller a safe alternative to access ResultSet methods of DatabaseMetadata in the + * consumer before closing connection. + * + * @return + * @throws SQLException + */ + public abstract DatabaseMetaData getMetaData() throws SQLException; + public abstract T executeMetadataQuery(Function query) throws SQLException; + + private static PreparedStatement getPreparedStatement(String sql, String[] params, Connection c) throws SQLException { + PreparedStatement statement = c.prepareStatement(sql); + int i = 1; + for (String param : params) { + statement.setString(i, param); + i++; + } + return statement; + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java index 3c7f3f4494a5..7a502d18bfc6 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java @@ -96,6 +96,12 @@ public static URI keyStoreFromCertificate(final String certString, return keyStoreFromCertificate(fromPEMString(certString), keyStorePassword, filesystem, directory); } + public static URI keyStoreFromCertificate(final String certString, + final String keyStorePassword) + throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException { + return keyStoreFromCertificate(fromPEMString(certString), keyStorePassword, null, null); + } + public static URI keyStoreFromCertificate(final String certString, final String keyStorePassword, final String directory) throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException { return keyStoreFromCertificate(certString, keyStorePassword, FileSystems.getDefault(), directory); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java new file mode 100644 index 000000000000..44f749e1694a --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Map; +import java.util.Optional; + +public abstract class JdbcConnector extends BaseConnector { + + public static final String POSTGRES_CONNECT_TIMEOUT_KEY = "connectTimeout"; + public static final Duration POSTGRES_CONNECT_TIMEOUT_DEFAULT_DURATION = Duration.ofSeconds(10); + + public static final String CONNECT_TIMEOUT_KEY = "connectTimeout"; + public static final Duration CONNECT_TIMEOUT_DEFAULT = Duration.ofSeconds(60); + + protected final String driverClassName; + + protected JdbcConnector(String driverClassName) { + this.driverClassName = driverClassName; + } + + protected Duration getConnectionTimeout(final Map connectionProperties) { + return getConnectionTimeout(connectionProperties, driverClassName); + } + + /** + * Retrieves connectionTimeout value from connection properties in millis, default minimum timeout + * is 60 seconds since Hikari default of 30 seconds is not enough for acceptance tests. In the case + * the value is 0, pass the value along as Hikari and Postgres use default max value for 0 timeout + * value. + * + * NOTE: Postgres timeout is measured in seconds: + * https://jdbc.postgresql.org/documentation/head/connect.html + * + * @param connectionProperties custom jdbc_url_parameters containing information on connection + * properties + * @param driverClassName name of the JDBC driver + * @return DataSourceBuilder class used to create dynamic fields for DataSource + */ + public static Duration getConnectionTimeout(final Map connectionProperties, String driverClassName) { + final Optional parsedConnectionTimeout = switch (DatabaseDriver.findByDriverClassName(driverClassName)) { + case POSTGRESQL -> maybeParseDuration(connectionProperties.get(POSTGRES_CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) + .or(() -> Optional.of(POSTGRES_CONNECT_TIMEOUT_DEFAULT_DURATION)); + case MYSQL -> maybeParseDuration(connectionProperties.get("connectTimeout"), ChronoUnit.MILLIS); + case MSSQLSERVER -> maybeParseDuration(connectionProperties.get("loginTimeout"), ChronoUnit.SECONDS); + default -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) + // Enforce minimum timeout duration for unspecified data sources. + .filter(d -> d.compareTo(CONNECT_TIMEOUT_DEFAULT) >= 0); + }; + return parsedConnectionTimeout.orElse(CONNECT_TIMEOUT_DEFAULT); + } + + private static Optional maybeParseDuration(final String stringValue, TemporalUnit unit) { + if (stringValue == null) { + return Optional.empty(); + } + final long number; + try { + number = Long.parseLong(stringValue); + } catch (NumberFormatException __) { + return Optional.empty(); + } + if (number < 0) { + return Optional.empty(); + } + return Optional.of(Duration.of(number, unit)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java index 64502fb55232..94c75b6f0118 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -12,8 +12,8 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.validation.constraints.NotNull; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,22 +71,26 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { final Optional deinterpolatableException = ExceptionUtils.getThrowableList(throwable).stream() .filter(t -> THROWABLES_TO_DEINTERPOLATE.stream().anyMatch(deinterpolatableClass -> deinterpolatableClass.isAssignableFrom(t.getClass()))) .findFirst(); + final boolean messageWasMangled; if (deinterpolatableException.isPresent()) { + final String originalMessage = deinterpolatableException.get().getMessage(); mangledMessage = STRINGS_TO_DEINTERPOLATE.stream() // Sort the strings longest to shortest, in case any target string is a substring of another // e.g. "airbyte_internal" should be swapped out before "airbyte" .sorted(Comparator.comparing(String::length).reversed()) - .reduce(deinterpolatableException.get().getMessage(), AirbyteExceptionHandler::deinterpolate); + .reduce(originalMessage, AirbyteExceptionHandler::deinterpolate); + messageWasMangled = !mangledMessage.equals(originalMessage); } else { mangledMessage = throwable.getMessage(); + messageWasMangled = false; } - // If we did not modify the message (either not a deinterpolatable class, or we tried to - // deinterpolate - // but made no changes) then emit our default trace message - if (mangledMessage.equals(throwable.getMessage())) { + if (!messageWasMangled) { + // If we did not modify the message (either not a deinterpolatable class, or we tried to + // deinterpolate but made no changes) then emit our default trace message AirbyteTraceMessageUtility.emitSystemErrorTrace(throwable, logMessage); } else { + // If we did modify the message, then emit a custom trace message AirbyteTraceMessageUtility.emitCustomErrorTrace(throwable.getMessage(), mangledMessage); } @@ -95,7 +99,8 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { @NotNull private static String deinterpolate(final String message, final String targetString) { - final String quotedTarget = '(' + Pattern.quote(targetString) + ')'; + // (?i) makes the pattern case-insensitive + final String quotedTarget = '(' + "(?i)" + Pattern.quote(targetString) + ')'; final String targetRegex = REGEX_PREFIX + quotedTarget + REGEX_SUFFIX; final Pattern pattern = Pattern.compile(targetRegex); final Matcher matcher = pattern.matcher(message); @@ -116,7 +121,7 @@ public static void addThrowableForDeinterpolation(final Class ORPHANED_THREAD_FILTER = runningThread -> !runningThread.getName().equals(Thread.currentThread().getName()) && !runningThread.isDaemon() && !TYPE_AND_DEDUPE_THREAD_NAME.equals(runningThread.getName()); - public static final int INTERRUPT_THREAD_DELAY_MINUTES = 60; - public static final int EXIT_THREAD_DELAY_MINUTES = 70; + public static final int INTERRUPT_THREAD_DELAY_MINUTES = 1; + public static final int EXIT_THREAD_DELAY_MINUTES = 2; public static final int FORCED_EXIT_CODE = 2; @@ -140,7 +140,7 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { case CHECK -> { final JsonNode config = parseConfig(parsed.getConfigPath()); if (integration instanceof Destination) { - DestinationConfig.initialize(config); + DestinationConfig.initialize(config, ((Destination) integration).isV2Destination()); } try { validateConfig(integration.spec().getConnectionSpecification(), config, "CHECK"); @@ -183,17 +183,13 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { final JsonNode config = parseConfig(parsed.getConfigPath()); validateConfig(integration.spec().getConnectionSpecification(), config, "WRITE"); // save config to singleton - DestinationConfig.initialize(config); + DestinationConfig.initialize(config, ((Destination) integration).isV2Destination()); final ConfiguredAirbyteCatalog catalog = parseConfig(parsed.getCatalogPath(), ConfiguredAirbyteCatalog.class); try (final SerializedAirbyteMessageConsumer consumer = destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector)) { consumeWriteStream(consumer); } finally { - stopOrphanedThreads(EXIT_HOOK, - INTERRUPT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES, - EXIT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES); + stopOrphanedThreads(); } } default -> throw new IllegalStateException("Unexpected value: " + parsed.getCommand()); @@ -263,11 +259,7 @@ private void readConcurrent(final JsonNode config, final ConfiguredAirbyteCatalo LOGGER.error("Unable to perform concurrent read.", e); throw e; } finally { - stopOrphanedThreads(EXIT_HOOK, - INTERRUPT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES, - EXIT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES); + stopOrphanedThreads(); } } @@ -275,11 +267,7 @@ private void readSerial(final JsonNode config, final ConfiguredAirbyteCatalog ca try (final AutoCloseableIterator messageIterator = source.read(config, catalog, stateOptional.orElse(null))) { produceMessages(messageIterator, outputRecordCollector); } finally { - stopOrphanedThreads(EXIT_HOOK, - INTERRUPT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES, - EXIT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES); + stopOrphanedThreads(); } } @@ -335,6 +323,23 @@ static void consumeWriteStream(final SerializedAirbyteMessageConsumer consumer, } } + /** + * Stops any non-daemon threads that could block the JVM from exiting when the main thread is done. + * + * If any active non-daemon threads would be left as orphans, this method will schedule some + * interrupt/exit hooks after giving it some time delay to close up properly. It is generally + * preferred to have a proper closing sequence from children threads instead of interrupting or + * force exiting the process, so this mechanism serve as a fallback while surfacing warnings in logs + * for maintainers to fix the code behavior instead. + */ + static void stopOrphanedThreads() { + stopOrphanedThreads(EXIT_HOOK, + INTERRUPT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES, + EXIT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES); + } + /** * Stops any non-daemon threads that could block the JVM from exiting when the main thread is done. *

@@ -343,6 +348,7 @@ static void consumeWriteStream(final SerializedAirbyteMessageConsumer consumer, * preferred to have a proper closing sequence from children threads instead of interrupting or * force exiting the process, so this mechanism serve as a fallback while surfacing warnings in logs * for maintainers to fix the code behavior instead. + *

* * @param exitHook The {@link Runnable} exit hook to execute for any orphaned threads. * @param interruptTimeDelay The time to delay execution of the orphaned thread interrupt attempt. diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java index b59e757b62c4..8820b1d7017f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java @@ -9,11 +9,12 @@ public class TypingAndDedupingFlag { public static boolean isDestinationV2() { - return DestinationConfig.getInstance().getBooleanValue("use_1s1t_format"); + return DestinationConfig.getInstance().getIsV2Destination() + || DestinationConfig.getInstance().getBooleanValue("use_1s1t_format"); } - public static Optional getRawNamespaceOverride(String option) { - String rawOverride = DestinationConfig.getInstance().getTextValue(option); + public static Optional getRawNamespaceOverride(final String option) { + final String rawOverride = DestinationConfig.getInstance().getTextValue(option); if (rawOverride == null || rawOverride.isEmpty()) { return Optional.empty(); } else { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java index 878eef089be0..81d508b0dd2c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java @@ -4,14 +4,9 @@ package io.airbyte.cdk.integrations.base.adaptive; -import io.airbyte.cdk.integrations.base.Command; import io.airbyte.cdk.integrations.base.Destination; -import io.airbyte.cdk.integrations.base.DestinationConfig; -import io.airbyte.cdk.integrations.base.IntegrationCliParser; -import io.airbyte.cdk.integrations.base.IntegrationConfig; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,15 +83,6 @@ private Destination getDestination() { } public void run(final String[] args) throws Exception { - // getDestination() sometimes depends on the singleton being initialized. - // Parse the CLI args just so we can accomplish that. - IntegrationConfig parsedArgs = new IntegrationCliParser().parse(args); - if (parsedArgs.getCommand() != Command.SPEC) { - DestinationConfig.initialize(IntegrationRunner.parseConfig(parsedArgs.getConfigPath())); - } else { - DestinationConfig.initialize(Jsons.emptyObject()); - } - final Destination destination = getDestination(); LOGGER.info("Starting destination: {}", destination.getClass().getName()); new IntegrationRunner(destination).run(args); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java index 367d429bdc2a..ab577063c0e3 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java @@ -10,8 +10,6 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.io.IOException; import java.util.Optional; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.testcontainers.containers.Container; public class SshHelpers { @@ -40,30 +38,4 @@ public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecificat return originalSpec; } - /** - * Returns the inner docker network ip address and port of a container. This can be used to reach a - * container from another container running on the same network - * - * @param container container - * @return a pair of host and port - */ - public static ImmutablePair getInnerContainerAddress(final Container container) { - return ImmutablePair.of( - container.getContainerInfo().getNetworkSettings().getNetworks().entrySet().stream().findFirst().get().getValue().getIpAddress(), - (Integer) container.getExposedPorts().stream().findFirst().get()); - } - - /** - * Returns the outer docker network ip address and port of a container. This can be used to reach a - * container from the host machine - * - * @param container container - * @return a pair of host and port - */ - public static ImmutablePair getOuterContainerAddress(final Container container) { - return ImmutablePair.of( - container.getHost(), - container.getFirstMappedPort()); - } - } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java index 159bd07f582c..649f0576a589 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java @@ -23,10 +23,13 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Optional; +import javax.validation.constraints.NotNull; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.keyverifier.AcceptAllServerKeyVerifier; import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.SshException; +import org.apache.sshd.common.session.SessionHeartbeatController; import org.apache.sshd.common.util.net.SshdSocketAddress; import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.core.CoreModuleProperties; @@ -46,12 +49,24 @@ public class SshTunnel implements AutoCloseable { public static final String SSH_TIMEOUT_DISPLAY_MESSAGE = "Timed out while opening a SSH Tunnel. Please double check the given SSH configurations and try again."; + public static final String CONNECTION_OPTIONS_KEY = "ssh_connection_options"; + public static final String SESSION_HEARTBEAT_INTERVAL_KEY = "session_heartbeat_interval"; + public static final long SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS = 1000; + public static final String GLOBAL_HEARTBEAT_INTERVAL_KEY = "global_heartbeat_interval"; + public static final long GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS = 2000; + public static final String IDLE_TIMEOUT_KEY = "idle_timeout"; + public static final long IDLE_TIMEOUT_DEFAULT_INFINITE = 0; + public enum TunnelMethod { NO_TUNNEL, SSH_PASSWORD_AUTH, SSH_KEY_AUTH } + public record SshConnectionOptions(Duration sessionHeartbeatInterval, + Duration globalHeartbeatInterval, + Duration idleTimeout) {} + public static final int TIMEOUT_MILLIS = 15000; // 15 seconds private final JsonNode config; @@ -99,6 +114,7 @@ public enum TunnelMethod { * tunnel host). * @param remoteServicePort - the actual port of the remote service (as it is known to the tunnel * host). + * @param connectionOptions - optional connection options for ssh client. */ public SshTunnel(final JsonNode config, final List hostKey, @@ -112,7 +128,8 @@ public SshTunnel(final JsonNode config, final String sshKey, final String tunnelUserPassword, final String remoteServiceHost, - final int remoteServicePort) { + final int remoteServicePort, + final Optional connectionOptions) { this.config = config; this.hostKey = hostKey; this.portKey = portKey; @@ -168,11 +185,42 @@ public SshTunnel(final JsonNode config, this.tunnelUser = tunnelUser; this.sshKey = sshKey; this.tunnelUserPassword = tunnelUserPassword; - this.sshclient = createClient(); + this.sshclient = connectionOptions.map(sshConnectionOptions -> createClient(sshConnectionOptions.sessionHeartbeatInterval(), + sshConnectionOptions.globalHeartbeatInterval(), + sshConnectionOptions.idleTimeout())).orElseGet(this::createClient); this.tunnelSession = openTunnel(sshclient); } } + public SshTunnel(final JsonNode config, + final List hostKey, + final List portKey, + final String endPointKey, + final String remoteServiceUrl, + final TunnelMethod tunnelMethod, + final String tunnelHost, + final int tunnelPort, + final String tunnelUser, + final String sshKey, + final String tunnelUserPassword, + final String remoteServiceHost, + final int remoteServicePort) { + this(config, + hostKey, + portKey, + endPointKey, + remoteServiceUrl, + tunnelMethod, + tunnelHost, + tunnelPort, + tunnelUser, + sshKey, + tunnelUserPassword, + remoteServiceHost, + remoteServicePort, + Optional.empty()); + } + public JsonNode getOriginalConfig() { return config; } @@ -216,7 +264,32 @@ public static SshTunnel getInstance(final JsonNode config, final List ho Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "ssh_key")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user_password")), Strings.safeTrim(Jsons.getStringOrNull(config, hostKey)), - Jsons.getIntOrZero(config, portKey)); + Jsons.getIntOrZero(config, portKey), + getSshConnectionOptions(config)); + } + + @NotNull + private static Optional getSshConnectionOptions(JsonNode config) { + // piggybacking on JsonNode config to make it configurable at connector level. + Optional connectionOptionConfig = Jsons.getOptional(config, CONNECTION_OPTIONS_KEY); + final Optional connectionOptions; + if (connectionOptionConfig.isPresent()) { + JsonNode connectionOptionsNode = connectionOptionConfig.get(); + Duration sessionHeartbeatInterval = Jsons.getOptional(connectionOptionsNode, SESSION_HEARTBEAT_INTERVAL_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ofSeconds(1)); + Duration globalHeartbeatInterval = Jsons.getOptional(connectionOptionsNode, GLOBAL_HEARTBEAT_INTERVAL_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ofSeconds(2)); + Duration idleTimeout = Jsons.getOptional(connectionOptionsNode, IDLE_TIMEOUT_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ZERO); + connectionOptions = Optional.of( + new SshConnectionOptions(sessionHeartbeatInterval, globalHeartbeatInterval, idleTimeout)); + } else { + connectionOptions = Optional.empty(); + } + return connectionOptions; } public static SshTunnel getInstance(final JsonNode config, final String endPointKey) throws Exception { @@ -237,7 +310,8 @@ public static SshTunnel getInstance(final JsonNode config, final String endPoint Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "ssh_key")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user_password")), - null, 0); + null, 0, + getSshConnectionOptions(config)); } public static void sshWrap(final JsonNode config, @@ -332,7 +406,22 @@ private SshClient createClient() { final SshClient client = SshClient.setUpDefaultClient(); client.setForwardingFilter(AcceptAllForwardingFilter.INSTANCE); client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE); - CoreModuleProperties.IDLE_TIMEOUT.set(client, Duration.ZERO); + return client; + } + + private SshClient createClient(Duration sessionHeartbeatInterval, Duration globalHeartbeatInterval, Duration idleTimeout) { + LOGGER.info("Creating SSH client with Heartbeat and Keepalive enabled"); + final SshClient client = createClient(); + // Session level heartbeat using SSH_MSG_IGNORE every second. + client.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.IGNORE, sessionHeartbeatInterval); + // idle-timeout zero indicates NoTimeout. + CoreModuleProperties.IDLE_TIMEOUT.set(client, idleTimeout); + // Use tcp keep-alive mechanism. + CoreModuleProperties.SOCKET_KEEPALIVE.set(client, true); + // Additional delay used for ChannelOutputStream to wait for space in the remote socket send buffer. + CoreModuleProperties.WAIT_FOR_SPACE_TIMEOUT.set(client, Duration.ofMinutes(2)); + // Global keepalive message sent every 2 seconds. This precedes the session level heartbeat. + CoreModuleProperties.HEARTBEAT_INTERVAL.set(client, globalHeartbeatInterval); return client; } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java index 54fb872f2ce7..f0873bb05edb 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java @@ -4,6 +4,14 @@ package io.airbyte.cdk.integrations.base.ssh; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.CONNECTION_OPTIONS_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.GLOBAL_HEARTBEAT_INTERVAL_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.SESSION_HEARTBEAT_INTERVAL_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.getInstance; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.sshWrap; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; @@ -18,6 +26,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,8 +73,8 @@ public ConnectorSpecification spec() throws Exception { @Override public AirbyteConnectionStatus check(final JsonNode config) throws Exception { try { - return (endPointKey != null) ? SshTunnel.sshWrap(config, endPointKey, delegate::check) - : SshTunnel.sshWrap(config, hostKey, portKey, delegate::check); + return (endPointKey != null) ? sshWrap(config, endPointKey, delegate::check) + : sshWrap(config, hostKey, portKey, delegate::check); } catch (final RuntimeException e) { final String sshErrorMessage = "Could not connect with provided SSH configuration. Error: " + e.getMessage(); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, sshErrorMessage); @@ -98,7 +107,17 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) throws Exception { - final SshTunnel tunnel = getTunnelInstance(config); + final JsonNode clone = Jsons.clone(config); + Optional connectionOptionsConfig = Jsons.getOptional(clone, CONNECTION_OPTIONS_KEY); + if (connectionOptionsConfig.isEmpty()) { + LOGGER.info("No SSH connection options found, using defaults"); + if (clone instanceof ObjectNode) { // Defensive check, it will always be object node + ObjectNode connectionOptions = ((ObjectNode) clone).putObject(CONNECTION_OPTIONS_KEY); + connectionOptions.put(SESSION_HEARTBEAT_INTERVAL_KEY, SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS); + connectionOptions.put(GLOBAL_HEARTBEAT_INTERVAL_KEY, GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS); + } + } + final SshTunnel tunnel = getTunnelInstance(clone); final SerializedAirbyteMessageConsumer delegateConsumer; try { delegateConsumer = delegate.getSerializedMessageConsumer(tunnel.getConfigInTunnel(), catalog, outputRecordCollector); @@ -112,8 +131,13 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception { return (endPointKey != null) - ? SshTunnel.getInstance(config, endPointKey) - : SshTunnel.getInstance(config, hostKey, portKey); + ? getInstance(config, endPointKey) + : getInstance(config, hostKey, portKey); + } + + @Override + public boolean isV2Destination() { + return delegate.isV2Destination(); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java index a0bb39cc5d25..cc9c2dc4cd15 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java @@ -31,6 +31,7 @@ public String getNamespace(final String namespace) { } @Override + // @Deprecated see https://github.com/airbytehq/airbyte/issues/35333 public String getRawTableName(final String streamName) { return convertStreamName("_airbyte_raw_" + streamName); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java new file mode 100644 index 000000000000..d4a76c862ac7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination; + +import java.util.Optional; + +/** + * @param recordsWritten The number of records written to the stream, or empty if the caller does + * not track this information. (this is primarily for backwards-compatibility with the legacy + * destinations framework; new implementations should always provide this information). If + * this value is empty, consumers should assume that the sync wrote nonzero records for this + * stream. + */ +public record StreamSyncSummary(Optional recordsWritten) { + + public static final StreamSyncSummary DEFAULT = new StreamSyncSummary(Optional.empty()); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java index 2226d7acb270..b4cdd9bd73ee 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java @@ -310,14 +310,16 @@ protected void close(final boolean hasFailed) throws Exception { * not bother committing. otherwise attempt to commit */ if (stateManager.listFlushed().isEmpty()) { - onClose.accept(hasFailed); + // Not updating this class to track record count, because we want to kill it in favor of the + // AsyncStreamConsumer + onClose.accept(hasFailed, new HashMap<>()); } else { /* * if any state message was flushed that means we should try to commit what we have. if * hasFailed=false, then it could be full success. if hasFailed=true, then going for partial * success. */ - onClose.accept(false); + onClose.accept(false, null); } stateManager.listCommitted().forEach(outputRecordCollector); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java index df4f8995fd00..39c4da662a88 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java @@ -4,15 +4,18 @@ package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; -import io.airbyte.commons.functional.CheckedConsumer; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.commons.functional.CheckedBiConsumer; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; /** * Interface allowing destination to specify clean up logic that must be executed after all * record-related logic has finished. + *

+ * The map of StreamSyncSummaries MUST be non-null, but MAY be empty. Streams not present in the map + * will be treated as equivalent to {@link StreamSyncSummary#DEFAULT}. */ -public interface OnCloseFunction extends CheckedConsumer { - - @Override - void accept(Boolean hasFailed) throws Exception; +public interface OnCloseFunction extends CheckedBiConsumer, Exception> { } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java index 5f4d47e6eb76..14d17485588a 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java @@ -25,7 +25,7 @@ */ public interface SqlOperations { - Logger LOGGER = LoggerFactory.getLogger(JdbcBufferedConsumerFactory.class); + Logger LOGGER = LoggerFactory.getLogger(SqlOperations.class); /** * Create a schema with provided name if it does not already exist. diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java index 698a9b269f22..80940bcf612b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java @@ -53,7 +53,7 @@ Stream toMessages(final String line) { if (Strings.isEmpty(line)) { return Stream.of(logMessage(Level.INFO, "")); } - final Optional json = Jsons.tryDeserialize(line); + final Optional json = Jsons.tryDeserializeWithoutWarn(line); if (json.isPresent()) { return jsonToMessage(json.get()); } else { @@ -96,7 +96,7 @@ private Stream jsonToMessage(final JsonNode jsonLine) { */ final String logLevel = (jsonLine.hasNonNull("level")) ? jsonLine.get("level").asText() : ""; String logMsg = jsonLine.hasNonNull("msg") ? jsonLine.get("msg").asText() : ""; - Level level; + final Level level; switch (logLevel) { case "debug" -> level = Level.DEBUG; case "info" -> level = Level.INFO; @@ -117,7 +117,7 @@ private Stream jsonToMessage(final JsonNode jsonLine) { } } - private static AirbyteMessage logMessage(Level level, String message) { + private static AirbyteMessage logMessage(final Level level, final String message) { return new AirbyteMessage() .withType(Type.LOG) .withLog(new AirbyteLogMessage() @@ -125,7 +125,7 @@ private static AirbyteMessage logMessage(Level level, String message) { .withMessage(message)); } - public static void main(String[] args) { + public static void main(final String[] args) { final NormalizationLogParser normalizationLogParser = new NormalizationLogParser(); final Stream airbyteMessageStream = normalizationLogParser.create(new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))); @@ -135,8 +135,8 @@ public static void main(String[] args) { final String dbtErrorStack = String.join("\n", errors); if (!"".equals(dbtErrorStack)) { final Map errorMap = SentryExceptionHelper.getUsefulErrorMessageAndTypeFromDbtError(dbtErrorStack); - String internalMessage = errorMap.get(ErrorMapKeys.ERROR_MAP_MESSAGE_KEY); - AirbyteMessage traceMessage = new AirbyteMessage() + final String internalMessage = errorMap.get(ErrorMapKeys.ERROR_MAP_MESSAGE_KEY); + final AirbyteMessage traceMessage = new AirbyteMessage() .withType(Type.TRACE) .withTrace(new AirbyteTraceMessage() .withType(AirbyteTraceMessage.Type.ERROR) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java similarity index 84% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java index fc04e995fb47..f91fada8b465 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java @@ -7,17 +7,28 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import java.time.Instant; import java.util.List; import java.util.UUID; -import org.joda.time.DateTime; /** * Staging operations focuses on the SQL queries that are needed to success move data into a staging * environment like GCS or S3. In general, the reference of staging is the usage of an object * storage for the purposes of efficiently uploading bulk data to destinations + * + * TODO: This interface is shared between Snowflake and Redshift connectors where the staging + * mechanism is different wire protocol. Make the interface more Generic and have sub interfaces to + * support BlobStorageOperations or Jdbc based staging operations. */ public interface StagingOperations extends SqlOperations { + /** + * @param outputTableName The name of the table this staging file will be loaded into (typically a + * raw table). Not all destinations use the table name in the staging path (e.g. Snowflake + * simply uses a timestamp + UUID), but e.g. Redshift does rely on this to ensure uniqueness. + */ + String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, Instant writeDatetime); + /** * Returns the staging environment's name * @@ -27,13 +38,6 @@ public interface StagingOperations extends SqlOperations { */ String getStageName(String namespace, String streamName); - /** - * @param outputTableName The name of the table this staging file will be loaded into (typically a - * raw table). Not all destinations use the table name in the staging path (e.g. Snowflake - * simply uses a timestamp + UUID), but e.g. Redshift does rely on this to ensure uniqueness. - */ - String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, DateTime writeDatetime); - /** * Create a staging folder where to upload temporary files before loading into the final destination */ @@ -45,7 +49,6 @@ public interface StagingOperations extends SqlOperations { * @param database database used for syncing * @param recordsData records stored in in-memory buffer * @param schemaName name of schema - * @param stageName name of the staging area folder * @param stagingPath path of staging folder to data files * @return the name of the file that was uploaded. */ @@ -56,7 +59,6 @@ String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsDat * Load the data stored in the stage area into a temporary table in the destination * * @param database database interface - * @param stageName name of staging area folder * @param stagingPath path to staging files * @param stagedFiles collection of staged files * @param tableName name of table to write staging files to @@ -70,21 +72,12 @@ void copyIntoTableFromStage(JdbcDatabase database, String schemaName) throws Exception; - /** - * Remove files that were just staged - * - * @param database database used for syncing - * @param stageName name of staging area folder - * @param stagedFiles collection of the staging files to remove - */ - void cleanUpStage(JdbcDatabase database, String stageName, List stagedFiles) throws Exception; - /** * Delete the stage area and all staged files that was in it * * @param database database used for syncing * @param stageName Name of the staging area used to store files */ - void dropStageIfExists(JdbcDatabase database, String stageName) throws Exception; + void dropStageIfExists(JdbcDatabase database, String stageName, String stagingPath) throws Exception; } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java index acd60edc51f6..711326fd919b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java @@ -4,10 +4,13 @@ package io.airbyte.cdk.integrations.destination_async; +import static java.util.stream.Collectors.toMap; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; import io.airbyte.cdk.integrations.destination_async.buffers.BufferEnqueue; import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; @@ -18,8 +21,14 @@ import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; @@ -47,6 +56,8 @@ public class AsyncStreamConsumer implements SerializedAirbyteMessageConsumer { private final Set streamNames; private final FlushFailure flushFailure; private final String defaultNamespace; + // Note that this map will only be populated for streams with nonzero records. + private final ConcurrentMap recordCounts; private boolean hasStarted; private boolean hasClosed; @@ -102,6 +113,7 @@ public AsyncStreamConsumer(final Consumer outputRecordCollector, flushWorkers = new FlushWorkers(bufferManager.getBufferDequeue(), flusher, outputRecordCollector, flushFailure, bufferManager.getStateManager(), workerPool); streamNames = StreamDescriptorUtils.fromConfiguredCatalog(catalog); + this.recordCounts = new ConcurrentHashMap<>(); } @VisibleForTesting @@ -113,18 +125,7 @@ public AsyncStreamConsumer(final Consumer outputRecordCollector, final BufferManager bufferManager, final FlushFailure flushFailure, final String defaultNamespace) { - this.defaultNamespace = defaultNamespace; - hasStarted = false; - hasClosed = false; - - this.onStart = onStart; - this.onClose = onClose; - this.catalog = catalog; - this.bufferManager = bufferManager; - bufferEnqueue = bufferManager.getBufferEnqueue(); - this.flushFailure = flushFailure; - flushWorkers = new FlushWorkers(bufferManager.getBufferDequeue(), flusher, outputRecordCollector, flushFailure, bufferManager.getStateManager()); - streamNames = StreamDescriptorUtils.fromConfiguredCatalog(catalog); + this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, flushFailure, defaultNamespace, Executors.newFixedThreadPool(5)); } @Override @@ -153,8 +154,10 @@ public void accept(final String messageString, final Integer sizeInBytes) throws message.getRecord().setNamespace(defaultNamespace); } validateRecord(message); + + getRecordCounter(message.getRecord().getStreamDescriptor()).incrementAndGet(); } - bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES); + bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES, defaultNamespace); } /** @@ -204,13 +207,22 @@ public void close() throws Exception { flushWorkers.close(); bufferManager.close(); - onClose.accept(hasFailed); + + final Map streamSyncSummaries = streamNames.stream().collect(toMap( + streamDescriptor -> streamDescriptor, + streamDescriptor -> new StreamSyncSummary( + Optional.of(getRecordCounter(streamDescriptor).get())))); + onClose.accept(hasFailed, streamSyncSummaries); // as this throws an exception, we need to be after all other close functions. propagateFlushWorkerExceptionIfPresent(); LOGGER.info("{} closed", AsyncStreamConsumer.class); } + private AtomicLong getRecordCounter(final StreamDescriptor streamDescriptor) { + return recordCounts.computeIfAbsent(streamDescriptor, sd -> new AtomicLong()); + } + private void propagateFlushWorkerExceptionIfPresent() throws Exception { if (flushFailure.isFailed()) { hasFailed = true; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java index 6e7ffd379098..22878f7780ba 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java @@ -55,7 +55,7 @@ public interface DestinationFlushFunction { * vague because I don't understand the specifics. */ default long getQueueFlushThresholdBytes() { - return 10 * 1024 * 1024; // 10MB + return Math.max(10 * 1024 * 1024, getOptimalBatchSizeBytes()); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index b02ebdf131b1..32b01c570291 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -6,13 +6,10 @@ import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; import io.airbyte.cdk.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; -import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.cdk.integrations.destination_async.state.FlushFailure; import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; -import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -170,7 +167,7 @@ private void flush(final StreamDescriptor desc, final UUID flushWorkerId) { AirbyteFileUtils.byteCountToDisplaySize(batch.getSizeInBytes())); flusher.flush(desc, batch.getData().stream().map(MessageWithMeta::message)); - emitStateMessages(batch.flushStates(stateIdToCount)); + batch.flushStates(stateIdToCount, outputRecordCollector); } log.info("Flush Worker ({}) -- Worker finished flushing. Current queue size: {}", @@ -220,26 +217,23 @@ public void close() throws Exception { log.info("Closing flush workers -- all buffers flushed"); // before shutting down the supervisor, flush all state. - emitStateMessages(stateManager.flushStates()); + stateManager.flushStates(outputRecordCollector); supervisorThread.shutdown(); - final var supervisorShut = supervisorThread.awaitTermination(5L, TimeUnit.MINUTES); - log.info("Closing flush workers -- Supervisor shutdown status: {}", supervisorShut); + while (!supervisorThread.awaitTermination(5L, TimeUnit.MINUTES)) { + log.info("Waiting for flush worker supervisor to shut down"); + } + log.info("Closing flush workers -- supervisor shut down"); log.info("Closing flush workers -- Starting worker pool shutdown.."); workerPool.shutdown(); - final var workersShut = workerPool.awaitTermination(5L, TimeUnit.MINUTES); - log.info("Closing flush workers -- Workers shutdown status: {}", workersShut); + while (!workerPool.awaitTermination(5L, TimeUnit.MINUTES)) { + log.info("Waiting for flush workers to shut down"); + } + log.info("Closing flush workers -- workers shut down"); debugLoop.shutdownNow(); } - private void emitStateMessages(final List partials) { - partials - .stream() - .map(partial -> Jsons.deserialize(partial.getSerialized(), AirbyteMessage.class)) - .forEach(outputRecordCollector); - } - private static String humanReadableFlushWorkerId(final UUID flushWorkerId) { return flushWorkerId.toString().substring(0, 5); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java index ee83e8c76fb3..ca8aea8fdbcb 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java @@ -88,8 +88,9 @@ public void free(final long bytes) { log.info("Freeing {} bytes..", bytes); currentMemoryBytes.addAndGet(-bytes); - if (currentMemoryBytes.get() < 0) { - log.warn("Freed more memory than allocated. This should never happen. Please report this bug."); + final long currentMemory = currentMemoryBytes.get(); + if (currentMemory < 0) { + log.info("Freed more memory than allocated ({} of {})", bytes, currentMemory + bytes); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java index 9b004ac0d451..c1bd6f097d8f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java @@ -4,13 +4,16 @@ package io.airbyte.cdk.integrations.destination_async; -import java.util.function.Consumer; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.function.BiConsumer; /** * Async version of * {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction}. * Separately out for easier versioning. */ -public interface OnCloseFunction extends Consumer { +public interface OnCloseFunction extends BiConsumer> { } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java index 0434678e12a2..09f67f62c786 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java @@ -38,11 +38,11 @@ public BufferEnqueue(final GlobalMemoryManager memoryManager, * @param message to buffer * @param sizeInBytes */ - public void addRecord(final PartialAirbyteMessage message, final Integer sizeInBytes) { + public void addRecord(final PartialAirbyteMessage message, final Integer sizeInBytes, final String defaultNamespace) { if (message.getType() == Type.RECORD) { handleRecord(message, sizeInBytes); } else if (message.getType() == Type.STATE) { - stateManager.trackState(message, sizeInBytes); + stateManager.trackState(message, sizeInBytes, defaultNamespace); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java index 2a0f541c6284..213f30e7768e 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java @@ -6,10 +6,11 @@ import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; import io.airbyte.cdk.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; -import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.protocol.models.v0.AirbyteMessage; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,16 +58,13 @@ public void close() throws Exception { } /** - * For the batch, marks all the states that have now been flushed. Also returns states that can be - * flushed. This method is descriptrive, it assumes that whatever consumes the state messages emits - * them, internally it purges the states it returns. message that it can. + * For the batch, marks all the states that have now been flushed. Also writes the states that can + * be flushed back to platform via stateManager. *

- * - * @return list of states that can be flushed */ - public List flushStates(final Map stateIdToCount) { + public void flushStates(final Map stateIdToCount, final Consumer outputRecordCollector) { stateIdToCount.forEach(stateManager::decrement); - return stateManager.flushStates(); + stateManager.flushStates(outputRecordCollector); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java index 3e8a57547845..ebd903fcfc87 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.Objects; // TODO: (ryankfu) remove this and test with low memory resources to ensure OOM is still not a @@ -116,4 +117,8 @@ public String toString() { '}'; } + public StreamDescriptor getStreamDescriptor() { + return new StreamDescriptor().withName(stream).withNamespace(namespace); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java index c704a36d753d..845dfdd629ea 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java @@ -7,24 +7,25 @@ import static java.lang.Thread.sleep; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteStreamState; +import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.ArrayList; +import java.time.Instant; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -68,21 +69,40 @@ public class GlobalAsyncStateManager { */ private final AtomicLong memoryUsed; - boolean preState = true; - private final ConcurrentMap> descToStateIdQ = new ConcurrentHashMap<>(); + private boolean preState = true; + private final ConcurrentMap> descToStateIdQ = new ConcurrentHashMap<>(); + /** + * Both {@link stateIdToCounter} and {@link stateIdToCounterForPopulatingDestinationStats} are used + * to maintain a counter for the number of records associated with a give state i.e. before a state + * was received, how many records were seen until that point. As records are received the value for + * both are incremented. The difference is the purpose of the two attributes. + * {@link stateIdToCounter} is used to determine whether a state is safe to emit or not. This is + * done by decrementing the value as records are committed to the destination. If the value hits 0, + * it means all the records associated with a given state have been committed to the destination, it + * is safe to emit the state back to platform. But because of this we can't use it to determine the + * actual number of records that are associated with a state to update the value of + * {@link AirbyteStateMessage#destinationStats} at the time of emitting the state message. That's + * where we need {@link stateIdToCounterForPopulatingDestinationStats}, which is only reset when a + * state message has been emitted. + */ private final ConcurrentMap stateIdToCounter = new ConcurrentHashMap<>(); - private final ConcurrentMap> stateIdToState = new ConcurrentHashMap<>(); + private final ConcurrentMap stateIdToCounterForPopulatingDestinationStats = new ConcurrentHashMap<>(); + private final ConcurrentMap> stateIdToState = new ConcurrentHashMap<>(); // Alias-ing only exists in the non-STREAM case where we have to convert existing state ids to one // single global id. // This only happens once. private final Set aliasIds = new ConcurrentHashSet<>(); private long retroactiveGlobalStateId = 0; + // All access to this field MUST be guarded by a synchronized(lock) block + private long arrivalNumber = 0; + + private final Object LOCK = new Object(); public GlobalAsyncStateManager(final GlobalMemoryManager memoryManager) { this.memoryManager = memoryManager; - memoryAllocated = new AtomicLong(memoryManager.requestMemory()); - memoryUsed = new AtomicLong(); + this.memoryAllocated = new AtomicLong(memoryManager.requestMemory()); + this.memoryUsed = new AtomicLong(); } // Always assume STREAM to begin, and convert only if needed. Most state is per stream anyway. @@ -97,7 +117,7 @@ public GlobalAsyncStateManager(final GlobalMemoryManager memoryManager) { * Because state messages are a watermark, all preceding records need to be flushed before the state * message can be processed. */ - public void trackState(final PartialAirbyteMessage message, final long sizeInBytes) { + public void trackState(final PartialAirbyteMessage message, final long sizeInBytes, final String defaultNamespace) { if (preState) { convertToGlobalIfNeeded(message); preState = false; @@ -105,7 +125,7 @@ public void trackState(final PartialAirbyteMessage message, final long sizeInByt // stateType should not change after a conversion. Preconditions.checkArgument(stateType == extractStateType(message)); - closeState(message, sizeInBytes); + closeState(message, sizeInBytes, defaultNamespace); } /** @@ -127,29 +147,27 @@ public long getStateIdAndIncrementCounter(final StreamDescriptor streamDescripto * @param count to decrement. */ public void decrement(final long stateId, final long count) { - log.trace("decrementing state id: {}, count: {}", stateId, count); - stateIdToCounter.get(getStateAfterAlias(stateId)).addAndGet(-count); + synchronized (LOCK) { + log.trace("decrementing state id: {}, count: {}", stateId, count); + stateIdToCounter.get(getStateAfterAlias(stateId)).addAndGet(-count); + } } /** - * Returns state messages with no more inflight records i.e. counter = 0 across all streams. + * Flushes state messages with no more inflight records i.e. counter = 0 across all streams. * Intended to be called by {@link io.airbyte.cdk.integrations.destination_async.FlushWorkers} after * a worker has finished flushing its record batch. *

- * The return list of states should be emitted back to the platform. - * - * @return list of state messages with no more inflight records. */ - public List flushStates() { - final List output = new ArrayList<>(); + public void flushStates(final Consumer outputRecordCollector) { Long bytesFlushed = 0L; - synchronized (this) { - for (final Map.Entry> entry : descToStateIdQ.entrySet()) { + synchronized (LOCK) { + for (final Map.Entry> entry : descToStateIdQ.entrySet()) { // Remove all states with 0 counters. // Per-stream synchronized is required to make sure the state (at the head of the queue) // logic is applied to is the state actually removed. - final LinkedList stateIdQueue = entry.getValue(); + final LinkedBlockingDeque stateIdQueue = entry.getValue(); while (true) { final Long oldestStateId = stateIdQueue.peek(); // no state to flush for this stream @@ -158,8 +176,11 @@ public List flushStates() { } // technically possible this map hasn't been updated yet. + // This can be if you call the flush method if there are 0 records/states final var oldestStateCounter = stateIdToCounter.get(oldestStateId); - Objects.requireNonNull(oldestStateCounter, "Invariant Violation: No record counter found for state message."); + if (oldestStateCounter == null) { + break; + } final var oldestState = stateIdToState.get(oldestStateId); // no state to flush for this stream @@ -169,13 +190,22 @@ public List flushStates() { final var allRecordsCommitted = oldestStateCounter.get() == 0; if (allRecordsCommitted) { - output.add(oldestState.getLeft()); + final StateMessageWithArrivalNumber stateMessage = oldestState.getLeft(); + final double flushedRecordsAssociatedWithState = stateIdToCounterForPopulatingDestinationStats.get(oldestStateId).doubleValue(); + + log.info("State with arrival number {} emitted from thread {} at {}", stateMessage.arrivalNumber(), Thread.currentThread().getName(), + Instant.now().toString()); + final AirbyteMessage message = Jsons.deserialize(stateMessage.partialAirbyteStateMessage.getSerialized(), AirbyteMessage.class); + message.getState().setDestinationStats(new AirbyteStateStats().withRecordCount(flushedRecordsAssociatedWithState)); + outputRecordCollector.accept(message); + bytesFlushed += oldestState.getRight(); // cleanup entry.getValue().poll(); stateIdToState.remove(oldestStateId); stateIdToCounter.remove(oldestStateId); + stateIdToCounterForPopulatingDestinationStats.remove(oldestStateId); } else { break; } @@ -184,7 +214,6 @@ public List flushStates() { } freeBytes(bytesFlushed); - return output; } private Long getStateIdAndIncrement(final StreamDescriptor streamDescriptor, final long increment) { @@ -194,10 +223,15 @@ private Long getStateIdAndIncrement(final StreamDescriptor streamDescriptor, fin if (descToStateIdQ.get(resolvedDescriptor) == null) { registerNewStreamDescriptor(resolvedDescriptor); } - final Long stateId = descToStateIdQ.get(resolvedDescriptor).peekLast(); - final var update = stateIdToCounter.get(stateId).addAndGet(increment); - log.trace("State id: {}, count: {}", stateId, update); - return stateId; + synchronized (LOCK) { + final Long stateId = descToStateIdQ.get(resolvedDescriptor).peekLast(); + final var update = stateIdToCounter.get(stateId).addAndGet(increment); + if (increment >= 0) { + stateIdToCounterForPopulatingDestinationStats.get(stateId).addAndGet(increment); + } + log.trace("State id: {}, count: {}", stateId, update); + return stateId; + } } /** @@ -238,20 +272,28 @@ private void convertToGlobalIfNeeded(final PartialAirbyteMessage message) { if (stateType != AirbyteStateMessage.AirbyteStateType.STREAM) {// alias old stream-level state ids to single global state id // upon conversion, all previous tracking data structures need to be cleared as we move // into the non-STREAM world for correctness. - - aliasIds.addAll(descToStateIdQ.values().stream().flatMap(Collection::stream).toList()); - descToStateIdQ.clear(); - retroactiveGlobalStateId = StateIdProvider.getNextId(); - - descToStateIdQ.put(SENTINEL_GLOBAL_DESC, new LinkedList<>()); - descToStateIdQ.get(SENTINEL_GLOBAL_DESC).add(retroactiveGlobalStateId); - - final long combinedCounter = stateIdToCounter.values() - .stream() - .mapToLong(AtomicLong::get) - .sum(); - stateIdToCounter.clear(); - stateIdToCounter.put(retroactiveGlobalStateId, new AtomicLong(combinedCounter)); + synchronized (LOCK) { + aliasIds.addAll(descToStateIdQ.values().stream().flatMap(Collection::stream).toList()); + descToStateIdQ.clear(); + retroactiveGlobalStateId = StateIdProvider.getNextId(); + + descToStateIdQ.put(SENTINEL_GLOBAL_DESC, new LinkedBlockingDeque<>()); + descToStateIdQ.get(SENTINEL_GLOBAL_DESC).add(retroactiveGlobalStateId); + + final long combinedCounter = stateIdToCounter.values() + .stream() + .mapToLong(AtomicLong::get) + .sum(); + stateIdToCounter.clear(); + stateIdToCounter.put(retroactiveGlobalStateId, new AtomicLong(combinedCounter)); + + final long statsCounter = stateIdToCounterForPopulatingDestinationStats.values() + .stream() + .mapToLong(AtomicLong::get) + .sum(); + stateIdToCounterForPopulatingDestinationStats.clear(); + stateIdToCounterForPopulatingDestinationStats.put(retroactiveGlobalStateId, new AtomicLong(statsCounter)); + } } } @@ -269,9 +311,13 @@ private AirbyteStateMessage.AirbyteStateType extractStateType(final PartialAirby * to the newly arrived state message. We also increment the state id in preparation for the next * state message. */ - private void closeState(final PartialAirbyteMessage message, final long sizeInBytes) { - final StreamDescriptor resolvedDescriptor = extractStream(message).orElse(SENTINEL_GLOBAL_DESC); - stateIdToState.put(getStateId(resolvedDescriptor), ImmutablePair.of(message, sizeInBytes)); + private void closeState(final PartialAirbyteMessage message, final long sizeInBytes, final String defaultNamespace) { + final StreamDescriptor resolvedDescriptor = extractStream(message, defaultNamespace).orElse(SENTINEL_GLOBAL_DESC); + synchronized (LOCK) { + log.info("State with arrival number {} received", arrivalNumber); + stateIdToState.put(getStateId(resolvedDescriptor), ImmutablePair.of(new StateMessageWithArrivalNumber(message, arrivalNumber), sizeInBytes)); + arrivalNumber++; + } registerNewStateId(resolvedDescriptor); allocateMemoryToState(sizeInBytes); } @@ -309,8 +355,29 @@ public String getMemoryUsageMessage() { (double) memoryUsed.get() / memoryAllocated.get()); } - private static Optional extractStream(final PartialAirbyteMessage message) { - return Optional.ofNullable(message.getState().getStream()).map(PartialAirbyteStreamState::getStreamDescriptor); + /** + * If the user has selected the Destination Namespace as the Destination default while setting up + * the connector, the platform sets the namespace as null in the StreamDescriptor in the + * AirbyteMessages (both record and state messages). The destination checks that if the namespace is + * empty or null, if yes then re-populates it with the defaultNamespace. See + * {@link io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer#accept(String,Integer)} + * But destination only does this for the record messages. So when state messages arrive without a + * namespace and since the destination doesn't repopulate it with the default namespace, there is a + * mismatch between the StreamDescriptor from record messages and state messages. That breaks the + * logic of the state management class as {@link descToStateIdQ} needs to have consistent + * StreamDescriptor. This is why while trying to extract the StreamDescriptor from state messages, + * we check if the namespace is null, if yes then replace it with defaultNamespace to keep it + * consistent with the record messages. + */ + private static Optional extractStream(final PartialAirbyteMessage message, final String defaultNamespace) { + if (message.getState().getType() != null && message.getState().getType() == AirbyteStateMessage.AirbyteStateType.STREAM) { + final StreamDescriptor streamDescriptor = message.getState().getStream().getStreamDescriptor(); + if (Strings.isNullOrEmpty(streamDescriptor.getNamespace())) { + return Optional.of(new StreamDescriptor().withName(streamDescriptor.getName()).withNamespace(defaultNamespace)); + } + return Optional.of(streamDescriptor); + } + return Optional.empty(); } private long getStateAfterAlias(final long stateId) { @@ -322,14 +389,19 @@ private long getStateAfterAlias(final long stateId) { } private void registerNewStreamDescriptor(final StreamDescriptor resolvedDescriptor) { - descToStateIdQ.put(resolvedDescriptor, new LinkedList<>()); + synchronized (LOCK) { + descToStateIdQ.put(resolvedDescriptor, new LinkedBlockingDeque<>()); + } registerNewStateId(resolvedDescriptor); } private void registerNewStateId(final StreamDescriptor resolvedDescriptor) { final long stateId = StateIdProvider.getNextId(); - stateIdToCounter.put(stateId, new AtomicLong(0)); - descToStateIdQ.get(resolvedDescriptor).add(stateId); + synchronized (LOCK) { + stateIdToCounter.put(stateId, new AtomicLong(0)); + stateIdToCounterForPopulatingDestinationStats.put(stateId, new AtomicLong(0)); + descToStateIdQ.get(resolvedDescriptor).add(stateId); + } } /** @@ -345,4 +417,6 @@ public static long getNextId() { } + private record StateMessageWithArrivalNumber(PartialAirbyteMessage partialAirbyteStateMessage, long arrivalNumber) {} + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt new file mode 100644 index 000000000000..42183f51fcbe --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt @@ -0,0 +1,20 @@ +package io.airbyte.cdk.integrations.util + +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog + +/** + * For streams in [catalog] which do not have a namespace specified, explicitly set their namespace + * to the [defaultNamespace] + */ + fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String?) { + if (defaultNamespace == null) { + return + } + // TODO: This logic exists in all V2 destinations. + // This is sad that if we forget to add this, there will be a null pointer during parseCatalog + for (catalogStream in catalog.streams) { + if (catalogStream.stream.namespace.isNullOrEmpty()) { + catalogStream.stream.namespace = defaultNamespace + } + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java index 9f4ae86cfe78..dc49697d3e99 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java @@ -10,6 +10,7 @@ import io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.commons.functional.Either; import java.sql.SQLException; import java.sql.SQLSyntaxErrorException; import java.util.Collection; @@ -85,6 +86,17 @@ public static void logAllAndThrowFirst(final String initia } } + public static List getResultsOrLogAndThrowFirst(final String initialMessage, + final List> eithers) + throws T { + List throwables = eithers.stream().filter(Either::isLeft).map(Either::getLeft).toList(); + if (!throwables.isEmpty()) { + logAllAndThrowFirst(initialMessage, throwables); + } + // No need to filter on isRight since isLeft will throw before reaching this line. + return eithers.stream().map(Either::getRight).toList(); + } + private static boolean isConfigErrorException(Throwable e) { return e instanceof ConfigErrorException; } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json index ea1e0f9f7b40..657126acde94 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json @@ -7,7 +7,7 @@ }, "message": { "$resolver": "pattern", - "pattern": "%level %C{1.}(%M):%L %m", + "pattern": "%level %thread %C{1.}(%M):%L %m", "stringified": true }, "stack_trace": { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 0566460826cc..b0d83063013b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.7.2 +version=0.23.2 diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java index 37086e620289..4b560ae4b876 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java @@ -39,7 +39,6 @@ void setup() { @AfterEach void cleanup() throws Exception { DataSourceFactory.close(dataSource); - dslContext.close(); container.stop(); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java index e618b1548138..d673b71cfa56 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import io.airbyte.cdk.integrations.JdbcConnector; import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; @@ -51,7 +52,8 @@ void testCreatingADslContextWithIndividualConfigurationAndConnectionProperties() container.getDriverClassName(), container.getJdbcUrl(), dialect, - connectionProperties); + connectionProperties, + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); assertNotNull(dslContext); assertEquals(dialect, dslContext.configuration().dialect()); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java index a8af1eb4abee..db8850af63a4 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java @@ -12,12 +12,12 @@ import static org.mockito.Mockito.verify; import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.integrations.JdbcConnector; import java.util.Map; import javax.sql.DataSource; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.MySQLContainer; /** @@ -55,7 +55,8 @@ void testCreatingDataSourceWithConnectionTimeoutSetAboveDefault() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(61000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -70,7 +71,8 @@ void testCreatingPostgresDataSourceWithConnectionTimeoutSetBelowDefault() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(30000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -87,23 +89,8 @@ void testCreatingMySQLDataSourceWithConnectionTimeoutSetBelowDefault() { mySQLContainer.getPassword(), mySQLContainer.getDriverClassName(), mySQLContainer.getJdbcUrl(), - connectionProperties); - assertNotNull(dataSource); - assertEquals(HikariDataSource.class, dataSource.getClass()); - assertEquals(5000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); - } - } - - @Test - void testCreatingMsSQLServerDataSourceWithConnectionTimeoutSetBelowDefault() { - try (var mssqlServerContainer = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense()) { - mssqlServerContainer.start(); - final DataSource dataSource = DataSourceFactory.create( - mssqlServerContainer.getUsername(), - mssqlServerContainer.getPassword(), - mssqlServerContainer.getDriverClassName(), - mssqlServerContainer.getJdbcUrl(), - Map.of("loginTimeout", "5")); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, mySQLContainer.getDriverClassName())); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(5000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -119,7 +106,8 @@ void testCreatingDataSourceWithConnectionTimeoutSetWithZero() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(Integer.MAX_VALUE, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -133,7 +121,8 @@ void testCreatingPostgresDataSourceWithConnectionTimeoutNotSet() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(10000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -149,7 +138,8 @@ void testCreatingMySQLDataSourceWithConnectionTimeoutNotSet() { mySQLContainer.getPassword(), mySQLContainer.getDriverClassName(), mySQLContainer.getJdbcUrl(), - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, mySQLContainer.getDriverClassName())); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(60000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -169,7 +159,13 @@ void testCreatingADataSourceWithJdbcUrl() { void testCreatingADataSourceWithJdbcUrlAndConnectionProperties() { final Map connectionProperties = Map.of("foo", "bar"); - final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl, connectionProperties); + final DataSource dataSource = DataSourceFactory.create( + username, + password, + driverClassName, + jdbcUrl, + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(10, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java index 25812410a01a..23f871f0bacd 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java @@ -64,12 +64,13 @@ void testMessageDeinterpolation() throws Exception { // foo and bar are added to the list explicitly // name and description are added implicitly by the exception handler. // all of them should be replaced by '?' - runTestWithMessage("Error happened in arst_foo_bar_zxcv (name: description)"); + // (including FOO, which should be detected case-insensitively) + runTestWithMessage("Error happened in arst_FOO_bar_zxcv (name: description)"); final AirbyteMessage traceMessage = findFirstTraceMessage(); assertAll( () -> assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()), - () -> assertEquals("Error happened in arst_foo_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), + () -> assertEquals("Error happened in arst_FOO_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), () -> assertEquals("Error happened in arst_?_?_zxcv (?: ?)", traceMessage.getTrace().getError().getInternalMessage()), () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType()), () -> Assertions.assertNull(traceMessage.getTrace().getError().getStackTrace(), diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java index 39795319dbf7..50307ebd1890 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java @@ -25,51 +25,45 @@ import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.apache.logging.log4j.spi.ExtendedLogger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.junit.platform.commons.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class AirbyteLogMessageTemplateTest { - private static final ByteArrayOutputStream outputContent = new ByteArrayOutputStream(); - private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteLogMessageTemplateTest.class); public static final String OUTPUT_STREAM_APPENDER = "OutputStreamAppender"; public static final String CONSOLE_JSON_APPENDER = "ConsoleJSONAppender"; - private static OutputStreamAppender outputStreamAppender; - private static LoggerConfig rootLoggerConfig; - private static LoggerContext loggerContext; + private LoggerContext loggerContext; + private LoggerConfig rootLoggerConfig; + private ExtendedLogger logger; + private OutputStreamAppender outputStreamAppender; + private ByteArrayOutputStream outputContent; - @BeforeAll - static void init() { + void getLogger() { // We are creating a log appender with the same output pattern // as the console json appender defined in this project's log4j2.xml file. // We then attach this log appender with the LOGGER instance so that we can validate the logs // produced by code and assert that it matches the expected format. loggerContext = Configurator.initialize(null, "log4j2.xml"); + final Configuration configuration = loggerContext.getConfiguration(); rootLoggerConfig = configuration.getLoggerConfig(""); + outputContent = new ByteArrayOutputStream(); outputStreamAppender = OutputStreamAppender.createAppender( rootLoggerConfig.getAppenders().get(CONSOLE_JSON_APPENDER).getLayout(), null, outputContent, OUTPUT_STREAM_APPENDER, false, true); outputStreamAppender.start(); rootLoggerConfig.addAppender(outputStreamAppender, Level.ALL, null); + logger = loggerContext.getLogger(AirbyteLogMessageTemplateTest.class); } - @BeforeEach - void setup() { - outputContent.reset(); - } - - @AfterAll - static void cleanUp() { + @AfterEach + void closeLogger() { outputStreamAppender.stop(); rootLoggerConfig.removeAppender(OUTPUT_STREAM_APPENDER); loggerContext.close(); @@ -77,7 +71,8 @@ static void cleanUp() { @Test public void testAirbyteLogMessageFormat() throws java.io.IOException { - LOGGER.info("hello"); + getLogger(); + logger.info("hello"); outputContent.flush(); final String logMessage = outputContent.toString(StandardCharsets.UTF_8); @@ -88,7 +83,8 @@ public void testAirbyteLogMessageFormat() throws java.io.IOException { // validate that the message inside AirbyteLogMessage matches the pattern. // pattern to check for is: LOG_LEVEL className(methodName):LineNumber logMessage final String connectorLogMessageRegex = - "^INFO [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$"; + String.format("^INFO %s [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$", + Pattern.compile(Thread.currentThread().getName())); final Pattern pattern = Pattern.compile(connectorLogMessageRegex); final Matcher matcher = pattern.matcher(connectorLogMessage); @@ -113,12 +109,13 @@ private AirbyteLogMessage validateAirbyteMessageIsLog(final AirbyteMessage airby @ParameterizedTest @ValueSource(ints = {2, 100, 9000}) - public void testAirbyteLogMessageLength(int stringRepeatitions) throws java.io.IOException { + public void testAirbyteLogMessageLength(int stringRepetitions) throws java.io.IOException { + getLogger(); final StringBuilder sb = new StringBuilder(); - for (int i = 0; i < stringRepeatitions; i++) { + for (int i = 0; i < stringRepetitions; i++) { sb.append("abcd"); } - LOGGER.info(sb.toString(), new RuntimeException("aaaaa bbbbbb ccccccc dddddd")); + logger.info(sb.toString(), new RuntimeException("aaaaa bbbbbb ccccccc dddddd")); outputContent.flush(); final String logMessage = outputContent.toString(StandardCharsets.UTF_8); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java index f75f7a01ac99..6ea6492d2960 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java @@ -26,7 +26,7 @@ public void setUpOut() { System.setOut(new PrintStream(outContent, true, StandardCharsets.UTF_8)); } - private void assertJsonNodeIsTraceMessage(JsonNode jsonNode) { + private void assertJsonNodeIsTraceMessage(final JsonNode jsonNode) { // todo: this check could be better by actually trying to convert the JsonNode to an // AirbyteTraceMessage instance Assertions.assertEquals("TRACE", jsonNode.get("type").asText()); @@ -36,7 +36,7 @@ private void assertJsonNodeIsTraceMessage(JsonNode jsonNode) { @Test void testEmitSystemErrorTrace() { AirbyteTraceMessageUtility.emitSystemErrorTrace(Mockito.mock(RuntimeException.class), "this is a system error"); - JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); + final JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); assertJsonNodeIsTraceMessage(outJson); Assertions.assertEquals("system_error", outJson.get("trace").get("error").get("failure_type").asText()); } @@ -44,7 +44,7 @@ void testEmitSystemErrorTrace() { @Test void testEmitConfigErrorTrace() { AirbyteTraceMessageUtility.emitConfigErrorTrace(Mockito.mock(RuntimeException.class), "this is a config error"); - JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); + final JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); assertJsonNodeIsTraceMessage(outJson); Assertions.assertEquals("config_error", outJson.get("trace").get("error").get("failure_type").asText()); } @@ -58,11 +58,11 @@ void testEmitErrorTrace() { @Test void testCorrectStacktraceFormat() { try { - int x = 1 / 0; - } catch (Exception e) { + final int x = 1 / 0; + } catch (final Exception e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "you exploded the universe"); } - JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); + final JsonNode outJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); Assertions.assertTrue(outJson.get("trace").get("error").get("stack_trace").asText().contains("\n\tat")); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java index 2d06503baf20..68044162bb72 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java @@ -33,18 +33,21 @@ public void testInitialization() { assertThrows(IllegalStateException.class, DestinationConfig::getInstance); // good initialization - DestinationConfig.initialize(NODE); + DestinationConfig.initialize(NODE, true); assertNotNull(DestinationConfig.getInstance()); assertEquals(NODE, DestinationConfig.getInstance().root); + assertEquals(true, DestinationConfig.getInstance().getIsV2Destination()); // initializing again doesn't change the config final JsonNode nodeUnused = Jsons.deserialize("{}"); - DestinationConfig.initialize(nodeUnused); + DestinationConfig.initialize(nodeUnused, false); assertEquals(NODE, DestinationConfig.getInstance().root); + assertEquals(true, DestinationConfig.getInstance().getIsV2Destination()); } @Test public void testValues() { + DestinationConfig.clearInstance(); DestinationConfig.initialize(NODE); assertEquals("bar", DestinationConfig.getInstance().getTextValue("foo")); @@ -60,6 +63,8 @@ public void testValues() { assertEquals(Jsons.deserialize("\"bar\""), DestinationConfig.getInstance().getNodeValue("foo")); assertEquals(Jsons.deserialize("true"), DestinationConfig.getInstance().getNodeValue("baz")); assertNull(DestinationConfig.getInstance().getNodeValue("blah")); + + assertEquals(false, DestinationConfig.getInstance().getIsV2Destination()); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index f474b4a4070f..cb5498b70487 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -37,13 +37,14 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -237,7 +238,7 @@ void testExceptionAfterNoStateMessages() throws Exception { @Test void testExceptionDuringOnClose() throws Exception { - doThrow(new IllegalStateException("induced exception")).when(onClose).accept(false); + doThrow(new IllegalStateException("induced exception")).when(onClose).accept(false, new HashMap<>()); final List expectedRecordsBatch1 = generateRecords(1_000); final List expectedRecordsBatch2 = generateRecords(1_000); @@ -507,13 +508,13 @@ private BufferedStreamConsumer getConsumerWithFlushFrequency() { private void verifyStartAndClose() throws Exception { verify(onStart).call(); - verify(onClose).accept(false); + verify(onClose).accept(false, new HashMap<>()); } /** Indicates that a failure occurred while consuming AirbyteMessages */ private void verifyStartAndCloseFailure() throws Exception { verify(onStart).call(); - verify(onClose).accept(true); + verify(onClose).accept(true, new HashMap<>()); } private static void consumeRecords(final BufferedStreamConsumer consumer, final Collection records) { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java index 0ea5e57599a3..67bc7c7dc427 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java @@ -31,6 +31,7 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -50,7 +51,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -137,7 +138,14 @@ void test1StreamWith1State() throws Exception { verifyRecords(STREAM_NAME, SCHEMA_NAME, expectedRecords); - verify(outputRecordCollector).accept(STATE_MESSAGE1); + final AirbyteMessage stateMessageWithDestinationStatsUpdated = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(STREAM1_DESC).withStreamState(Jsons.jsonNode(1))) + .withDestinationStats(new AirbyteStateStats().withRecordCount((double) expectedRecords.size()))); + + verify(outputRecordCollector).accept(stateMessageWithDestinationStatsUpdated); } @Test @@ -154,7 +162,14 @@ void test1StreamWith2State() throws Exception { verifyRecords(STREAM_NAME, SCHEMA_NAME, expectedRecords); - verify(outputRecordCollector, times(1)).accept(STATE_MESSAGE2); + final AirbyteMessage stateMessageWithDestinationStatsUpdated = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(STREAM1_DESC).withStreamState(Jsons.jsonNode(2))) + .withDestinationStats(new AirbyteStateStats().withRecordCount(0.0))); + + verify(outputRecordCollector, times(1)).accept(stateMessageWithDestinationStatsUpdated); } @Test @@ -186,7 +201,7 @@ void testBackPressure() throws Exception { consumer = new AsyncStreamConsumer( m -> {}, () -> {}, - (hasFailed) -> {}, + (hasFailed, recordCounts) -> {}, flushFunction, CATALOG, new BufferManager(1024 * 10), @@ -365,7 +380,7 @@ private static List generateRecords(final long targetSizeInBytes private void verifyStartAndClose() throws Exception { verify(onStart).call(); - verify(onClose).accept(any()); + verify(onClose).accept(any(), any()); } @SuppressWarnings({"unchecked", "SameParameterValue"}) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java index eb345f9b0c69..669579c7af96 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java @@ -21,7 +21,7 @@ public class BufferDequeueTest { private static final int RECORD_SIZE_20_BYTES = 20; - public static final String RECORD_20_BYTES = "abc"; + private static final String DEFAULT_NAMESPACE = "foo_namespace"; private static final String STREAM_NAME = "stream1"; private static final StreamDescriptor STREAM_DESC = new StreamDescriptor().withName(STREAM_NAME); private static final PartialAirbyteMessage RECORD_MSG_20_BYTES = new PartialAirbyteMessage() @@ -38,10 +38,10 @@ void testTakeShouldBestEffortRead() { final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); // total size of records is 80, so we expect 50 to get us 2 records (prefer to under-pull records // than over-pull). @@ -60,9 +60,9 @@ void testTakeShouldReturnAllIfPossible() { final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, 60)) { assertEquals(3, take.getData().size()); @@ -77,8 +77,8 @@ void testTakeFewerRecordsThanSizeLimitShouldNotError() { final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, Long.MAX_VALUE)) { assertEquals(2, take.getData().size()); @@ -95,13 +95,13 @@ void testMetadataOperationsCorrect() { final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); final var secondStream = new StreamDescriptor().withName("stream_2"); final PartialAirbyteMessage recordFromSecondStream = Jsons.clone(RECORD_MSG_20_BYTES); recordFromSecondStream.getRecord().withStream(secondStream.getName()); - enqueue.addRecord(recordFromSecondStream, RECORD_SIZE_20_BYTES); + enqueue.addRecord(recordFromSecondStream, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); assertEquals(60, dequeue.getTotalGlobalQueueSizeBytes()); @@ -144,12 +144,12 @@ void cleansUpMemoryForEmptyQueues() throws Exception { assertEquals(BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); // allocate a block for new stream - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); assertEquals(2 * BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); // no re-allocates as we haven't breached block size assertEquals(2 * BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java index 11e61c6e4eb9..a555c403e5c0 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java @@ -19,6 +19,7 @@ public class BufferEnqueueTest { private static final int RECORD_SIZE_20_BYTES = 20; + private static final String DEFAULT_NAMESPACE = "foo_namespace"; @Test void testAddRecordShouldAdd() { @@ -33,7 +34,7 @@ final var record = new PartialAirbyteMessage() .withRecord(new PartialAirbyteRecordMessage() .withStream(streamName)); - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); assertEquals(1, streamToBuffer.get(stream).size()); assertEquals(20L, streamToBuffer.get(stream).getCurrentMemoryUsage()); @@ -53,8 +54,8 @@ final var record = new PartialAirbyteMessage() .withRecord(new PartialAirbyteRecordMessage() .withStream(streamName)); - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); assertEquals(2, streamToBuffer.get(stream).size()); assertEquals(40, streamToBuffer.get(stream).getCurrentMemoryUsage()); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java index 8c6f3ecf2e3b..b77c4419cd1c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java @@ -8,69 +8,140 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteStateMessage; import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteStreamState; +import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.List; -import java.util.Set; +import java.util.*; +import java.util.stream.Collectors; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class GlobalAsyncStateManagerTest { private static final long TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES = 100 * 1024 * 1024; // 10MB - + private static final String DEFAULT_NAMESPACE = "foo_namespace"; private static final long STATE_MSG_SIZE = 1000; + private static final String NAMESPACE = "namespace"; private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = STREAM_NAME + 2; private static final String STREAM_NAME3 = STREAM_NAME + 3; private static final StreamDescriptor STREAM1_DESC = new StreamDescriptor() - .withName(STREAM_NAME); + .withName(STREAM_NAME).withNamespace(NAMESPACE); private static final StreamDescriptor STREAM2_DESC = new StreamDescriptor() - .withName(STREAM_NAME2); + .withName(STREAM_NAME2).withNamespace(NAMESPACE); private static final StreamDescriptor STREAM3_DESC = new StreamDescriptor() - .withName(STREAM_NAME3); + .withName(STREAM_NAME3).withNamespace(NAMESPACE); private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE1 = new PartialAirbyteMessage() .withType(Type.STATE) .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.GLOBAL)); + .withType(AirbyteStateType.GLOBAL)) + .withSerialized(serializedState(STREAM1_DESC, AirbyteStateType.GLOBAL, Jsons.jsonNode(ImmutableMap.of("cursor", 1)))); private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE2 = new PartialAirbyteMessage() .withType(Type.STATE) .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.GLOBAL)); + .withType(AirbyteStateType.GLOBAL)) + .withSerialized(serializedState(STREAM2_DESC, AirbyteStateType.GLOBAL, Jsons.jsonNode(ImmutableMap.of("cursor", 2)))); + + private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE3 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL)) + .withSerialized(serializedState(STREAM3_DESC, AirbyteStateType.GLOBAL, Jsons.jsonNode(ImmutableMap.of("cursor", 2)))); private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE1 = new PartialAirbyteMessage() .withType(Type.STATE) .withState(new PartialAirbyteStateMessage() .withType(AirbyteStateType.STREAM) - .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))) + .withSerialized(serializedState(STREAM1_DESC, AirbyteStateType.STREAM, Jsons.jsonNode(ImmutableMap.of("cursor", 1)))); private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE2 = new PartialAirbyteMessage() .withType(Type.STATE) .withState(new PartialAirbyteStateMessage() .withType(AirbyteStateType.STREAM) - .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))) + .withSerialized(serializedState(STREAM1_DESC, AirbyteStateType.STREAM, Jsons.jsonNode(ImmutableMap.of("cursor", 2)))); + + private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE3 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))) + .withSerialized(serializedState(STREAM1_DESC, AirbyteStateType.STREAM, Jsons.jsonNode(ImmutableMap.of("cursor", 3)))); + private static final PartialAirbyteMessage STREAM2_STATE_MESSAGE = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM2_DESC))) + .withSerialized(serializedState(STREAM2_DESC, AirbyteStateType.STREAM, Jsons.jsonNode(ImmutableMap.of("cursor", 4)))); + + public static String serializedState(final StreamDescriptor streamDescriptor, final AirbyteStateType type, final JsonNode state) { + switch (type) { + case GLOBAL -> { + return Jsons.serialize(new AirbyteMessage().withType(Type.STATE).withState( + new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(new AirbyteGlobalState() + .withSharedState(state) + .withStreamStates(Collections.singletonList(new AirbyteStreamState() + .withStreamState(Jsons.emptyObject()) + .withStreamDescriptor(streamDescriptor)))))); + + } + case STREAM -> { + return Jsons.serialize(new AirbyteMessage().withType(Type.STATE).withState( + new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamState(state) + .withStreamDescriptor(streamDescriptor)))); + } + default -> throw new RuntimeException("LEGACY STATE NOT SUPPORTED"); + } + } @Test void testBasic() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); final var firstStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); final var secondStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); assertEquals(firstStateId, secondStateId); stateManager.decrement(firstStateId, 2); + stateManager.flushStates(emittedStatesFromDestination::add); // because no state message has been tracked, there is nothing to flush yet. - var flushed = stateManager.flushStates(); - assertEquals(0, flushed.size()); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals(0, stateWithStats.size()); + + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.flushStates(emittedStatesFromDestination::add); + + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(2.0); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats2.values().stream().toList()); + } - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - flushed = stateManager.flushStates(); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), flushed); + public AirbyteMessage attachDestinationStateStats(final AirbyteMessage stateMessage, final AirbyteStateStats airbyteStateStats) { + stateMessage.getState().withDestinationStats(airbyteStateStats); + return stateMessage; } @Nested @@ -78,68 +149,183 @@ class GlobalState { @Test void testEmptyQueuesGlobalState() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); // GLOBAL - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); - - assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE)); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(0.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + // + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + + assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE)); } @Test void testConversion() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); final var preConvertId2 = simulateIncomingRecords(STREAM3_DESC, 10, stateManager); assertEquals(3, Set.of(preConvertId0, preConvertId1, preConvertId2).size()); - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); // Since this is actually a global state, we can only flush after all streams are done. stateManager.decrement(preConvertId0, 10); - assertEquals(List.of(), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + assertEquals(0, emittedStatesFromDestination.size()); stateManager.decrement(preConvertId1, 10); - assertEquals(List.of(), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + assertEquals(0, emittedStatesFromDestination.size()); stateManager.decrement(preConvertId2, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(30.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + } @Test void testCorrectFlushingOneStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(preConvertId0, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(10.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + + emittedStatesFromDestination.clear(); final var afterConvertId1 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE); + stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(afterConvertId1, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE2.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats2.values().stream().toList()); + } + + @Test + void testZeroRecordFlushing() { + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(preConvertId0, 10); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(10.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + emittedStatesFromDestination.clear(); + + stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats2 = new AirbyteStateStats().withRecordCount(0.0); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE2.getSerialized(), AirbyteMessage.class), expectedDestinationStats2)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats2), stateWithStats2.values().stream().toList()); + emittedStatesFromDestination.clear(); + + final var afterConvertId2 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(GLOBAL_STATE_MESSAGE3, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(afterConvertId2, 10); + stateManager.flushStates(emittedStatesFromDestination::add); + final Map stateWithStats3 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE3.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats3.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats3.values().stream().toList()); } @Test void testCorrectFlushingManyStreams() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); assertNotEquals(preConvertId0, preConvertId1); - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(preConvertId0, 10); stateManager.decrement(preConvertId1, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(20.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + emittedStatesFromDestination.clear(); final var afterConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); final var afterConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); assertEquals(afterConvertId0, afterConvertId1); - stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE); + stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(afterConvertId0, 20); - assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(GLOBAL_STATE_MESSAGE2.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats2.values().stream().toList()); } } @@ -149,48 +335,148 @@ class PerStreamState { @Test void testEmptyQueues() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); // GLOBAL - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); - - assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE)); + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(0.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + + assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE)); } @Test void testCorrectFlushingOneStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); var stateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(stateId, 3); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(3.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + + emittedStatesFromDestination.clear(); stateId = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE); + stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(stateId, 10); - assertEquals(List.of(STREAM1_STATE_MESSAGE2), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats2 = new AirbyteStateStats().withRecordCount(10.0); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals(List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE2.getSerialized(), AirbyteMessage.class), expectedDestinationStats2)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats2), stateWithStats2.values().stream().toList()); + } + + @Test + void testZeroRecordFlushing() { + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + var stateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stateId, 3); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(3.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + emittedStatesFromDestination.clear(); + + stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.flushStates(emittedStatesFromDestination::add); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + final AirbyteStateStats expectedDestinationStats2 = new AirbyteStateStats().withRecordCount(0.0); + assertEquals(List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE2.getSerialized(), AirbyteMessage.class), expectedDestinationStats2)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats2), stateWithStats2.values().stream().toList()); + emittedStatesFromDestination.clear(); + + stateId = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(STREAM1_STATE_MESSAGE3, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stateId, 10); + stateManager.flushStates(emittedStatesFromDestination::add); + final Map stateWithStats3 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + final AirbyteStateStats expectedDestinationStats3 = new AirbyteStateStats().withRecordCount(10.0); + assertEquals(List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE3.getSerialized(), AirbyteMessage.class), expectedDestinationStats3)), + stateWithStats3.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats3), stateWithStats3.values().stream().toList()); } @Test void testCorrectFlushingManyStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); final var stream1StateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); final var stream2StateId = simulateIncomingRecords(STREAM2_DESC, 7, stateManager); - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(stream1StateId, 3); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats = new AirbyteStateStats().withRecordCount(3.0); + final Map stateWithStats = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM1_STATE_MESSAGE1.getSerialized(), AirbyteMessage.class), expectedDestinationStats)), + stateWithStats.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats), stateWithStats.values().stream().toList()); + emittedStatesFromDestination.clear(); stateManager.decrement(stream2StateId, 4); - assertEquals(List.of(), stateManager.flushStates()); - stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE); + stateManager.flushStates(emittedStatesFromDestination::add); + assertEquals(List.of(), emittedStatesFromDestination); + stateManager.trackState(STREAM2_STATE_MESSAGE, STATE_MSG_SIZE, DEFAULT_NAMESPACE); stateManager.decrement(stream2StateId, 3); // only flush state if counter is 0. - assertEquals(List.of(STREAM1_STATE_MESSAGE2), stateManager.flushStates()); + stateManager.flushStates(emittedStatesFromDestination::add); + final AirbyteStateStats expectedDestinationStats2 = new AirbyteStateStats().withRecordCount(7.0); + final Map stateWithStats2 = + emittedStatesFromDestination.stream() + .collect(Collectors.toMap(c -> c, c -> c.getState().getDestinationStats())); + assertEquals( + List.of( + attachDestinationStateStats(Jsons.deserialize(STREAM2_STATE_MESSAGE.getSerialized(), AirbyteMessage.class), expectedDestinationStats2)), + stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(expectedDestinationStats2), stateWithStats2.values().stream().toList()); } } @@ -203,4 +489,20 @@ private static long simulateIncomingRecords(final StreamDescriptor desc, final l return stateId; } + @Test + void flushingRecordsShouldNotReduceStatsCounterForGlobalState() { + final List emittedStatesFromDestination = new ArrayList<>(); + final GlobalAsyncStateManager stateManager = + new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + final long stateId = simulateIncomingRecords(STREAM1_DESC, 6, stateManager); + stateManager.decrement(stateId, 4); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, 1, STREAM1_DESC.getNamespace()); + stateManager.flushStates(emittedStatesFromDestination::add); + assertEquals(0, emittedStatesFromDestination.size()); + stateManager.decrement(stateId, 2); + stateManager.flushStates(emittedStatesFromDestination::add); + assertEquals(1, emittedStatesFromDestination.size()); + assertEquals(6.0, emittedStatesFromDestination.getFirst().getState().getDestinationStats().getRecordCount()); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt deleted file mode 100644 index 5094aa2fd4f6..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt +++ /dev/null @@ -1,23 +0,0 @@ -create table "public"."airbyte_toy_migrations"( - "installed_rank" int4 not null, - "version" varchar(50) null, - "description" varchar(200) not null, - "type" varchar(20) not null, - "script" varchar(1000) not null, - "checksum" int4 null, - "installed_by" varchar(100) not null, - "installed_on" timestamp(29) not null default null, - "execution_time" int4 not null, - "success" bool not null, - constraint "airbyte_toy_migrations_pk" - primary key ("installed_rank") -); -create table "public"."toy_cars"( - "id" int8 generated by default as identity not null, - "value" varchar(50) null, - constraint "toy_cars_pkey" - primary key ("id") -); -create unique index "airbyte_toy_migrations_pk" on "public"."airbyte_toy_migrations"("installed_rank" asc); -create index "airbyte_toy_migrations_s_idx" on "public"."airbyte_toy_migrations"("success" asc); -create unique index "toy_cars_pkey" on "public"."toy_cars"("id" asc); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql deleted file mode 100644 index d32999831ce5..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE - TABLE - IF NOT EXISTS TOY_CARS( - id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - value VARCHAR(50) - ); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt deleted file mode 100644 index efca08018f9f..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt +++ /dev/null @@ -1,24 +0,0 @@ -create table "public"."airbyte_toy_migrations"( - "installed_rank" int4 not null, - "version" varchar(50) null, - "description" varchar(200) not null, - "type" varchar(20) not null, - "script" varchar(1000) not null, - "checksum" int4 null, - "installed_by" varchar(100) not null, - "installed_on" timestamp(29) not null default null, - "execution_time" int4 not null, - "success" bool not null, - constraint "airbyte_toy_migrations_pk" - primary key ("installed_rank") -); -create table "public"."toy_cars"( - "id" int8 generated by default as identity not null, - "value" varchar(50) null, - "created_at" timestamp(29) not null default null, - constraint "toy_cars_pkey" - primary key ("id") -); -create unique index "airbyte_toy_migrations_pk" on "public"."airbyte_toy_migrations"("installed_rank" asc); -create index "airbyte_toy_migrations_s_idx" on "public"."airbyte_toy_migrations"("success" asc); -create unique index "toy_cars_pkey" on "public"."toy_cars"("id" asc); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/extensions/LoggingInvocationInterceptor.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/extensions/LoggingInvocationInterceptor.java new file mode 100644 index 000000000000..6ccaf15a1255 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/extensions/LoggingInvocationInterceptor.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.extensions; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.junit.jupiter.api.extension.DynamicTestInvocationContext; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * By default, junit only output logs to the console, and nothing makes it into log4j logs. This + * class fixes that by using the interceptor facility to print progress and timing information. This + * allows us to have junit loglines in our test logs. This is instanciated via Java's + * ServiceLoader The declaration can be found in + * resources/META-INF/services/org.junit.jupiter.api.extension.Extension + */ +public class LoggingInvocationInterceptor implements InvocationInterceptor { + + private static final class LoggingInvocationInterceptorHandler implements InvocationHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingInvocationInterceptor.class); + + private static final Pattern methodPattern = Pattern.compile("intercept(.*)Method"); + + @Override + @SuppressWarnings("unchecked") + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (LoggingInvocationInterceptor.class.getDeclaredMethod(method.getName(), Invocation.class, ReflectiveInvocationContext.class, + ExtensionContext.class) == null) { + LOGGER.error("Junit LoggingInvocationInterceptor executing unknown interception point {}", method.getName()); + return method.invoke(proxy, args); + } + var invocation = (Invocation) args[0]; + var invocationContext = (ReflectiveInvocationContext) args[1]; + var extensionContext = (ExtensionContext) args[2]; + String methodName = method.getName(); + String logLineSuffix; + Matcher methodMatcher = methodPattern.matcher(methodName); + if (methodName.equals("interceptDynamicTest")) { + logLineSuffix = "execution of DynamicTest %s".formatted(extensionContext.getDisplayName()); + } else if (methodName.equals("interceptTestClassConstructor")) { + logLineSuffix = "instance creation for %s".formatted(invocationContext.getTargetClass()); + } else if (methodMatcher.matches()) { + String interceptedEvent = methodMatcher.group(1); + logLineSuffix = "execution of @%s method %s.%s".formatted(invocationContext.getExecutable().getDeclaringClass().getSimpleName(), + interceptedEvent, invocationContext.getExecutable().getName()); + } else { + logLineSuffix = "execution of unknown intercepted call %s".formatted(methodName); + } + LOGGER.info("Junit starting {}", logLineSuffix); + try { + Instant start = Instant.now(); + Object retVal = invocation.proceed(); + long elapsedMs = Duration.between(start, Instant.now()).toMillis(); + LOGGER.info("Junit completed {} in {} ms", logLineSuffix, elapsedMs); + return retVal; + } catch (Throwable t) { + String stackTrace = Arrays.stream(ExceptionUtils.getStackFrames(t)).takeWhile(s -> !s.startsWith("\tat org.junit")).collect( + Collectors.joining("\n ")); + LOGGER.warn("Junit exception throw during {}:\n{}", logLineSuffix, stackTrace); + throw t; + } + } + + } + + private final InvocationInterceptor proxy = (InvocationInterceptor) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {InvocationInterceptor.class}, + new LoggingInvocationInterceptorHandler()); + + @Override + public void interceptAfterAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptAfterAllMethod(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptAfterEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptAfterEachMethod(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptBeforeAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptBeforeAllMethod(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptBeforeEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptBeforeEachMethod(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptDynamicTest(Invocation invocation, + DynamicTestInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptDynamicTest(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptTestMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptTestMethod(invocation, invocationContext, extensionContext); + } + + @Override + public void interceptTestTemplateMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + proxy.interceptTestTemplateMethod(invocation, invocationContext, extensionContext); + } + + @Override + public T interceptTestFactoryMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + return proxy.interceptTestFactoryMethod(invocation, invocationContext, extensionContext); + } + + @Override + public T interceptTestClassConstructor(Invocation invocation, + ReflectiveInvocationContext> invocationContext, + ExtensionContext extensionContext) + throws Throwable { + return proxy.interceptTestClassConstructor(invocation, invocationContext, extensionContext); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java similarity index 76% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java index 07a1786f60dd..20c395d2e720 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java @@ -4,8 +4,6 @@ package io.airbyte.cdk.integrations.base.ssh; -import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getInnerContainerAddress; -import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getOuterContainerAddress; import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; @@ -16,6 +14,8 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.Network; @@ -98,4 +98,30 @@ public GenericContainer getContainer() { return bastion; } + /** + * Returns the inner docker network ip address and port of a container. This can be used to reach a + * container from another container running on the same network + * + * @param container container + * @return a pair of host and port + */ + public static ImmutablePair getInnerContainerAddress(final Container container) { + return ImmutablePair.of( + container.getContainerInfo().getNetworkSettings().getNetworks().entrySet().stream().findFirst().get().getValue().getIpAddress(), + (Integer) container.getExposedPorts().stream().findFirst().get()); + } + + /** + * Returns the outer docker network ip address and port of a container. This can be used to reach a + * container from the host machine + * + * @param container container + * @return a pair of host and port + */ + public static ImmutablePair getOuterContainerAddress(final Container container) { + return ImmutablePair.of( + container.getHost(), + container.getFirstMappedPort()); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java similarity index 78% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java index 7b8b4f4ccfc1..4d29d36c9848 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java @@ -4,8 +4,6 @@ package io.airbyte.cdk.integrations.util; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.Objects; import org.testcontainers.containers.GenericContainer; @@ -23,13 +21,6 @@ public static String resolveIpAddress(GenericContainer container) { return getIpAddress(container); } - public static String encodeValue(final String value) { - if (value != null) { - return URLEncoder.encode(value, StandardCharsets.UTF_8); - } - return null; - } - private static String getIpAddress(GenericContainer container) { return Objects.requireNonNull(container.getContainerInfo() .getNetworkSettings() diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java new file mode 100644 index 000000000000..6c0b8e40e89f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.testutils; + +import io.airbyte.commons.logging.LoggingHelper; +import io.airbyte.commons.logging.MdcScope; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; + +/** + * ContainerFactory is the companion to {@link TestDatabase} and provides it with suitable + * testcontainer instances. + */ +public abstract class ContainerFactory> { + + static private final Logger LOGGER = LoggerFactory.getLogger(ContainerFactory.class); + + private record ContainerKey(Class clazz, DockerImageName imageName, List methods) {}; + + private static class ContainerOrException { + + private final Supplier> containerSupplier; + private volatile RuntimeException _exception = null; + private volatile GenericContainer _container = null; + + ContainerOrException(Supplier> containerSupplier) { + this.containerSupplier = containerSupplier; + } + + GenericContainer container() { + if (_exception == null && _container == null) { + synchronized (this) { + if (_container == null && _exception == null) { + try { + _container = containerSupplier.get(); + if (_container == null) { + throw new IllegalStateException("testcontainer instance was not constructed"); + } + } catch (RuntimeException e) { + _exception = e; + } + } + } + } + if (_exception != null) { + throw _exception; + } + return _container; + } + + } + + private static final ConcurrentMap SHARED_CONTAINERS = new ConcurrentHashMap<>(); + private static final AtomicInteger containerId = new AtomicInteger(0); + + private static final MdcScope.Builder getTestContainerLogMdcBuilder(DockerImageName imageName, List methods) { + return new MdcScope.Builder() + .setLogPrefix("testcontainer %s (%s[%s]):".formatted(containerId.incrementAndGet(), imageName, StringUtils.join(methods, ","))) + .setPrefixColor(LoggingHelper.Color.RED_BACKGROUND); + } + + /** + * Creates a new, unshared testcontainer instance. This usually wraps the default constructor for + * the testcontainer type. + */ + protected abstract C createNewContainer(DockerImageName imageName); + + /** + * Returns a shared instance of the testcontainer. + */ + @SuppressWarnings("unchecked") + public final C shared(String imageName, String... methods) { + final var containerKey = new ContainerKey(getClass(), DockerImageName.parse(imageName), Stream.of(methods).toList()); + // We deliberately avoid creating the container itself eagerly during the evaluation of the map + // value. + // Container creation can be exceedingly slow. + // Furthermore, we need to handle exceptions raised during container creation. + ContainerOrException containerOrError = SHARED_CONTAINERS.computeIfAbsent(containerKey, + key -> new ContainerOrException(() -> createAndStartContainer(key.imageName(), key.methods()))); + // Instead, the container creation (if applicable) is deferred to here. + return (C) containerOrError.container(); + } + + /** + * Returns an exclusive instance of the testcontainer. + */ + @SuppressWarnings("unchecked") + public final C exclusive(String imageName, String... methods) { + return (C) createAndStartContainer(DockerImageName.parse(imageName), Stream.of(methods).toList()); + } + + private GenericContainer createAndStartContainer(DockerImageName imageName, List methodNames) { + LOGGER.info("Creating new shared container based on {} with {}.", imageName, methodNames); + try { + GenericContainer container = createNewContainer(imageName); + final var methods = new ArrayList(); + for (String methodName : methodNames) { + methods.add(getClass().getMethod(methodName, container.getClass())); + } + final var logConsumer = new Slf4jLogConsumer(LOGGER) { + + public void accept(OutputFrame frame) { + if (frame.getUtf8StringWithoutLineEnding().trim().length() > 0) { + super.accept(frame); + } + } + + }; + getTestContainerLogMdcBuilder(imageName, methodNames).produceMappings(logConsumer::withMdc); + container.withLogConsumer(logConsumer); + for (Method method : methods) { + LOGGER.info("Calling {} in {} on new shared container based on {}.", + method.getName(), getClass().getName(), imageName); + method.invoke(this, container); + } + container.start(); + return container; + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java new file mode 100644 index 000000000000..badf004d4f99 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.testutils; + +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** + * This is used when a source (such as Snowflake) relies on an always-on resource and therefore + * doesn't need an actual container. compatible + */ +public class NonContainer extends JdbcDatabaseContainer { + + private final String username; + private final String password; + private final String jdbcUrl; + + private final String driverClassName; + + public NonContainer(final String userName, + final String password, + final String jdbcUrl, + final String driverClassName, + final String dockerImageName) { + super(dockerImageName); + this.username = userName; + this.password = password; + this.jdbcUrl = jdbcUrl; + this.driverClassName = driverClassName; + } + + @Override + public String getDriverClassName() { + return driverClassName; + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getUsername() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java similarity index 95% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java index 6a5d80104718..f8ef633ae989 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java @@ -12,6 +12,7 @@ import io.airbyte.cdk.db.factory.DataSourceFactory; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -102,7 +103,8 @@ final public T initialized() { getPassword(), getDatabaseDriver().getDriverClassName(), getJdbcUrl(), - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, getDatabaseDriver().getDriverClassName())); this.dslContext = DSLContextFactory.create(dataSource, getSqlDialect()); return self(); } @@ -165,10 +167,13 @@ public Database getDatabase() { return new Database(getDslContext()); } - protected void execSQL(Stream sql) { + protected void execSQL(final Stream sql) { try { getDatabase().query(ctx -> { - sql.forEach(ctx::execute); + sql.forEach(statement -> { + LOGGER.debug("{}", statement); + ctx.execute(statement); + }); return null; }); } catch (SQLException e) { @@ -228,7 +233,6 @@ public B integrationTestConfigBuilder() { @Override public void close() { execSQL(this.cleanupSQL.stream()); - dslContext.close(); execInContainer(inContainerUndoBootstrapCmd()); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 000000000000..90378b469448 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1 @@ +io.airbyte.cdk.extensions.LoggingInvocationInterceptor \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle new file mode 100644 index 000000000000..7f35ad042821 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + api 'com.google.cloud:google-cloud-bigquery:2.37.0' + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle new file mode 100644 index 000000000000..d4d81f958939 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle @@ -0,0 +1,15 @@ +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-try,-unchecked" + } +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + api 'org.mongodb:mongodb-driver-sync:4.10.2' + + testFixturesApi 'org.testcontainers:mongodb:1.19.0' +} diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java similarity index 98% rename from airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java rename to airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java index fe6cffd02290..8b87aaf0eac0 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.mongodb; +package io.airbyte.cdk.db.mongodb; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java new file mode 100644 index 000000000000..c05844f81652 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +public class MongoDatabaseException extends RuntimeException { + + public static final String MONGO_DATA_BASE_NOT_FOUND = "Data Base with given name - %s not found."; + + public MongoDatabaseException(final String databaseName) { + super(String.format(MONGO_DATA_BASE_NOT_FOUND, databaseName)); + } + +} diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java similarity index 98% rename from airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java rename to airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java index 40a0df293b6b..ac607c1d2943 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.mongodb; +package io.airbyte.cdk.db.mongodb; import static java.util.Arrays.asList; import static org.bson.BsonType.ARRAY; @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.api.client.util.DateTime; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.mongodb.DBRefCodecProvider; @@ -31,6 +30,7 @@ import io.airbyte.commons.util.MoreIterators; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.JsonSchemaType; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -121,8 +121,8 @@ public static Object getBsonValue(final BsonType type, final String value) { case INT64 -> new BsonInt64(Long.parseLong(value)); case DOUBLE -> new BsonDouble(Double.parseDouble(value)); case DECIMAL128 -> Decimal128.parse(value); - case TIMESTAMP -> new BsonTimestamp(new DateTime(value).getValue()); - case DATE_TIME -> new BsonDateTime(new DateTime(value).getValue()); + case TIMESTAMP -> new BsonTimestamp((int) Instant.parse(value).getEpochSecond(), 0); + case DATE_TIME -> new BsonDateTime(Instant.parse(value).toEpochMilli()); case OBJECT_ID -> new ObjectId(value); case SYMBOL -> new Symbol(value); case STRING -> new BsonString(value); diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java similarity index 95% rename from airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java rename to airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java index 672030f750de..214d14c0f099 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.mongodb; +package io.airbyte.cdk.db.mongodb; import java.util.LinkedList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java similarity index 93% rename from airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java index 46af33f9c3ab..18f2828e0860 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.mongodb; +package io.airbyte.cdk.db.mongodb; -import static io.airbyte.integrations.destination.mongodb.MongoUtils.AIRBYTE_SUFFIX; +import static io.airbyte.cdk.db.mongodb.MongoUtils.AIRBYTE_SUFFIX; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle new file mode 100644 index 000000000000..868702b6c080 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle @@ -0,0 +1,15 @@ +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-deprecation,-this-escape" + } +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + api 'org.postgresql:postgresql:42.6.0' + + testFixturesApi 'org.testcontainers:postgresql:1.19.0' +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PgLsn.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PgLsn.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PostgresUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PostgresUtils.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PgLsnTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PgLsnTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index 2a8b2bb524e1..8e43c23355bd 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -1,93 +1,28 @@ - java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-deprecation" + options.compilerArgs += "-Xlint:-deprecation,-removal,-this-escape" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-try,-this-escape" } } dependencies { - // Depends on core CDK classes (OK 👍) - implementation project(':airbyte-cdk:java:airbyte-cdk:core') + api 'org.apache.commons:commons-csv:1.10.0' + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') compileOnly project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - - - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + implementation 'io.aesy:datasize:1.0.0' - testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" - - implementation libs.bundles.junit - // implementation libs.junit.jupiter.api - implementation libs.junit.jupiter.params - implementation 'org.junit.platform:junit-platform-launcher:1.7.0' - implementation libs.jooq - testImplementation libs.junit.jupiter.engine - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' - implementation "io.aesy:datasize:1.0.0" - implementation libs.apache.commons - implementation libs.apache.commons.lang - testImplementation 'commons-lang:commons-lang:2.6' - implementation 'commons-cli:commons-cli:1.4' - implementation 'org.apache.commons:commons-csv:1.4' - - implementation libs.google.cloud.storage - - // Optional dependencies - // TODO: Change these to 'compileOnly' or 'testCompileOnly' - implementation 'com.azure:azure-storage-blob:12.12.0' - implementation('com.google.cloud:google-cloud-bigquery:1.133.1') - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - implementation libs.postgresql - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - - // testImplementation libs.junit.jupiter.api - implementation libs.hikaricp - implementation libs.debezium.api - implementation libs.debezium.embedded - implementation libs.debezium.sqlserver - implementation libs.debezium.mysql - implementation libs.debezium.postgres - implementation libs.debezium.mongodb - - implementation libs.bundles.datadog - // implementation 'com.datadoghq:dd-trace-api' - implementation 'org.apache.sshd:sshd-mina:2.8.0' - - implementation libs.testcontainers - implementation libs.testcontainers.mysql - implementation libs.testcontainers.jdbc - implementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.jdbc - testImplementation libs.testcontainers.mysql - testImplementation libs.testcontainers.postgresql - implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - - implementation 'org.bouncycastle:bcprov-jdk15on:1.66' - - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' - - implementation ('org.apache.hadoop:hadoop-common:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} + testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation libs.junit.jupiter.system.stubs + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 8bcf64c04887..b12fd56c93f7 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -5,13 +5,15 @@ package io.airbyte.cdk.integrations.destination.jdbc; import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.util.ConfiguredCatalogUtilKt.addDefaultNamespaceToStreams; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import io.airbyte.cdk.db.factory.DataSourceFactory; import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.base.Destination; @@ -30,14 +32,15 @@ import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -46,11 +49,10 @@ import java.util.function.Consumer; import javax.sql.DataSource; import org.apache.commons.lang3.NotImplementedException; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractJdbcDestination extends BaseConnector implements Destination { +public abstract class AbstractJdbcDestination extends JdbcConnector implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcDestination.class); @@ -58,7 +60,6 @@ public abstract class AbstractJdbcDestination extends BaseConnector implements D public static final String DISABLE_TYPE_DEDUPE = "disable_type_dedupe"; - private final String driverClass; private final NamingConventionTransformer namingResolver; private final SqlOperations sqlOperations; @@ -70,10 +71,14 @@ protected SqlOperations getSqlOperations() { return sqlOperations; } + protected String getConfigSchemaKey() { + return "schema"; + } + public AbstractJdbcDestination(final String driverClass, final NamingConventionTransformer namingResolver, final SqlOperations sqlOperations) { - this.driverClass = driverClass; + super(driverClass); this.namingResolver = namingResolver; this.sqlOperations = sqlOperations; } @@ -90,6 +95,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { final var v2RawSchema = namingResolver.getIdentifier(TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) .orElse(JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE)); attemptTableOperations(v2RawSchema, database, namingResolver, sqlOperations, false); + destinationSpecificTableOperations(database); } return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (final ConnectionErrorException ex) { @@ -112,6 +118,15 @@ public AirbyteConnectionStatus check(final JsonNode config) { } } + /** + * Specific Databases may have additional checks unique to them which they need to perform, override + * this method to add additional checks. + * + * @param database the database to run checks against + * @throws Exception + */ + protected void destinationSpecificTableOperations(final JdbcDatabase database) throws Exception {} + /** * This method is deprecated. It verifies table creation, but not insert right to a newly created * table. Use attemptTableOperations with the attemptInsert argument instead. @@ -188,17 +203,30 @@ private static PartialAirbyteMessage getDummyRecord() { .withSerialized(dummyDataToInsert.toString()); } - protected DataSource getDataSource(final JsonNode config) { + /** + * Subclasses which need to modify the DataSource should override + * {@link #modifyDataSourceBuilder(DataSourceFactory.DataSourceBuilder)} rather than this method. + */ + @VisibleForTesting + public DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = toJdbcConfig(config); - return DataSourceFactory.create( + final Map connectionProperties = getConnectionProperties(config); + final DataSourceFactory.DataSourceBuilder builder = new DataSourceFactory.DataSourceBuilder( jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - getConnectionProperties(config)); + driverClassName, + jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()) + .withConnectionProperties(connectionProperties) + .withConnectionTimeout(getConnectionTimeout(connectionProperties)); + return modifyDataSourceBuilder(builder).build(); + } + + protected DataSourceFactory.DataSourceBuilder modifyDataSourceBuilder(final DataSourceFactory.DataSourceBuilder builder) { + return builder; } - protected JdbcDatabase getDatabase(final DataSource dataSource) { + @VisibleForTesting + public JdbcDatabase getDatabase(final DataSource dataSource) { return new DefaultJdbcDatabase(dataSource); } @@ -224,9 +252,7 @@ private void assertCustomParametersDontOverwriteDefaultParameters(final Map outputRecordCollector) throws Exception { - final DataSource dataSource = getDataSource(config); - final JdbcDatabase database = getDatabase(dataSource); + final JdbcDatabase database = getDatabase(getDataSource(config)); + final String defaultNamespace; + final TyperDeduper typerDeduper; if (TypingAndDedupingFlag.isDestinationV2()) { - // TODO: This logic exists in all V2 destinations. - // This is sad that if we forget to add this, there will be a null pointer during parseCatalog - final String defaultNamespace = config.get("schema").asText(); - for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - if (StringUtils.isEmpty(stream.getStream().getNamespace())) { - stream.getStream().setNamespace(defaultNamespace); - } - } - final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); - final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) - .map(override -> new CatalogParser(sqlGenerator, override)) - .orElse(new CatalogParser(sqlGenerator)) - .parseCatalog(catalog); - final String databaseName = getDatabaseName(config); - final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); - final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); - final TyperDeduper typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, 8); - return JdbcBufferedConsumerFactory.createAsync( - outputRecordCollector, - database, - sqlOperations, - namingResolver, - config, - catalog, - defaultNamespace, - typerDeduper); + defaultNamespace = config.get(getConfigSchemaKey()).asText(); + addDefaultNamespaceToStreams(catalog, defaultNamespace); + typerDeduper = getV2TyperDeduper(config, catalog, database); + } else { + defaultNamespace = null; + typerDeduper = new NoopTyperDeduper(); } return JdbcBufferedConsumerFactory.createAsync( outputRecordCollector, @@ -288,8 +295,37 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN namingResolver, config, catalog, - null, - new NoopTyperDeduper()); + defaultNamespace, + typerDeduper); + } + + /** + * Creates the appropriate TyperDeduper class for the jdbc destination and the user's configuration + * + * @param config the configuration for the connection + * @param catalog the catalog for the connection + * @param database a database instance + * @return the appropriate TyperDeduper instance for this connection. + */ + private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JdbcDatabase database) { + final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); + final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) + .map(override -> new CatalogParser(sqlGenerator, override)) + .orElse(new CatalogParser(sqlGenerator)) + .parseCatalog(catalog); + final String databaseName = getDatabaseName(config); + final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); + final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); + final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); + final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final TyperDeduper typerDeduper; + if (disableTypeDedupe) { + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator); + } else { + typerDeduper = + new DefaultTyperDeduper(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator); + } + return typerDeduper; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java index 68e715d2cbb2..fe41101366c2 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java @@ -4,8 +4,13 @@ package io.airbyte.cdk.integrations.destination.jdbc; -import java.sql.SQLType; - -public record ColumnDefinition(String name, String type, SQLType sqlType, int columnSize) { +/** + * Jdbc destination column definition representation + * + * @param name + * @param type + * @param columnSize + */ +public record ColumnDefinition(String name, String type, int columnSize, boolean isNullable) { } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java deleted file mode 100644 index dad853bb8e08..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc; - -import java.sql.SQLType; - -/** - * Custom SqlType definition when there is no mapping in {@link java.sql.JDBCType} - * - * @param name - * @param vendor - * @param vendorTypeNumber - */ -public record CustomSqlType(String name, String vendor, Integer vendorTypeNumber) implements SQLType { - - @Override - public String getName() { - return name; - } - - @Override - public String getVendor() { - return vendor; - } - - @Override - public Integer getVendorTypeNumber() { - return vendorTypeNumber; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java index b05876ac6282..d0d488c71284 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java @@ -214,9 +214,9 @@ private static RecordWriter recordWriterFunction(final Jd * Tear down functionality */ private static OnCloseFunction onCloseFunction(final TyperDeduper typerDeduper) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { try { - typerDeduper.typeAndDedupe(); + typerDeduper.typeAndDedupe(streamSyncSummaries); typerDeduper.commitFinalTables(); typerDeduper.cleanup(); } catch (final Exception e) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java index 99b82dd8f8c5..1ffd5f0c93ae 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java @@ -11,7 +11,6 @@ import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -72,6 +71,9 @@ protected Optional checkForKnownConfigExceptions(final Exc public void createTableIfNotExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { try { database.execute(createTableQuery(database, schemaName, tableName)); + for (final String postCreateSql : postCreateTableQueries(schemaName, tableName)) { + database.execute(postCreateSql); + } } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } @@ -86,37 +88,52 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN } } + /** + * Some subclasses may want to execute additional SQL statements after creating the raw table. For + * example, Postgres does not support index definitions within a CREATE TABLE statement, so we need + * to run CREATE INDEX statements after creating the table. + */ + protected List postCreateTableQueries(final String schemaName, final String tableName) { + return List.of(); + } + protected String createTableQueryV1(final String schemaName, final String tableName) { return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s VARCHAR PRIMARY KEY,\n" - + "%s JSONB,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP\n" - + ");\n", + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s JSONB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + """, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); } protected String createTableQueryV2(final String schemaName, final String tableName) { return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s VARCHAR PRIMARY KEY,\n" - + "%s JSONB,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT NULL\n" - + ");\n", + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s JSONB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + %s TIMESTAMP WITH TIME ZONE DEFAULT NULL + ); + """, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT); } // TODO: This method seems to be used by Postgres and others while staging to local temp files. // Should there be a Local staging operations equivalent - protected void writeBatchToFile(final File tmpFile, final List records) throws Exception { + protected void writeBatchToFile(final File tmpFile, final List records) throws Exception { try (final PrintWriter writer = new PrintWriter(tmpFile, StandardCharsets.UTF_8); final CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) { - for (final AirbyteRecordMessage record : records) { + for (final PartialAirbyteMessage record : records) { final var uuid = UUID.randomUUID().toString(); - final var jsonData = Jsons.serialize(formatData(record.getData())); - final var extractedAt = Timestamp.from(Instant.ofEpochMilli(record.getEmittedAt())); + // TODO we only need to do this is formatData is overridden. If not, we can just do jsonData = + // record.getSerialized() + final var jsonData = Jsons.serialize(formatData(Jsons.deserializeExact(record.getSerialized()))); + final var extractedAt = Timestamp.from(Instant.ofEpochMilli(record.getRecord().getEmittedAt())); if (TypingAndDedupingFlag.isDestinationV2()) { csvPrinter.printRecord(uuid, jsonData, extractedAt, null); } else { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java index 353d6d03cb44..c8fc4f2e7ca8 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java @@ -7,7 +7,7 @@ import java.util.LinkedHashMap; /** - * Jdbc destination table definition representation + * Jdbc destination table definition representation with a map of column names to column definitions * * @param columns */ diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java deleted file mode 100644 index f87d57218c48..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc; - -public record TableSchemaRecordSet() { - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java deleted file mode 100644 index 2ef35e795b24..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc; - -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.util.LinkedHashMap; - -/** - * A record representing the {@link java.sql.ResultSet} returned by calling - * {@link DatabaseMetaData#getTypeInfo()} - *

- * See that method for a better description of the parameters to this record - */ -public record TypeInfoRecordSet( - String typeName, - int dataType, - int precision, - String literalPrefix, - String literalSuffix, - String createParams, - short nullable, - boolean caseSensitive, - short searchable, - boolean unsignedAttribute, - boolean fixedPrecScale, - boolean autoIncrement, - String localTypeName, - short minimumScale, - short maximumScale, - - // Unused - int sqlDataType, - - // Unused - int sqlDatetimeSub, - int numPrecRadix) { - - public static LinkedHashMap getTypeInfoList(final DatabaseMetaData databaseMetaData) throws Exception { - final LinkedHashMap types = new LinkedHashMap<>(); - try (final ResultSet rs = databaseMetaData.getTypeInfo()) { - while (rs.next()) { - final var typeName = rs.getString("TYPE_NAME"); - types.put(typeName, - new TypeInfoRecordSet( - typeName, - rs.getInt("DATA_TYPE"), - rs.getInt("PRECISION"), - rs.getString("LITERAL_PREFIX"), - rs.getString("LITERAL_SUFFIX"), - rs.getString("CREATE_PARAMS"), - rs.getShort("NULLABLE"), - rs.getBoolean("CASE_SENSITIVE"), - rs.getShort("SEARCHABLE"), - rs.getBoolean("UNSIGNED_ATTRIBUTE"), - rs.getBoolean("FIXED_PREC_SCALE"), - rs.getBoolean("AUTO_INCREMENT"), - rs.getString("LOCAL_TYPE_NAME"), - rs.getShort("MINIMUM_SCALE"), - rs.getShort("MAXIMUM_SCALE"), - rs.getInt("SQL_DATA_TYPE"), - rs.getInt("SQL_DATETIME_SUB"), - rs.getInt("NUM_PREC_RADIX"))); - } - } - return types; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java index fde85f704ec0..35b8380f70bb 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java @@ -5,8 +5,7 @@ package io.airbyte.cdk.integrations.destination.jdbc; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; +import java.time.Instant; /** * Write configuration POJO (plain old java object) for all destinations extending @@ -20,7 +19,7 @@ public class WriteConfig { private final String tmpTableName; private final String outputTableName; private final DestinationSyncMode syncMode; - private final DateTime writeDatetime; + private final Instant writeDatetime; public WriteConfig(final String streamName, final String namespace, @@ -28,7 +27,7 @@ public WriteConfig(final String streamName, final String tmpTableName, final String outputTableName, final DestinationSyncMode syncMode) { - this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, DateTime.now(DateTimeZone.UTC)); + this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, Instant.now()); } public WriteConfig(final String streamName, @@ -37,7 +36,7 @@ public WriteConfig(final String streamName, final String tmpTableName, final String outputTableName, final DestinationSyncMode syncMode, - final DateTime writeDatetime) { + final Instant writeDatetime) { this.streamName = streamName; this.namespace = namespace; this.outputSchemaName = outputSchemaName; @@ -77,7 +76,7 @@ public DestinationSyncMode getSyncMode() { return syncMode; } - public DateTime getWriteDatetime() { + public Instant getWriteDatetime() { return writeDatetime; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java index 6a8224bda117..d553f3bf2696 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java @@ -93,7 +93,7 @@ private static OnStartFunction onStartFunction(final Map recordWriterFunction(final Map pairToCopier, final SqlOperations sqlOperations, final Map pairToIgnoredRecordCount) { - return (AirbyteStreamNameNamespacePair pair, List records) -> { + return (final AirbyteStreamNameNamespacePair pair, final List records) -> { final var fileName = pairToCopier.get(pair).prepareStagingFile(); for (final AirbyteRecordMessage recordMessage : records) { final var id = UUID.randomUUID(); @@ -109,7 +109,7 @@ private static RecordWriter recordWriterFunction(final Map } private static CheckAndRemoveRecordWriter removeStagingFilePrinter(final Map pairToCopier) { - return (AirbyteStreamNameNamespacePair pair, String stagingFileName) -> { + return (final AirbyteStreamNameNamespacePair pair, final String stagingFileName) -> { final String currentFileName = pairToCopier.get(pair).getCurrentFile(); if (stagingFileName != null && currentFileName != null && !stagingFileName.equals(currentFileName)) { pairToCopier.get(pair).closeNonCurrentStagingFileWriters(); @@ -123,7 +123,7 @@ private static OnCloseFunction onCloseFunction(final Map pairToIgnoredRecordCount, final DataSource dataSource) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { pairToIgnoredRecordCount .forEach((pair, count) -> LOGGER.warn("A total of {} record(s) of data from stream {} were invalid and were ignored.", count, pair)); closeAsOneTransaction(pairToCopier, hasFailed, database, sqlOperations, dataSource); diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java index 2cb302b8eaa8..1aa0b687f8c5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java @@ -4,27 +4,54 @@ package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS; +import static org.jooq.impl.DSL.exists; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.DSL.selectOne; + import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition; -import io.airbyte.cdk.integrations.destination.jdbc.CustomSqlType; import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; +import io.airbyte.commons.concurrency.CompletableFutures; +import io.airbyte.commons.exceptions.SQLRuntimeException; +import io.airbyte.commons.functional.Either; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialStateImpl; +import io.airbyte.integrations.base.destination.typing_deduping.InitialRawTableState; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; -import java.sql.DatabaseMetaData; -import java.sql.JDBCType; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.SQLType; +import java.sql.Timestamp; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Slf4j -public class JdbcDestinationHandler implements DestinationHandler { +public abstract class JdbcDestinationHandler implements DestinationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcDestinationHandler.class); @@ -37,49 +64,121 @@ public JdbcDestinationHandler(final String databaseName, this.jdbcDatabase = jdbcDatabase; } - @Override - public Optional findExistingTable(final StreamId id) throws Exception { + private Optional findExistingTable(final StreamId id) throws Exception { return findExistingTable(jdbcDatabase, databaseName, id.finalNamespace(), id.finalName()); } - @Override - public boolean isFinalTableEmpty(final StreamId id) throws Exception { - final int rowCount = jdbcDatabase.queryInt( - """ - SELECT row_count - FROM information_schema.tables - WHERE table_catalog = ? - AND table_schema = ? - AND table_name = ? - """, - databaseName, - id.finalNamespace(), - id.finalName()); - return rowCount == 0; + private boolean isFinalTableEmpty(final StreamId id) throws Exception { + return !jdbcDatabase.queryBoolean( + select( + field(exists( + selectOne() + .from(name(id.finalNamespace(), id.finalName())) + .limit(1)))) + .getSQL(ParamType.INLINED)); } - @Override - public Optional getMinTimestampForSync(final StreamId id) throws Exception { - return Optional.empty(); + private InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { + boolean tableExists = jdbcDatabase.executeMetadataQuery(dbmetadata -> { + LOGGER.info("Retrieving table from Db metadata: {} {} {}", databaseName, id.rawNamespace(), id.rawName()); + try (final ResultSet table = dbmetadata.getTables(databaseName, id.rawNamespace(), id.rawName(), null)) { + return table.next(); + } catch (SQLException e) { + LOGGER.error("Failed to retrieve table info from metadata", e); + throw new SQLRuntimeException(e); + } + }); + if (!tableExists) { + // There's no raw table at all. Therefore there are no unprocessed raw records, and this sync + // should not filter raw records by timestamp. + return new InitialRawTableState(false, Optional.empty()); + } + // And use two explicit queries because COALESCE might not short-circuit evaluation. + // This first query tries to find the oldest raw record with loaded_at = NULL. + // Unsafe query requires us to explicitly close the Stream, which is inconvenient, + // but it's also the only method in the JdbcDatabase interface to return non-string/int types + try (final Stream timestampStream = jdbcDatabase.unsafeQuery( + conn -> conn.prepareStatement( + select(field("MIN(_airbyte_extracted_at)").as("min_timestamp")) + .from(name(id.rawNamespace(), id.rawName())) + .where(DSL.condition("_airbyte_loaded_at IS NULL")) + .getSQL()), + record -> record.getTimestamp("min_timestamp"))) { + // Filter for nonNull values in case the query returned NULL (i.e. no unloaded records). + final Optional minUnloadedTimestamp = timestampStream.filter(Objects::nonNull).findFirst(); + if (minUnloadedTimestamp.isPresent()) { + // Decrement by 1 second since timestamp precision varies between databases. + final Optional ts = minUnloadedTimestamp + .map(Timestamp::toInstant) + .map(i -> i.minus(1, ChronoUnit.SECONDS)); + return new InitialRawTableState(true, ts); + } + } + // If there are no unloaded raw records, then we can safely skip all existing raw records. + // This second query just finds the newest raw record. + try (final Stream timestampStream = jdbcDatabase.unsafeQuery( + conn -> conn.prepareStatement( + select(field("MAX(_airbyte_extracted_at)").as("min_timestamp")) + .from(name(id.rawNamespace(), id.rawName())) + .getSQL()), + record -> record.getTimestamp("min_timestamp"))) { + // Filter for nonNull values in case the query returned NULL (i.e. no raw records at all). + final Optional minUnloadedTimestamp = timestampStream.filter(Objects::nonNull).findFirst(); + return new InitialRawTableState(false, minUnloadedTimestamp.map(Timestamp::toInstant)); + } } @Override - public void execute(final String sql) throws Exception { - if (sql == null || sql.isEmpty()) { - return; - } + public void execute(final Sql sql) throws Exception { + final List> transactions = sql.transactions(); final UUID queryId = UUID.randomUUID(); - LOGGER.info("Executing sql {}: {}", queryId, sql); - final long startTime = System.currentTimeMillis(); - - try { - jdbcDatabase.execute(sql); - } catch (final SQLException e) { - LOGGER.error("Sql {} failed", queryId, e); - throw e; + for (final List transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + LOGGER.info("Executing sql {}-{}: {}", queryId, transactionId, String.join("\n", transaction)); + final long startTime = System.currentTimeMillis(); + + try { + jdbcDatabase.executeWithinTransaction(transaction); + } catch (final SQLException e) { + LOGGER.error("Sql {}-{} failed", queryId, transactionId, e); + throw e; + } + + LOGGER.info("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); } + } - LOGGER.info("Sql {} completed in {} ms", queryId, System.currentTimeMillis() - startTime); + @Override + public List gatherInitialState(List streamConfigs) throws Exception { + final List> initialStates = streamConfigs.stream() + .map(this::retrieveState) + .toList(); + final List> states = CompletableFutures.allOf(initialStates).toCompletableFuture().join(); + return ConnectorExceptionUtil.getResultsOrLogAndThrowFirst("Failed to retrieve initial state", states); + } + + private CompletionStage retrieveState(final StreamConfig streamConfig) { + return CompletableFuture.supplyAsync(() -> { + try { + final Optional finalTableDefinition = findExistingTable(streamConfig.id()); + final boolean isSchemaMismatch; + final boolean isFinalTableEmpty; + if (finalTableDefinition.isPresent()) { + isSchemaMismatch = !existingSchemaMatchesStreamConfig(streamConfig, finalTableDefinition.get()); + isFinalTableEmpty = isFinalTableEmpty(streamConfig.id()); + } else { + // If the final table doesn't exist, then by definition it doesn't have a schema mismatch and has no + // records. + isSchemaMismatch = false; + isFinalTableEmpty = true; + } + final InitialRawTableState initialRawTableState = getInitialRawTableState(streamConfig.id()); + return new DestinationInitialStateImpl(streamConfig, finalTableDefinition.isPresent(), initialRawTableState, + isSchemaMismatch, isFinalTableEmpty); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public static Optional findExistingTable(final JdbcDatabase jdbcDatabase, @@ -87,33 +186,83 @@ public static Optional findExistingTable(final JdbcDatabase jdb final String schemaName, final String tableName) throws SQLException { - final DatabaseMetaData metaData = jdbcDatabase.getMetaData(); - // TODO: normalize namespace and finalName strings to quoted-lowercase (as needed. Snowflake - // requires uppercase) - final LinkedHashMap columnDefinitions = new LinkedHashMap<>(); - try (final ResultSet columns = metaData.getColumns(databaseName, schemaName, tableName, null)) { - while (columns.next()) { - final String columnName = columns.getString("COLUMN_NAME"); - final String typeName = columns.getString("TYPE_NAME"); - final int columnSize = columns.getInt("COLUMN_SIZE"); - final int datatype = columns.getInt("DATA_TYPE"); - SQLType sqlType; - try { - sqlType = JDBCType.valueOf(datatype); - } catch (final IllegalArgumentException e) { - // Unknown jdbcType convert to customSqlType - LOGGER.warn("Unrecognized JDBCType {}; falling back to UNKNOWN", datatype, e); - sqlType = new CustomSqlType("Unknown", "Unknown", datatype); + final LinkedHashMap retrievedColumnDefns = jdbcDatabase.executeMetadataQuery(dbMetadata -> { + + // TODO: normalize namespace and finalName strings to quoted-lowercase (as needed. Snowflake + // requires uppercase) + final LinkedHashMap columnDefinitions = new LinkedHashMap<>(); + LOGGER.info("Retrieving existing columns for {}.{}.{}", databaseName, schemaName, tableName); + try (final ResultSet columns = dbMetadata.getColumns(databaseName, schemaName, tableName, null)) { + while (columns.next()) { + final String columnName = columns.getString("COLUMN_NAME"); + final String typeName = columns.getString("TYPE_NAME"); + final int columnSize = columns.getInt("COLUMN_SIZE"); + final String isNullable = columns.getString("IS_NULLABLE"); + columnDefinitions.put(columnName, new ColumnDefinition(columnName, typeName, columnSize, fromIsNullableIsoString(isNullable))); } - columnDefinitions.put(columnName, new ColumnDefinition(columnName, typeName, sqlType, columnSize)); + } catch (final SQLException e) { + LOGGER.error("Failed to retrieve column info for {}.{}.{}", databaseName, schemaName, tableName, e); + throw new SQLRuntimeException(e); } - } + return columnDefinitions; + }); // Guard to fail fast - if (columnDefinitions.isEmpty()) { + if (retrievedColumnDefns.isEmpty()) { return Optional.empty(); } - return Optional.of(new TableDefinition(columnDefinitions)); + return Optional.of(new TableDefinition(retrievedColumnDefns)); + } + + public static boolean fromIsNullableIsoString(final String isNullable) { + return "YES".equalsIgnoreCase(isNullable); + } + + private boolean isAirbyteRawIdColumnMatch(final TableDefinition existingTable) { + return existingTable.columns().containsKey(COLUMN_NAME_AB_RAW_ID) && + toJdbcTypeName(AirbyteProtocolType.STRING).equals(existingTable.columns().get(COLUMN_NAME_AB_RAW_ID).type()); + } + + private boolean isAirbyteExtractedAtColumnMatch(final TableDefinition existingTable) { + return existingTable.columns().containsKey(COLUMN_NAME_AB_EXTRACTED_AT) && + toJdbcTypeName(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE).equals(existingTable.columns().get(COLUMN_NAME_AB_EXTRACTED_AT).type()); } + private boolean isAirbyteMetaColumnMatch(final TableDefinition existingTable) { + return existingTable.columns().containsKey(COLUMN_NAME_AB_META) && + toJdbcTypeName(new Struct(new LinkedHashMap<>())).equals(existingTable.columns().get(COLUMN_NAME_AB_META).type()); + } + + protected boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final TableDefinition existingTable) { + // Check that the columns match, with special handling for the metadata columns. + if (!isAirbyteRawIdColumnMatch(existingTable) || + !isAirbyteExtractedAtColumnMatch(existingTable) || + !isAirbyteMetaColumnMatch(existingTable)) { + // Missing AB meta columns from final table, we need them to do proper T+D so trigger soft-reset + return false; + } + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey().name(), toJdbcTypeName(column.getValue())), + LinkedHashMap::putAll); + + // Filter out Meta columns since they don't exist in stream config. + final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() + .filter(column -> V2_FINAL_TABLE_METADATA_COLUMNS.stream() + .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey(), column.getValue().type()), + LinkedHashMap::putAll); + + return actualColumns.equals(intendedColumns); + } + + /** + * Convert to the TYPE_NAME retrieved from {@link java.sql.DatabaseMetaData#getColumns} + * + * @param airbyteType + * @return + */ + protected abstract String toJdbcTypeName(final AirbyteType airbyteType); + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java index 0be3da56988c..d32a84de478e 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java @@ -4,27 +4,79 @@ package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; +import static java.util.stream.Collectors.toList; +import static org.jooq.impl.DSL.alterTable; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.cast; +import static org.jooq.impl.DSL.dropTableIfExists; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.noCondition; +import static org.jooq.impl.DSL.quotedName; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.DSL.table; +import static org.jooq.impl.DSL.update; +import static org.jooq.impl.DSL.with; + +import com.google.common.annotations.VisibleForTesting; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.base.destination.typing_deduping.Struct; import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import org.jooq.CommonTableExpression; +import org.jooq.Condition; +import org.jooq.CreateSchemaFinalStep; +import org.jooq.CreateTableColumnStep; +import org.jooq.DSLContext; import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.Record; import org.jooq.SQLDialect; +import org.jooq.SelectConditionStep; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; -public abstract class JdbcSqlGenerator implements SqlGenerator { +public abstract class JdbcSqlGenerator implements SqlGenerator { + + protected static final String ROW_NUMBER_COLUMN_NAME = "row_number"; + private static final String TYPING_CTE_ALIAS = "intermediate_data"; + private static final String NUMBERED_ROWS_CTE_ALIAS = "numbered_rows"; - private final NamingConventionTransformer namingTransformer; + protected final NamingConventionTransformer namingTransformer; + protected final ColumnId cdcDeletedAtColumn; public JdbcSqlGenerator(final NamingConventionTransformer namingTransformer) { this.namingTransformer = namingTransformer; + this.cdcDeletedAtColumn = buildColumnId("_ab_cdc_deleted_at"); } @Override @@ -60,7 +112,8 @@ protected DataType toDialectType(final AirbyteType type) { }; } - protected DataType toDialectType(final AirbyteProtocolType airbyteProtocolType) { + @VisibleForTesting + public DataType toDialectType(final AirbyteProtocolType airbyteProtocolType) { return switch (airbyteProtocolType) { // Many destinations default to a very short length (e.g. Redshift defaults to 256). // Explicitly set 64KiB here. Subclasses may want to override this value. @@ -83,8 +136,379 @@ protected DataType toDialectType(final AirbyteProtocolType airbyteProtocolTyp protected abstract DataType getArrayType(); + @VisibleForTesting + public DataType getTimestampWithTimeZoneType() { + return toDialectType(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + } + protected abstract DataType getWidestType(); protected abstract SQLDialect getDialect(); + /** + * @param columns from the schema to be extracted from _airbyte_data column. Use the destination + * specific syntax to extract data + * @param useExpensiveSaferCasting + * @return a list of jooq fields for the final table insert statement. + */ + protected abstract List> extractRawDataFields(final LinkedHashMap columns, boolean useExpensiveSaferCasting); + + /** + * + * @param columns from the schema to be used for type casting errors and construct _airbyte_meta + * column + * @return + */ + protected abstract Field buildAirbyteMetaColumn(final LinkedHashMap columns); + + /** + * Get the cdc_deleted_at column condition for append_dedup mode by extracting it from _airbyte_data + * column in raw table. + * + * @return + */ + protected abstract Condition cdcDeletedAtNotNullCondition(); + + /** + * Get the window step function row_number() over (partition by primary_key order by cursor_field) + * as row_number. + * + * @param primaryKey list of primary keys + * @param cursorField cursor field used for ordering + * @return + */ + protected abstract Field getRowNumber(final List primaryKey, final Optional cursorField); + + protected DSLContext getDslContext() { + return DSL.using(getDialect()); + } + + /** + * build jooq fields for final table with customers columns first and then meta columns. + * + * @param columns + * @param metaColumns + * @return + */ + @VisibleForTesting + List> buildFinalTableFields(final LinkedHashMap columns, final Map> metaColumns) { + final List> fields = + metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(toList()); + final List> dataFields = + columns.entrySet().stream().map(column -> field(quotedName(column.getKey().name()), toDialectType(column.getValue()))).collect( + toList()); + dataFields.addAll(fields); + return dataFields; + } + + /** + * Use this method to get the final table meta columns with or without _airbyte_meta column. + * + * @param includeMetaColumn + * @return + */ + LinkedHashMap> getFinalTableMetaColumns(final boolean includeMetaColumn) { + final LinkedHashMap> metaColumns = new LinkedHashMap<>(); + metaColumns.put(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)); + metaColumns.put(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)); + if (includeMetaColumn) + metaColumns.put(COLUMN_NAME_AB_META, getStructType().nullable(false)); + return metaColumns; + } + + /** + * build jooq fields for raw table with type-casted data columns first and then meta columns without + * _airbyte_meta. + * + * @param columns + * @param metaColumns + * @return + */ + @VisibleForTesting + List> buildRawTableSelectFields(final LinkedHashMap columns, + final Map> metaColumns, + final boolean useExpensiveSaferCasting) { + final List> fields = + metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(toList()); + // Use originalName with non-sanitized characters when extracting data from _airbyte_data + final List> dataFields = extractRawDataFields(columns, useExpensiveSaferCasting); + dataFields.addAll(fields); + return dataFields; + } + + @VisibleForTesting + Condition rawTableCondition(final DestinationSyncMode syncMode, final boolean isCdcDeletedAtPresent, final Optional minRawTimestamp) { + Condition condition = field(name(COLUMN_NAME_AB_LOADED_AT)).isNull(); + if (syncMode == DestinationSyncMode.APPEND_DEDUP) { + if (isCdcDeletedAtPresent) { + condition = condition.or(cdcDeletedAtNotNullCondition()); + } + } + if (minRawTimestamp.isPresent()) { + condition = condition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); + } + return condition; + } + + @Override + public Sql createSchema(final String schema) { + return Sql.of(createSchemaSql(schema)); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + // TODO: Use Naming transformer to sanitize these strings with redshift restrictions. + final String finalTableIdentifier = stream.id().finalName() + suffix.toLowerCase(); + if (!force) { + return transactionally(Stream.concat( + Stream.of(createTableSql(stream.id().finalNamespace(), finalTableIdentifier, stream.columns())), + createIndexSql(stream, suffix).stream()).toList()); + } + return transactionally(Stream.concat( + Stream.of( + dropTableIfExists(quotedName(stream.id().finalNamespace(), finalTableIdentifier)).getSQL(ParamType.INLINED), + createTableSql(stream.id().finalNamespace(), finalTableIdentifier, stream.columns())), + createIndexSql(stream, suffix).stream()).toList()); + } + + @Override + public Sql updateTable(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + + // TODO: Add flag to use merge vs insert/delete + return insertAndDeleteTransaction(streamConfig, finalSuffix, minRawTimestamp, useExpensiveSaferCasting); + + } + + @Override + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return transactionally( + dropTableIfExists(name(stream.finalNamespace(), stream.finalName())).getSQL(ParamType.INLINED), + alterTable(name(stream.finalNamespace(), stream.finalName() + finalSuffix)) + .renameTo(name(stream.finalName())) + .getSQL()); + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + final Name rawTableName = name(streamId.rawNamespace(), streamId.rawName()); + final DSLContext dsl = getDslContext(); + return transactionally( + dsl.createSchemaIfNotExists(streamId.rawNamespace()).getSQL(), + dsl.dropTableIfExists(rawTableName).getSQL(), + DSL.createTable(rawTableName) + .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_AB_LOADED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .as(select( + field(COLUMN_NAME_AB_ID).as(COLUMN_NAME_AB_RAW_ID), + field(COLUMN_NAME_EMITTED_AT).as(COLUMN_NAME_AB_EXTRACTED_AT), + cast(null, getTimestampWithTimeZoneType()).as(COLUMN_NAME_AB_LOADED_AT), + field(COLUMN_NAME_DATA).as(COLUMN_NAME_DATA)).from(table(name(namespace, tableName)))) + .getSQL(ParamType.INLINED)); + } + + @Override + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(update(table(name(streamId.rawNamespace(), streamId.rawName()))) + .set(field(COLUMN_NAME_AB_LOADED_AT), inline((String) null)) + .getSQL()); + } + + @VisibleForTesting + SelectConditionStep selectFromRawTable(final String schemaName, + final String tableName, + final LinkedHashMap columns, + final Map> metaColumns, + final Condition condition, + final boolean useExpensiveSaferCasting) { + final DSLContext dsl = getDslContext(); + return dsl + .select(buildRawTableSelectFields(columns, metaColumns, useExpensiveSaferCasting)) + .select(buildAirbyteMetaColumn(columns)) + .from(table(quotedName(schemaName, tableName))) + .where(condition); + } + + @VisibleForTesting + InsertValuesStepN insertIntoFinalTable(final String schemaName, + final String tableName, + final LinkedHashMap columns, + final Map> metaFields) { + final DSLContext dsl = getDslContext(); + return dsl + .insertInto(table(quotedName(schemaName, tableName))) + .columns(buildFinalTableFields(columns, metaFields)); + } + + private Sql insertAndDeleteTransaction(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + final String finalSchema = streamConfig.id().finalNamespace(); + final String finalTable = streamConfig.id().finalName() + (finalSuffix != null ? finalSuffix.toLowerCase() : ""); + final String rawSchema = streamConfig.id().rawNamespace(); + final String rawTable = streamConfig.id().rawName(); + + // Poor person's guarantee of ordering of fields by using same source of ordered list of columns to + // generate fields. + final CommonTableExpression rawTableRowsWithCast = name(TYPING_CTE_ALIAS).as( + selectFromRawTable(rawSchema, rawTable, streamConfig.columns(), + getFinalTableMetaColumns(false), + rawTableCondition(streamConfig.destinationSyncMode(), + streamConfig.columns().containsKey(cdcDeletedAtColumn), + minRawTimestamp), + useExpensiveSaferCasting)); + final List> finalTableFields = buildFinalTableFields(streamConfig.columns(), getFinalTableMetaColumns(true)); + final Field rowNumber = getRowNumber(streamConfig.primaryKey(), streamConfig.cursor()); + final CommonTableExpression filteredRows = name(NUMBERED_ROWS_CTE_ALIAS).as( + select(asterisk(), rowNumber).from(rawTableRowsWithCast)); + + // Used for append-dedupe mode. + final String insertStmtWithDedupe = + insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) + .select(with(rawTableRowsWithCast) + .with(filteredRows) + .select(finalTableFields) + .from(filteredRows) + .where(field(name(ROW_NUMBER_COLUMN_NAME), Integer.class).eq(1)) // Can refer by CTE.field but no use since we don't strongly type + // them. + ) + .getSQL(ParamType.INLINED); + + // Used for append and overwrite modes. + final String insertStmt = + insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) + .select(with(rawTableRowsWithCast) + .select(finalTableFields) + .from(rawTableRowsWithCast)) + .getSQL(ParamType.INLINED); + final String deleteStmt = deleteFromFinalTable(finalSchema, finalTable, streamConfig.primaryKey(), streamConfig.cursor()); + final String deleteCdcDeletesStmt = + streamConfig.columns().containsKey(cdcDeletedAtColumn) ? deleteFromFinalTableCdcDeletes(finalSchema, finalTable) : ""; + final String checkpointStmt = checkpointRawTable(rawSchema, rawTable, minRawTimestamp); + + if (streamConfig.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { + return transactionally( + insertStmt, + checkpointStmt); + } + + // For append-dedupe + return transactionally( + insertStmtWithDedupe, + deleteStmt, + deleteCdcDeletesStmt, + checkpointStmt); + } + + private String mergeTransaction(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + + throw new UnsupportedOperationException("Not implemented yet"); + + } + + protected String createSchemaSql(final String namespace) { + final DSLContext dsl = getDslContext(); + final CreateSchemaFinalStep createSchemaSql = dsl.createSchemaIfNotExists(quotedName(namespace)); + return createSchemaSql.getSQL(); + } + + protected String createTableSql(final String namespace, final String tableName, final LinkedHashMap columns) { + final DSLContext dsl = getDslContext(); + final CreateTableColumnStep createTableSql = dsl + .createTable(quotedName(namespace, tableName)) + .columns(buildFinalTableFields(columns, getFinalTableMetaColumns(true))); + return createTableSql.getSQL(); + } + + /** + * Subclasses may override this method to add additional indexes after their CREATE TABLE statement. + * This is useful if the destination's CREATE TABLE statement does not accept an index definition. + */ + protected List createIndexSql(final StreamConfig stream, final String suffix) { + return Collections.emptyList(); + } + + protected String beginTransaction() { + return "BEGIN"; + } + + protected String commitTransaction() { + return "COMMIT"; + } + + private String commitTransactionInternal() { + return commitTransaction() + ";"; + } + + private String deleteFromFinalTable(final String schemaName, + final String tableName, + final List primaryKeys, + final Optional cursor) { + final DSLContext dsl = getDslContext(); + // Unknown type doesn't play well with where .. in (select..) + final Field airbyteRawId = field(quotedName(COLUMN_NAME_AB_RAW_ID)); + final Field rowNumber = getRowNumber(primaryKeys, cursor); + return dsl.deleteFrom(table(quotedName(schemaName, tableName))) + .where(airbyteRawId.in( + select(airbyteRawId) + .from(select(airbyteRawId, rowNumber) + .from(table(quotedName(schemaName, tableName))).asTable("airbyte_ids")) + .where(field(name(ROW_NUMBER_COLUMN_NAME)).ne(1)))) + .getSQL(ParamType.INLINED); + } + + private String deleteFromFinalTableCdcDeletes(final String schema, final String tableName) { + final DSLContext dsl = getDslContext(); + return dsl.deleteFrom(table(quotedName(schema, tableName))) + .where(field(quotedName(cdcDeletedAtColumn.name())).isNotNull()) + .getSQL(ParamType.INLINED); + } + + private String checkpointRawTable(final String schemaName, final String tableName, final Optional minRawTimestamp) { + final DSLContext dsl = getDslContext(); + Condition extractedAtCondition = noCondition(); + if (minRawTimestamp.isPresent()) { + extractedAtCondition = extractedAtCondition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); + } + return dsl.update(table(quotedName(schemaName, tableName))) + .set(field(quotedName(COLUMN_NAME_AB_LOADED_AT)), currentTimestamp()) + .where(field(quotedName(COLUMN_NAME_AB_LOADED_AT)).isNull()).and(extractedAtCondition) + .getSQL(ParamType.INLINED); + } + + protected Field castedField( + final Field field, + final AirbyteType type, + final String alias, + final boolean useExpensiveSaferCasting) { + if (type instanceof final AirbyteProtocolType airbyteProtocolType) { + return castedField(field, airbyteProtocolType, useExpensiveSaferCasting).as(quotedName(alias)); + } + + // Redshift SUPER can silently cast an array type to struct and vice versa. + return switch (type.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE -> cast(field, getStructType()).as(quotedName(alias)); + case Array.TYPE -> cast(field, getArrayType()).as(quotedName(alias)); + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias, useExpensiveSaferCasting); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + }; + } + + protected Field castedField(final Field field, final AirbyteProtocolType type, final boolean useExpensiveSaferCasting) { + return cast(field, toDialectType(type)); + } + + protected Field currentTimestamp() { + return DSL.currentTimestamp(); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java index d5374140fc3e..b398f202fc06 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java @@ -7,10 +7,12 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.commons.exceptions.SQLRuntimeException; import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Collection; import java.util.Optional; import lombok.SneakyThrows; @@ -33,15 +35,21 @@ public JdbcV1V2Migrator(final NamingConventionTransformer namingConventionTransf @SneakyThrows @Override - protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) throws Exception { - String retrievedSchema = ""; - try (ResultSet columns = database.getMetaData().getSchemas(databaseName, streamConfig.id().rawNamespace())) { - while (columns.next()) { - retrievedSchema = columns.getString("TABLE_SCHEM"); - // Catalog can be null, so don't do anything with it. - String catalog = columns.getString("TABLE_CATALOG"); + protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) { + final String retrievedSchema = database.executeMetadataQuery(dbMetadata -> { + try (ResultSet columns = dbMetadata.getSchemas(databaseName, streamConfig.id().rawNamespace())) { + String schema = ""; + while (columns.next()) { + // Catalog can be null, so don't do anything with it. + // columns.getString("TABLE_CATALOG"); + schema = columns.getString("TABLE_SCHEM"); + } + return schema; + } catch (SQLException e) { + throw new SQLRuntimeException(e); } - } + }); + return !retrievedSchema.isEmpty(); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt new file mode 100644 index 000000000000..5c9963f5bad8 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -0,0 +1,59 @@ +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping + +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig +import org.jooq.Condition +import org.jooq.DataType +import org.jooq.Field +import org.jooq.SQLDialect +import java.util.* + +/** + * Some Destinations do not support Typing and Deduping but have the updated raw table format + * SqlGenerator implementations are only for "final" tables and are a required input for + * TyperDeduper classes. This implementation appeases that requirement but does not implement + * any "final" table operations. + */ +class RawOnlySqlGenerator(private val namingTransformer: NamingConventionTransformer) : + JdbcSqlGenerator(namingTransformer) { + override fun getStructType(): DataType<*>? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun getArrayType(): DataType<*>? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun getWidestType(): DataType<*>? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun getDialect(): SQLDialect? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun extractRawDataFields( + columns: LinkedHashMap, + useExpensiveSaferCasting: Boolean + ): List>? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun buildAirbyteMetaColumn(columns: LinkedHashMap): Field<*>? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun cdcDeletedAtNotNullCondition(): Condition? { + throw NotImplementedError("This Destination does not support final tables") + } + + override fun getRowNumber( + primaryKey: List, + cursorField: Optional + ): Field? { + throw NotImplementedError("This Destination does not support final tables") + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java index 66eb71bd240b..0eef0c5343bf 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java @@ -13,6 +13,7 @@ import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.locks.Lock; import lombok.extern.slf4j.Slf4j; @@ -22,6 +23,16 @@ @Slf4j public class GeneralStagingFunctions { + // using a random string here as a placeholder for the moment. + // This would avoid mixing data in the staging area between different syncs (especially if they + // manipulate streams with similar names) + // if we replaced the random connection id by the actual connection_id, we'd gain the opportunity to + // leverage data that was uploaded to stage + // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. + // This would also allow other programs/scripts + // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. + public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); + public static OnStartFunction onStartFunction(final JdbcDatabase database, final StagingOperations stagingOperations, final List writeConfigs, @@ -96,8 +107,6 @@ public static void copyIntoTableFromStage(final JdbcDatabase database, typerDeduperValve.updateTimeAndIncreaseInterval(streamId); } } catch (final Exception e) { - stagingOperations.cleanUpStage(database, stageName, stagedFiles); - log.info("Cleaning stage path {}", stagingPath); throw new RuntimeException("Failed to upload data from stage " + stagingPath, e); } } @@ -116,18 +125,26 @@ public static OnCloseFunction onCloseFunction(final JdbcDatabase database, final List writeConfigs, final boolean purgeStagingData, final TyperDeduper typerDeduper) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { // After moving data from staging area to the target table (airybte_raw) clean up the staging // area (if user configured) log.info("Cleaning up destination started for {} streams", writeConfigs.size()); - typerDeduper.typeAndDedupe(); + typerDeduper.typeAndDedupe(streamSyncSummaries); for (final WriteConfig writeConfig : writeConfigs) { final String schemaName = writeConfig.getOutputSchemaName(); if (purgeStagingData) { final String stageName = stagingOperations.getStageName(schemaName, writeConfig.getOutputTableName()); + final String stagePath = stagingOperations.getStagingPath( + RANDOM_CONNECTION_ID, + schemaName, + writeConfig.getStreamName(), + writeConfig.getOutputTableName(), + writeConfig.getWriteDatetime()); log.info("Cleaning stage in destination started for stream {}. schema {}, stage: {}", writeConfig.getStreamName(), schemaName, - stageName); - stagingOperations.dropStageIfExists(database, stageName); + stagePath); + // TODO: This is another weird manifestation of Redshift vs Snowflake using either or variables from + // stageName/StagingPath. + stagingOperations.dropStageIfExists(database, stageName, stagePath); } } typerDeduper.commitFinalTables(); diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java index dc37391f8b06..9f2b4c4c8f34 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java @@ -24,12 +24,11 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.time.Instant; import java.util.List; import java.util.UUID; import java.util.function.Consumer; import java.util.function.Function; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +48,7 @@ public class SerialStagingConsumerFactory { // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. // This would also allow other programs/scripts // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. - private static final DateTime SYNC_DATETIME = DateTime.now(DateTimeZone.UTC); + private static final Instant SYNC_DATETIME = Instant.now(); public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); public AirbyteMessageConsumer create(final Consumer outputRecordCollector, diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java index f532b8ba8766..a5a07903ad48 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -9,8 +9,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; @@ -139,6 +141,11 @@ protected JdbcSqlGenerator getSqlGenerator() { return null; } + @Override + protected JdbcDestinationHandler getDestinationHandler(String databaseName, JdbcDatabase database) { + return null; + } + } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index e3204dd54899..8094c8fc214c 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -35,9 +35,7 @@ import io.airbyte.configoss.StandardCheckConnectionInput; import io.airbyte.configoss.StandardCheckConnectionOutput; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.configoss.StandardDestinationDefinition; import io.airbyte.configoss.WorkerDestinationConfig; -import io.airbyte.configoss.init.LocalDefinitionsProvider; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -65,6 +63,8 @@ import io.airbyte.workers.process.DockerProcessFactory; import io.airbyte.workers.process.ProcessFactory; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -74,7 +74,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -83,7 +82,6 @@ import java.util.stream.Stream; import lombok.Builder; import lombok.Getter; -import org.joda.time.DateTime; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -99,7 +97,7 @@ public abstract class DestinationAcceptanceTest { - protected static final HashSet TEST_SCHEMAS = new HashSet<>(); + protected HashSet TEST_SCHEMAS; private static final Random RANDOM = new Random(); private static final String NORMALIZATION_VERSION = "dev"; @@ -142,20 +140,31 @@ private String getImageNameWithoutTag() { return getImageName().contains(":") ? getImageName().split(":")[0] : getImageName(); } - protected static Optional getOptionalDestinationDefinitionFromProvider( - final String imageNameWithoutTag) { - final LocalDefinitionsProvider provider = new LocalDefinitionsProvider(); - return provider.getDestinationDefinitions().stream() - .filter(definition -> imageNameWithoutTag.equalsIgnoreCase(definition.getDockerRepository())) - .findFirst(); + private JsonNode readMetadata() { + try { + return Jsons.jsonNodeFromFile(MoreResources.readResourceAsFile("metadata.yaml")); + } catch (IllegalArgumentException | URISyntaxException e) { + // Resource is not found. + return Jsons.emptyObject(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } protected String getNormalizationImageName() { - return getOptionalDestinationDefinitionFromProvider(getDestinationDefinitionKey()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository() + ":" - + NORMALIZATION_VERSION) - .orElse(null); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return null; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return null; + } + var normalizationRepository = normalizationConfig.get("normalizationRepository"); + if (normalizationRepository == null) { + return null; + } + return normalizationRepository.asText() + ":" + NORMALIZATION_VERSION; } /** @@ -240,18 +249,24 @@ protected boolean implementsAppend() throws TestHarnessException { } protected boolean normalizationFromDefinition() { - return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository()) - && Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationTag())) - .orElse(false); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return false; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return false; + } + return normalizationConfig.has("normalizationRepository") && normalizationConfig.has("normalizationTag"); } protected boolean dbtFromDefinition() { - return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag()) - .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getSupportsDbt()) - && standardDestinationDefinition.getSupportsDbt()) - .orElse(false); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return false; + } + var supportsDbt = metadata.get("supportsDbt"); + return supportsDbt != null && supportsDbt.asBoolean(false); } protected String getDestinationDefinitionKey() { @@ -259,10 +274,19 @@ protected String getDestinationDefinitionKey() { } protected String getNormalizationIntegrationType() { - return getOptionalDestinationDefinitionFromProvider(getDestinationDefinitionKey()) - .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig())) - .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationIntegrationType()) - .orElse(null); + var metadata = readMetadata().get("data"); + if (metadata == null) { + return null; + } + var normalizationConfig = metadata.get("normalizationConfig"); + if (normalizationConfig == null) { + return null; + } + var normalizationIntegrationType = normalizationConfig.get("normalizationIntegrationType"); + if (normalizationIntegrationType == null) { + return null; + } + return normalizationIntegrationType.asText(); } /** @@ -357,7 +381,7 @@ void setUpInternal() throws Exception { LOGGER.info("localRoot: {}", localRoot); testEnv = new TestDestinationEnv(localRoot); mConnectorConfigUpdater = Mockito.mock(ConnectorConfigUpdater.class); - + TEST_SCHEMAS = new HashSet<>(); setup(testEnv, TEST_SCHEMAS); processFactory = new DockerProcessFactory( @@ -765,7 +789,7 @@ public void testIncrementalDedupeSync() throws Exception { .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); - runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, true); + runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, supportsNormalization()); final List secondSyncMessages = Lists.newArrayList( new AirbyteMessage() @@ -796,7 +820,7 @@ public void testIncrementalDedupeSync() throws Exception { .withType(Type.STATE) .withState(new AirbyteStateMessage().withData( Jsons.jsonNode(ImmutableMap.of("checkpoint", 2))))); - runSyncAndVerifyStateOutput(config, secondSyncMessages, configuredCatalog, true); + runSyncAndVerifyStateOutput(config, secondSyncMessages, configuredCatalog, false); final List expectedMessagesAfterSecondSync = new ArrayList<>(); expectedMessagesAfterSecondSync.addAll(firstSyncMessages); @@ -829,22 +853,11 @@ public void testIncrementalDedupeSync() throws Exception { final String defaultSchema = getDefaultSchema(config); retrieveRawRecordsAndAssertSameMessages(catalog, expectedMessagesAfterSecondSync, defaultSchema); - final List actualMessages = retrieveNormalizedRecords(catalog, - defaultSchema); - assertSameMessages(expectedMessages, actualMessages, true); - } - - private String generateBigString(final int addExtraCharacters) { - final int length = getMaxRecordValueLimit() + addExtraCharacters; - return RANDOM - .ints('a', 'z' + 1) - .limit(length) - .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - .toString(); - } - - protected int getGenerateBigStringAddExtraCharacters() { - return 0; + if (normalizationFromDefinition()) { + final List actualMessages = retrieveNormalizedRecords(catalog, + defaultSchema); + assertSameMessages(expectedMessages, actualMessages, true); + } } /** @@ -1268,6 +1281,13 @@ protected void runSyncAndVerifyStateOutput(final JsonNode config, .stream() .filter(m -> m.getType() == Type.STATE) .findFirst() + .map(msg -> { + // Modify state message to remove destination stats. + final AirbyteStateMessage clone = msg.getState(); + clone.setDestinationStats(null); + msg.setState(clone); + return msg; + }) .orElseGet(() -> { fail("Destination failed to output state"); return null; @@ -1316,7 +1336,7 @@ private List runSync( destination.close(); - if (!runNormalization || (runNormalization && supportsInDestinationNormalization())) { + if (!runNormalization || (supportsInDestinationNormalization())) { return destinationOutput; } @@ -1517,7 +1537,7 @@ public void testStressPerformance() throws Exception { while (true) { System.out.println( "currentStreamNumber=" + currentStreamNumber + ", currentRecordNumberForStream=" - + currentRecordNumberForStream + ", " + DateTime.now()); + + currentRecordNumberForStream + ", " + Instant.now()); try { Thread.sleep(10000); } catch (final InterruptedException e) { @@ -1829,6 +1849,10 @@ public Stream provideArguments(final ExtensionContext conte } + private boolean supportsNormalization() { + return supportsInDestinationNormalization() || normalizationFromDefinition(); + } + private static V0 convertProtocolObject(final V1 v1, final Class klass) { return Jsons.object(Jsons.jsonNode(v1), klass); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..df4ca42e004b --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.standardtest.destination.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.quotedName; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import org.jooq.DSLContext; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.SQLDialect; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public abstract class JdbcSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { + + protected abstract JdbcDatabase getDatabase(); + + protected abstract DataType getStructType(); + + // TODO - can we move this class into db_destinations/testFixtures? + // then we could redefine getSqlGenerator() to return a JdbcSqlGenerator + // and this could be a private method getSqlGenerator().getTimestampWithTimeZoneType() + private DataType getTimestampWithTimeZoneType() { + return getSqlGenerator().toDialectType(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + } + + @Override + protected abstract JdbcSqlGenerator getSqlGenerator(); + + protected abstract SQLDialect getSqlDialect(); + + private DSLContext getDslContext() { + return DSL.using(getSqlDialect()); + } + + /** + * Many destinations require special handling to create JSON values. For example, redshift requires + * you to invoke JSON_PARSE('{...}'), and postgres requires you to CAST('{...}' AS JSONB). This + * method allows subclasses to implement that logic. + */ + protected abstract Field toJsonValue(String valueAsString); + + private void insertRecords(final Name tableName, final List columnNames, final List records, final String... columnsToParseJson) + throws SQLException { + InsertValuesStepN insert = getDslContext().insertInto( + DSL.table(tableName), + columnNames.stream().map(columnName -> field(quotedName(columnName))).toList()); + for (final JsonNode record : records) { + insert = insert.values( + columnNames.stream() + .map(fieldName -> { + // Convert this field to a string. Pretty naive implementation. + final JsonNode column = record.get(fieldName); + final String columnAsString; + if (column == null) { + columnAsString = null; + } else if (column.isTextual()) { + columnAsString = column.asText(); + } else { + columnAsString = column.toString(); + } + + if (Arrays.asList(columnsToParseJson).contains(fieldName)) { + return toJsonValue(columnAsString); + } else { + return DSL.val(columnAsString); + } + }) + .toList()); + } + getDatabase().execute(insert.getSQL(ParamType.INLINED)); + } + + @Override + protected void createNamespace(final String namespace) throws Exception { + getDatabase().execute(getDslContext().createSchemaIfNotExists(namespace).getSQL(ParamType.INLINED)); + } + + @Override + protected void createRawTable(final StreamId streamId) throws Exception { + getDatabase().execute(getDslContext().createTable(DSL.name(streamId.rawNamespace(), streamId.rawName())) + .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_AB_LOADED_AT, getTimestampWithTimeZoneType()) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .getSQL(ParamType.INLINED)); + } + + @Override + protected void createV1RawTable(final StreamId v1RawTable) throws Exception { + getDatabase().execute(getDslContext().createTable(DSL.name(v1RawTable.rawNamespace(), v1RawTable.rawName())) + .column(COLUMN_NAME_AB_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_EMITTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .getSQL(ParamType.INLINED)); + } + + @Override + protected void insertRawTableRecords(final StreamId streamId, final List records) throws Exception { + insertRecords( + DSL.name(streamId.rawNamespace(), streamId.rawName()), + JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES, + records, + COLUMN_NAME_DATA); + } + + @Override + protected void insertV1RawTableRecords(final StreamId streamId, final List records) throws Exception { + insertRecords( + DSL.name(streamId.rawNamespace(), streamId.rawName()), + LEGACY_RAW_TABLE_COLUMNS, + records, + COLUMN_NAME_DATA); + } + + @Override + protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, + final StreamId streamId, + final String suffix, + final List records) + throws Exception { + final List columnNames = + includeCdcDeletedAt ? BaseSqlGeneratorIntegrationTest.FINAL_TABLE_COLUMN_NAMES_CDC : BaseSqlGeneratorIntegrationTest.FINAL_TABLE_COLUMN_NAMES; + insertRecords( + DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix), + columnNames, + records, + COLUMN_NAME_AB_META, "struct", "array", "unknown"); + } + + @Override + protected List dumpRawTableRecords(final StreamId streamId) throws Exception { + return getDatabase().queryJsons(getDslContext().selectFrom(DSL.name(streamId.rawNamespace(), streamId.rawName())).getSQL(ParamType.INLINED)); + } + + @Override + protected List dumpFinalTableRecords(final StreamId streamId, final String suffix) throws Exception { + return getDatabase() + .queryJsons(getDslContext().selectFrom(DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix)).getSQL(ParamType.INLINED)); + } + + @Override + protected void teardownNamespace(final String namespace) throws Exception { + getDatabase().execute(getDslContext().dropSchema(namespace).cascade().getSQL(ParamType.INLINED)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java new file mode 100644 index 000000000000..f77448d62170 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.standardtest.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.util.List; +import javax.sql.DataSource; +import org.jooq.impl.DSL; + +/** + * This class is largely the same as + * {@link io.airbyte.integrations.destination.snowflake.typing_deduping.AbstractSnowflakeTypingDedupingTest}. + * But (a) it uses jooq to construct the sql statements, and (b) it doesn't need to upcase anything. + * At some point we might (?) want to do a refactor to combine them. + */ +public abstract class JdbcTypingDedupingTest extends BaseTypingDedupingTest { + + private JdbcDatabase database; + private DataSource dataSource; + + /** + * Get the config as declared in GSM (or directly from the testcontainer). This class will do + * further modification to the config to ensure test isolation.i + */ + protected abstract ObjectNode getBaseConfig(); + + protected abstract DataSource getDataSource(JsonNode config); + + /** + * Subclasses may need to return a custom source operations if the default one does not handle + * vendor-specific types correctly. For example, you most likely need to override this method to + * deserialize JSON columns to JsonNode. + */ + protected JdbcCompatibleSourceOperations getSourceOperations() { + return JdbcUtils.getDefaultSourceOperations(); + } + + /** + * Subclasses using a config with a nonstandard raw table schema should override this method. + */ + protected String getRawSchema() { + return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + } + + /** + * Subclasses using a config where the default schema is not in the {@code schema} key should + * override this method and {@link #setDefaultSchema(JsonNode, String)}. + */ + protected String getDefaultSchema(final JsonNode config) { + return config.get("schema").asText(); + } + + /** + * Subclasses using a config where the default schema is not in the {@code schema} key should + * override this method and {@link #getDefaultSchema(JsonNode)}. + */ + protected void setDefaultSchema(final JsonNode config, final String schema) { + ((ObjectNode) config).put("schema", schema); + } + + @Override + protected JsonNode generateConfig() { + final JsonNode config = getBaseConfig(); + setDefaultSchema(config, "typing_deduping_default_schema" + getUniqueSuffix()); + dataSource = getDataSource(config); + database = new DefaultJdbcDatabase(dataSource, getSourceOperations()); + return config; + } + + @Override + protected List dumpRawTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + final String tableName = StreamId.concatenateRawTableName(streamNamespace, streamName); + final String schema = getRawSchema(); + return database.queryJsons(DSL.selectFrom(DSL.name(schema, tableName)).getSQL()); + } + + @Override + protected List dumpFinalTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + return database.queryJsons(DSL.selectFrom(DSL.name(streamNamespace, streamName)).getSQL()); + } + + @Override + protected void teardownStreamAndNamespace(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + database.execute(DSL.dropTableIfExists(DSL.name(getRawSchema(), StreamId.concatenateRawTableName(streamNamespace, streamName))).getSQL()); + database.execute(DSL.dropSchemaIfExists(DSL.name(streamNamespace)).cascade().getSQL()); + } + + @Override + protected void globalTeardown() throws Exception { + DataSourceFactory.close(dataSource); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle index a34ad8242ed3..f615108ba0da 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle @@ -1,43 +1,16 @@ import org.jsonschema2pojo.SourceType -import org.jsoup.Jsoup - -buildscript { - dependencies { - // from standard-source-test: - classpath 'org.jsoup:jsoup:1.13.1' // for generateSourceTestDocs - } -} plugins { id "com.github.eirnym.js2p" version "1.0" - - id 'application' - id 'airbyte-integration-test-java' - id "java-library" - id "java-test-fixtures" // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures } java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal" + options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal,-this-escape" } } -project.configurations { - // From `base-debezium`: - testFixturesImplementation.extendsFrom implementation - - // From source-jdbc - testFixturesImplementation.extendsFrom implementation - testFixturesRuntimeOnly.extendsFrom runtimeOnly -} - -configurations.all { - // From airbyte-test-utils - exclude group: 'io.micronaut.jaxrs' - exclude group: 'io.micronaut.sql' -} - // Convert yaml to java: relationaldb.models jsonSchema2Pojo { sourceType = SourceType.YAMLSCHEMA @@ -54,162 +27,28 @@ jsonSchema2Pojo { } dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:core') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - - testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" - - - implementation libs.bundles.junit - // implementation libs.junit.jupiter.api - implementation libs.junit.jupiter.params - implementation 'org.junit.platform:junit-platform-launcher:1.7.0' - implementation libs.jooq - testImplementation libs.junit.jupiter.engine - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' - implementation "io.aesy:datasize:1.0.0" - implementation libs.apache.commons - implementation libs.apache.commons.lang - testImplementation 'commons-lang:commons-lang:2.6' - implementation 'commons-cli:commons-cli:1.4' - implementation 'org.apache.commons:commons-csv:1.4' - - // Optional dependencies - // TODO: Change these to 'compileOnly' or 'testCompileOnly' - implementation libs.hikaricp - implementation libs.debezium.api - implementation libs.debezium.embedded - implementation libs.debezium.sqlserver - implementation libs.debezium.mysql - implementation libs.debezium.postgres - implementation libs.debezium.mongodb - - implementation libs.bundles.datadog - // implementation 'com.datadoghq:dd-trace-api' - implementation 'org.apache.sshd:sshd-mina:2.8.0' - - implementation libs.testcontainers - implementation libs.testcontainers.mysql - implementation libs.testcontainers.jdbc - implementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.jdbc - testImplementation libs.testcontainers.mysql - testImplementation libs.testcontainers.postgresql - implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - - implementation 'org.bouncycastle:bcprov-jdk15on:1.66' - - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' - - testImplementation libs.junit.jupiter.system.stubs - - // From `base-debezium`: - // implementation project(':airbyte-db:db-lib') - // testFixturesImplementation project(':airbyte-db:db-lib') - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.4.2' - - // From source-jdbc - implementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.postgresql - integrationTestJavaImplementation libs.testcontainers.postgresql - testFixturesImplementation libs.airbyte.protocol - // todo (cgardens) - the java-test-fixtures plugin doesn't by default extend from test. - // we cannot make it depend on the dependencies of source-jdbc:test, because source-jdbc:test - // is going to depend on these fixtures. need to find a way to get fixtures to inherit the - // common test classes without duplicating them. this should be part of whatever solution we - // decide on for a "test-java-lib". the current implementation is leveraging the existing - // plugin, but we can something different if we don't like this tool. - testFixturesRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - testFixturesImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '4.0.0' - - // From `standard-source-test`: - testFixturesImplementation 'org.mockito:mockito-core:4.6.1' - testFixturesRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.1' - - // From `airbyte-test-utils`: - // api project(':airbyte-db:db-lib') - testFixturesImplementation 'io.fabric8:kubernetes-client:5.12.2' - testFixturesImplementation libs.temporal.sdk - testFixturesApi libs.junit.jupiter.api - // Mark as compile only to avoid leaking transitively to connectors - testFixturesCompileOnly libs.testcontainers.jdbc - testFixturesCompileOnly libs.testcontainers.postgresql - testFixturesCompileOnly libs.testcontainers.cockroachdb - testFixturesImplementation libs.testcontainers.cockroachdb -} - -def getFullPath(String className) { - def matchingFiles = project.fileTree("src/testFixtures/java") - .filter { file -> file.getName().equals("${className}.java".toString()) }.asCollection() - if (matchingFiles.size() == 0) { - throw new IllegalArgumentException("Ambiguous class name ${className}: no file found.") - } - if (matchingFiles.size() > 1) { - throw new IllegalArgumentException("Ambiguous class name ${className}: more than one matching file was found. Files found: ${matchingFiles}") - } - def absoluteFilePath = matchingFiles[0].toString() - def pathInPackage = project.relativePath(absoluteFilePath.toString()).replaceAll("src/testFixtures/java/", "").replaceAll("\\.java", "") - return pathInPackage -} + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') -def generateSourceTestDocs = tasks.register('generateSourceTestDocs', Javadoc) { - def javadocOutputDir = project.file("${project.buildDir}/docs/testFixturesJavadoc") + implementation 'io.debezium:debezium-api:2.4.0.Final' + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'org.codehaus.plexus:plexus-utils:4.0.0' - options.addStringOption('Xdoclint:none', '-quiet') - classpath = sourceSets.testFixtures.compileClasspath - source = sourceSets.testFixtures.allJava - destinationDir = javadocOutputDir + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) - doLast { - def className = "SourceAcceptanceTest" - // this can be made into a list once we have multiple standard tests, and can also be used for destinations - def pathInPackage = getFullPath(className) - def stdSrcTest = project.file("${javadocOutputDir}/${pathInPackage}.html").readLines().join("\n") - def methodList = Jsoup.parse(stdSrcTest).body().select("section.methodDetails>ul>li>section") - def md = "" - for (methodInfo in methodList) { - def annotations = methodInfo.select(".memberSignature>.annotations").text() - if (!annotations.contains("@Test")) { - continue - } - def methodName = methodInfo.selectFirst("div>span.memberName").text() - def methodDocstring = methodInfo.selectFirst("div.block") + testFixturesImplementation 'net.sourceforge.argparse4j:argparse4j:0.9.0' + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.13' + testFixturesImplementation 'org.hamcrest:hamcrest-all:1.3' + testFixturesImplementation 'org.junit.platform:junit-platform-launcher:1.10.1' - md += "## ${methodName}\n\n" - md += "${methodDocstring != null ? methodDocstring.text().replaceAll(/([()])/, '\\\\$1') : 'No method description was provided'}\n\n" - } - def outputDoc = new File("${rootDir}/docs/connector-development/testing-connectors/standard-source-tests.md") - outputDoc.write "# Standard Source Test Suite\n\n" - outputDoc.append "Test methods start with `test`. Other methods are internal helpers in the java class implementing the test suite.\n\n" - outputDoc.append md - } - outputs.upToDateWhen { false } -} + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testImplementation project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres')) -tasks.register('generate').configure { - dependsOn generateSourceTestDocs + testImplementation 'uk.org.webcompere:system-stubs-jupiter:2.0.1' } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java index 49f7afa14a6b..d3c2cad31622 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java @@ -13,9 +13,8 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; -import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter; import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordIterator; import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordPublisher; @@ -23,7 +22,6 @@ import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateDecoratingIterator; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.commons.util.MoreIterators; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; @@ -31,10 +29,7 @@ import io.debezium.engine.ChangeEvent; import io.debezium.engine.DebeziumEngine; import java.time.Duration; -import java.time.Instant; import java.util.Optional; -import java.util.OptionalInt; -import java.util.Properties; import java.util.concurrent.LinkedBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,88 +46,47 @@ public class AirbyteDebeziumHandler { * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_BATCH_SIZE}is 2048 * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_QUEUE_SIZE} is 8192 */ - private static final int QUEUE_CAPACITY = 10000; + public static final int QUEUE_CAPACITY = 10_000; private final JsonNode config; private final CdcTargetPosition targetPosition; private final boolean trackSchemaHistory; private final Duration firstRecordWaitTime, subsequentRecordWaitTime; - private final OptionalInt queueSize; + private final int queueSize; + private final boolean addDbNameToOffsetState; public AirbyteDebeziumHandler(final JsonNode config, final CdcTargetPosition targetPosition, final boolean trackSchemaHistory, final Duration firstRecordWaitTime, final Duration subsequentRecordWaitTime, - final OptionalInt queueSize) { + final int queueSize, + final boolean addDbNameToOffsetState) { this.config = config; this.targetPosition = targetPosition; this.trackSchemaHistory = trackSchemaHistory; this.firstRecordWaitTime = firstRecordWaitTime; this.subsequentRecordWaitTime = subsequentRecordWaitTime; this.queueSize = queueSize; + this.addDbNameToOffsetState = addDbNameToOffsetState; } - public AutoCloseableIterator getSnapshotIterators( - final ConfiguredAirbyteCatalog catalogContainingStreamsToSnapshot, - final CdcMetadataInjector cdcMetadataInjector, - final Properties snapshotProperties, - final CdcStateHandler cdcStateHandler, - final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, - final Instant emittedAt) { - - LOGGER.info("Running snapshot for " + catalogContainingStreamsToSnapshot.getStreams().size() + " new tables"); - final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(queueSize.orElse(QUEUE_CAPACITY)); - - final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeDummyStateForSnapshotPurpose(); - final DebeziumRecordPublisher tableSnapshotPublisher = new DebeziumRecordPublisher(snapshotProperties, - config, - catalogContainingStreamsToSnapshot, - offsetManager, - schemaHistoryManager(new SchemaHistory<>(Optional.empty(), false), cdcStateHandler.compressSchemaHistoryForState()), - debeziumConnectorType); - tableSnapshotPublisher.start(queue); - - final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( - queue, - targetPosition, - tableSnapshotPublisher::hasClosed, - new DebeziumShutdownProcedure<>(queue, tableSnapshotPublisher::close, tableSnapshotPublisher::hasClosed), - firstRecordWaitTime, - subsequentRecordWaitTime); - - return AutoCloseableIterators.concatWithEagerClose(AutoCloseableIterators - .transform( - eventIterator, - (event) -> DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, catalogContainingStreamsToSnapshot, emittedAt, - debeziumConnectorType)), - AutoCloseableIterators - .fromIterator(MoreIterators.singletonIteratorFromSupplier(cdcStateHandler::saveStateAfterCompletionOfSnapshotOfNewStreams))); - } - - public AutoCloseableIterator getIncrementalIterators(final ConfiguredAirbyteCatalog catalog, + public AutoCloseableIterator getIncrementalIterators(final DebeziumPropertiesManager debeziumPropertiesManager, + final DebeziumEventConverter eventConverter, final CdcSavedInfoFetcher cdcSavedInfoFetcher, - final CdcStateHandler cdcStateHandler, - final CdcMetadataInjector cdcMetadataInjector, - final Properties connectorProperties, - final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, - final Instant emittedAt, - final boolean addDbNameToState) { + final CdcStateHandler cdcStateHandler) { LOGGER.info("Using CDC: {}", true); LOGGER.info("Using DBZ version: {}", DebeziumEngine.class.getPackage().getImplementationVersion()); final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState( cdcSavedInfoFetcher.getSavedOffset(), - addDbNameToState ? Optional.ofNullable(config.get(JdbcUtils.DATABASE_KEY).asText()) : Optional.empty()); - final Optional schemaHistoryManager = - trackSchemaHistory ? schemaHistoryManager( - cdcSavedInfoFetcher.getSavedSchemaHistory(), - cdcStateHandler.compressSchemaHistoryForState()) - : Optional.empty(); - - final var publisher = new DebeziumRecordPublisher( - connectorProperties, config, catalog, offsetManager, schemaHistoryManager, debeziumConnectorType); - final var queue = new LinkedBlockingQueue>(queueSize.orElse(QUEUE_CAPACITY)); - publisher.start(queue); + addDbNameToOffsetState ? Optional.ofNullable(config.get(JdbcUtils.DATABASE_KEY).asText()) : Optional.empty()); + final var schemaHistoryManager = trackSchemaHistory + ? Optional.of(AirbyteSchemaHistoryStorage.initializeDBHistory( + cdcSavedInfoFetcher.getSavedSchemaHistory(), cdcStateHandler.compressSchemaHistoryForState())) + : Optional.empty(); + final var publisher = new DebeziumRecordPublisher(debeziumPropertiesManager); + final var queue = new LinkedBlockingQueue>(queueSize); + publisher.start(queue, offsetManager, schemaHistoryManager); // handle state machine around pub/sub logic. final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( queue, @@ -142,33 +96,22 @@ public AutoCloseableIterator getIncrementalIterators(final Confi firstRecordWaitTime, subsequentRecordWaitTime); - final Duration syncCheckpointDuration = - config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) - : SYNC_CHECKPOINT_DURATION; - final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() + final Duration syncCheckpointDuration = config.has(SYNC_CHECKPOINT_DURATION_PROPERTY) + ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) + : SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.has(SYNC_CHECKPOINT_RECORDS_PROPERTY) + ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() : SYNC_CHECKPOINT_RECORDS; return AutoCloseableIterators.fromIterator(new DebeziumStateDecoratingIterator<>( eventIterator, cdcStateHandler, targetPosition, - cdcMetadataInjector, - emittedAt, + eventConverter, offsetManager, trackSchemaHistory, schemaHistoryManager.orElse(null), syncCheckpointDuration, - syncCheckpointRecords, - catalog, - debeziumConnectorType)); - } - - private Optional schemaHistoryManager(final SchemaHistory> schemaHistory, - final boolean compressSchemaHistoryForState) { - if (trackSchemaHistory) { - return Optional.of(AirbyteSchemaHistoryStorage.initializeDBHistory(schemaHistory, compressSchemaHistoryForState)); - } - - return Optional.empty(); + syncCheckpointRecords)); } public static boolean isAnyStreamIncrementalSyncMode(final ConfiguredAirbyteCatalog catalog) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java index 2af71dfc0849..56ae64066283 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java @@ -74,7 +74,7 @@ default boolean isEventAheadOffset(final Map offset, final Chang * @return Returns `true` if both offsets are at the same position. Otherwise, it returns `false` */ default boolean isSameOffset(final Map offsetA, final Map offsetB) { - return true; + return false; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java index abab49414b71..d1576c3f2868 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java @@ -20,6 +20,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.Properties; import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; @@ -49,10 +50,6 @@ public AirbyteFileOffsetBackingStore(final Path offsetFilePath, final Optional read() { final Map raw = load(); @@ -185,4 +182,12 @@ public static AirbyteFileOffsetBackingStore initializeDummyStateForSnapshotPurpo return new AirbyteFileOffsetBackingStore(cdcOffsetFilePath, Optional.empty()); } + public void setDebeziumProperties(Properties props) { + // debezium engine configuration + // https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties + props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); + props.setProperty("offset.storage.file.filename", offsetFilePath.toString()); + props.setProperty("offset.flush.interval.ms", "1000"); // todo: make this longer + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java index 3ad851796acb..a3525cc0a3c8 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Optional; +import java.util.Properties; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import org.apache.commons.io.FileUtils; @@ -38,7 +39,7 @@ public class AirbyteSchemaHistoryStorage { private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteSchemaHistoryStorage.class); - private static final long SIZE_LIMIT_TO_COMPRESS_MB = 3; + private static final long SIZE_LIMIT_TO_COMPRESS_MB = 1; public static final int ONE_MB = 1024 * 1024; private static final Charset UTF8 = StandardCharsets.UTF_8; @@ -52,10 +53,6 @@ public AirbyteSchemaHistoryStorage(final Path path, final boolean compressSchema this.compressSchemaHistoryForState = compressSchemaHistoryForState; } - public Path getPath() { - return path; - } - public record SchemaHistory (T schema, boolean isCompressed) {} public SchemaHistory read() { @@ -224,4 +221,14 @@ public static AirbyteSchemaHistoryStorage initializeDBHistory(final SchemaHistor return schemaHistoryManager; } + public void setDebeziumProperties(Properties props) { + // https://debezium.io/documentation/reference/2.2/operations/debezium-server.html#debezium-source-database-history-class + // https://debezium.io/documentation/reference/development/engine.html#_in_the_code + // As mentioned in the documents above, debezium connector for MySQL needs to track the schema + // changes. If we don't do this, we can't fetch records for the table. + props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); + props.setProperty("schema.history.internal.file.filename", path.toString()); + props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventConverter.java new file mode 100644 index 000000000000..74c6e026a0b9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventConverter.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.time.Instant; + +public interface DebeziumEventConverter { + + String CDC_LSN = "_ab_cdc_lsn"; + String CDC_UPDATED_AT = "_ab_cdc_updated_at"; + String CDC_DELETED_AT = "_ab_cdc_deleted_at"; + String AFTER_EVENT = "after"; + String BEFORE_EVENT = "before"; + String OPERATION_FIELD = "op"; + String SOURCE_EVENT = "source"; + + static AirbyteMessage buildAirbyteMessage( + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final Instant emittedAt, + final JsonNode data) { + final String streamNamespace = cdcMetadataInjector.namespace(source); + final String streamName = cdcMetadataInjector.name(source); + + final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage() + .withStream(streamName) + .withNamespace(streamNamespace) + .withEmittedAt(emittedAt.toEpochMilli()) + .withData(data); + + return new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(airbyteRecordMessage); + } + + static JsonNode addCdcMetadata( + final ObjectNode baseNode, + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final boolean isDelete) { + + final long transactionMillis = source.get("ts_ms").asLong(); + final String transactionTimestamp = Instant.ofEpochMilli(transactionMillis).toString(); + + baseNode.put(CDC_UPDATED_AT, transactionTimestamp); + cdcMetadataInjector.addMetaData(baseNode, source); + + if (isDelete) { + baseNode.put(CDC_DELETED_AT, transactionTimestamp); + } else { + baseNode.put(CDC_DELETED_AT, (String) null); + } + + return baseNode; + } + + AirbyteMessage toAirbyteMessage(final ChangeEventWithMetadata event); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java deleted file mode 100644 index 0b1c3ca1d6b8..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.time.Instant; -import java.util.Set; -import java.util.stream.Collectors; - -public class DebeziumEventUtils { - - public static final String CDC_LSN = "_ab_cdc_lsn"; - public static final String CDC_UPDATED_AT = "_ab_cdc_updated_at"; - public static final String CDC_DELETED_AT = "_ab_cdc_deleted_at"; - - @VisibleForTesting - static final String AFTER_EVENT = "after"; - @VisibleForTesting - static final String BEFORE_EVENT = "before"; - @VisibleForTesting - static final String OPERATION_FIELD = "op"; - @VisibleForTesting - static final String SOURCE_EVENT = "source"; - - public static AirbyteMessage toAirbyteMessage(final ChangeEventWithMetadata event, - final CdcMetadataInjector cdcMetadataInjector, - final ConfiguredAirbyteCatalog configuredAirbyteCatalog, - final Instant emittedAt, - final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType) { - return switch (debeziumConnectorType) { - case MONGODB -> formatMongoDbEvent(event, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt); - case RELATIONALDB -> formatRelationalDbEvent(event, cdcMetadataInjector, emittedAt); - }; - } - - private static AirbyteMessage buildAirbyteMessage(final JsonNode source, - final CdcMetadataInjector cdcMetadataInjector, - final Instant emittedAt, - final JsonNode data) { - final String streamNamespace = cdcMetadataInjector.namespace(source); - final String streamName = cdcMetadataInjector.name(source); - - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage() - .withStream(streamName) - .withNamespace(streamNamespace) - .withEmittedAt(emittedAt.toEpochMilli()) - .withData(data); - - return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(airbyteRecordMessage); - } - - private static AirbyteMessage formatMongoDbEvent(final ChangeEventWithMetadata event, - final CdcMetadataInjector cdcMetadataInjector, - final ConfiguredAirbyteCatalog configuredAirbyteCatalog, - final Instant emittedAt) { - final JsonNode debeziumEventKey = event.eventKeyAsJson(); - final JsonNode debeziumEvent = event.eventValueAsJson(); - final JsonNode before = debeziumEvent.get(BEFORE_EVENT); - final JsonNode after = debeziumEvent.get(AFTER_EVENT); - final JsonNode source = debeziumEvent.get(SOURCE_EVENT); - final String operation = debeziumEvent.get(OPERATION_FIELD).asText(); - final Set configuredFields = getConfiguredMongoDbCollectionFields(source, configuredAirbyteCatalog, cdcMetadataInjector); - - /* - * Delete events need to be handled separately from other CrUD events, as depending on the version - * of the MongoDB server, the contents Debezium event data will be different. See - * #formatMongoDbDeleteDebeziumData() for more details. - */ - final JsonNode data = switch (operation) { - case "c", "i", "u" -> formatMongoDbDebeziumData(before, after, source, debeziumEventKey, cdcMetadataInjector, configuredFields); - case "d" -> formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields); - default -> throw new IllegalArgumentException("Unsupported MongoDB change event operation '" + operation + "'."); - }; - - return buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); - } - - private static AirbyteMessage formatRelationalDbEvent(final ChangeEventWithMetadata event, - final CdcMetadataInjector cdcMetadataInjector, - final Instant emittedAt) { - final JsonNode debeziumEvent = event.eventValueAsJson(); - final JsonNode before = debeziumEvent.get(BEFORE_EVENT); - final JsonNode after = debeziumEvent.get(AFTER_EVENT); - final JsonNode source = debeziumEvent.get(SOURCE_EVENT); - - final JsonNode data = formatRelationalDbDebeziumData(before, after, source, cdcMetadataInjector); - return buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); - } - - private static JsonNode formatMongoDbDebeziumData(final JsonNode before, - final JsonNode after, - final JsonNode source, - final JsonNode debeziumEventKey, - final CdcMetadataInjector cdcMetadataInjector, - final Set configuredFields) { - - if ((before == null || before.isNull()) && (after == null || after.isNull())) { - // In case a mongodb document was updated and then deleted, the update change event will not have - // any information ({after: null}) - // We are going to treat it as a delete. - return formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields); - } else { - final String eventJson = (after.isNull() ? before : after).asText(); - return addCdcMetadata(MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields), source, cdcMetadataInjector, false); - } - } - - private static JsonNode formatMongoDbDeleteDebeziumData(final JsonNode before, - final JsonNode debeziumEventKey, - final JsonNode source, - final CdcMetadataInjector cdcMetadataInjector, - final Set configuredFields) { - final String eventJson; - - /* - * The change events produced by MongoDB differ based on the server version. For version BEFORE 6.x, - * the event does not contain the before document. Therefore, the only data that can be extracted is - * the object ID of the deleted document, which is stored in the event key. Otherwise, if the server - * is version 6.+ AND the pre-image support has been enabled on the collection, we can use the - * "before" document from the event to represent the deleted document. - * - * See - * https://www.mongodb.com/docs/manual/reference/change-events/delete/#document-pre--and-post-images - * for more details. - */ - if (!before.isNull()) { - eventJson = before.asText(); - } else { - eventJson = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); - } - - return addCdcMetadata(MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields), source, cdcMetadataInjector, true); - } - - private static JsonNode formatRelationalDbDebeziumData(final JsonNode before, - final JsonNode after, - final JsonNode source, - final CdcMetadataInjector cdcMetadataInjector) { - final ObjectNode baseNode = (ObjectNode) (after.isNull() ? before : after); - return addCdcMetadata(baseNode, source, cdcMetadataInjector, after.isNull()); - - } - - private static JsonNode addCdcMetadata(final ObjectNode baseNode, - final JsonNode source, - final CdcMetadataInjector cdcMetadataInjector, - final boolean isDelete) { - - final long transactionMillis = source.get("ts_ms").asLong(); - final String transactionTimestamp = Instant.ofEpochMilli(transactionMillis).toString(); - - baseNode.put(CDC_UPDATED_AT, transactionTimestamp); - cdcMetadataInjector.addMetaData(baseNode, source); - - if (isDelete) { - baseNode.put(CDC_DELETED_AT, transactionTimestamp); - } else { - baseNode.put(CDC_DELETED_AT, (String) null); - } - - return baseNode; - } - - private static Set getConfiguredMongoDbCollectionFields(final JsonNode source, - final ConfiguredAirbyteCatalog configuredAirbyteCatalog, - final CdcMetadataInjector cdcMetadataInjector) { - final String streamNamespace = cdcMetadataInjector.namespace(source); - final String streamName = cdcMetadataInjector.name(source); - return configuredAirbyteCatalog.getStreams().stream() - .filter(s -> streamName.equals(s.getStream().getName()) && streamNamespace.equals(s.getStream().getNamespace())) - .map(CatalogHelpers::getTopLevelFieldNames) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java index c95e65be5de9..4bae69e9999b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.spi.common.ReplacementFunction; import java.util.Optional; import java.util.Properties; @@ -17,33 +18,29 @@ public abstract class DebeziumPropertiesManager { public static final String TOPIC_PREFIX_KEY = "topic.prefix"; private final JsonNode config; - private final AirbyteFileOffsetBackingStore offsetManager; - private final Optional schemaHistoryManager; - private final Properties properties; private final ConfiguredAirbyteCatalog catalog; public DebeziumPropertiesManager(final Properties properties, final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager) { + final ConfiguredAirbyteCatalog catalog) { this.properties = properties; this.config = config; this.catalog = catalog; - this.offsetManager = offsetManager; - this.schemaHistoryManager = schemaHistoryManager; } - public Properties getDebeziumProperties() { + public Properties getDebeziumProperties(final AirbyteFileOffsetBackingStore offsetManager) { + return getDebeziumProperties(offsetManager, Optional.empty()); + } + + public Properties getDebeziumProperties( + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager) { final Properties props = new Properties(); props.putAll(properties); // debezium engine configuration - // https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties - props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - props.setProperty("offset.storage.file.filename", offsetManager.getOffsetFilePath().toString()); - props.setProperty("offset.flush.interval.ms", "1000"); // todo: make this longer + offsetManager.setDebeziumProperties(props); // default values from debezium CommonConnectorConfig props.setProperty("max.batch.size", "2048"); props.setProperty("max.queue.size", "8192"); @@ -57,15 +54,7 @@ public Properties getDebeziumProperties() { props.setProperty("errors.retry.delay.initial.ms", "299"); props.setProperty("errors.retry.delay.max.ms", "300"); - if (schemaHistoryManager.isPresent()) { - // https://debezium.io/documentation/reference/2.2/operations/debezium-server.html#debezium-source-database-history-class - // https://debezium.io/documentation/reference/development/engine.html#_in_the_code - // As mentioned in the documents above, debezium connector for MySQL needs to track the schema - // changes. If we don't do this, we can't fetch records for the table. - props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); - props.setProperty("schema.history.internal.file.filename", schemaHistoryManager.get().getPath().toString()); - props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); - } + schemaHistoryManager.ifPresent(m -> m.setDebeziumProperties(props)); // https://debezium.io/documentation/reference/2.2/configuration/avro.html props.setProperty("key.converter.schemas.enable", "false"); @@ -88,8 +77,8 @@ public Properties getDebeziumProperties() { props.setProperty("max.queue.size.in.bytes", BYTE_VALUE_256_MB); // WARNING : Never change the value of this otherwise all the connectors would start syncing from - // scratch - props.setProperty(TOPIC_PREFIX_KEY, getName(config)); + // scratch. + props.setProperty(TOPIC_PREFIX_KEY, sanitizeTopicPrefix(getName(config))); // includes props.putAll(getIncludeConfiguration(catalog, config)); @@ -97,15 +86,37 @@ public Properties getDebeziumProperties() { return props; } + public static String sanitizeTopicPrefix(final String topicName) { + StringBuilder sanitizedNameBuilder = new StringBuilder(topicName.length()); + boolean changed = false; + + for (int i = 0; i < topicName.length(); ++i) { + char c = topicName.charAt(i); + if (isValidCharacter(c)) { + sanitizedNameBuilder.append(c); + } else { + sanitizedNameBuilder.append(ReplacementFunction.UNDERSCORE_REPLACEMENT.replace(c)); + changed = true; + } + } + + if (changed) { + return sanitizedNameBuilder.toString(); + } else { + return topicName; + } + } + + // We need to keep the validation rule the same as debezium engine, which is defined here: + // https://github.com/debezium/debezium/blob/c51ef3099a688efb41204702d3aa6d4722bb4825/debezium-core/src/main/java/io/debezium/schema/AbstractTopicNamingStrategy.java#L178 + private static boolean isValidCharacter(char c) { + return c == '.' || c == '_' || c == '-' || c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9'; + } + protected abstract Properties getConnectionConfiguration(final JsonNode config); protected abstract String getName(final JsonNode config); protected abstract Properties getIncludeConfiguration(final ConfiguredAirbyteCatalog catalog, final JsonNode config); - public enum DebeziumConnectorType { - RELATIONALDB, - MONGODB; - } - } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java index bc5a3ec037f6..93a05b70f586 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java @@ -4,15 +4,11 @@ package io.airbyte.cdk.integrations.debezium.internals; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.engine.ChangeEvent; import io.debezium.engine.DebeziumEngine; import io.debezium.engine.format.Json; import io.debezium.engine.spi.OffsetCommitPolicy; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -38,14 +34,8 @@ public class DebeziumRecordPublisher implements AutoCloseable { private final CountDownLatch engineLatch; private final DebeziumPropertiesManager debeziumPropertiesManager; - public DebeziumRecordPublisher(final Properties properties, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager, - final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType) { - this.debeziumPropertiesManager = createDebeziumPropertiesManager(debeziumConnectorType, properties, config, catalog, offsetManager, - schemaHistoryManager); + public DebeziumRecordPublisher(DebeziumPropertiesManager debeziumPropertiesManager) { + this.debeziumPropertiesManager = debeziumPropertiesManager; this.hasClosed = new AtomicBoolean(false); this.isClosing = new AtomicBoolean(false); this.thrownError = new AtomicReference<>(); @@ -53,21 +43,11 @@ public DebeziumRecordPublisher(final Properties properties, this.engineLatch = new CountDownLatch(1); } - private DebeziumPropertiesManager createDebeziumPropertiesManager(final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, - final Properties properties, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager) { - return switch (debeziumConnectorType) { - case MONGODB -> new MongoDbDebeziumPropertiesManager(properties, config, catalog, offsetManager); - default -> new RelationalDbDebeziumPropertiesManager(properties, config, catalog, offsetManager, schemaHistoryManager); - }; - } - - public void start(final BlockingQueue> queue) { + public void start(final BlockingQueue> queue, + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager) { engine = DebeziumEngine.create(Json.class) - .using(debeziumPropertiesManager.getDebeziumProperties()) + .using(debeziumPropertiesManager.getDebeziumProperties(offsetManager, schemaHistoryManager)) .using(new OffsetCommitPolicy.AlwaysCommitOffsetPolicy()) .notifying(e -> { // debezium outputs a tombstone event that has a value of null. this is an artifact of how it @@ -86,12 +66,14 @@ public void start(final BlockingQueue> queue) { .using((success, message, error) -> { LOGGER.info("Debezium engine shutdown. Engine terminated successfully : {}", success); LOGGER.info(message); - thrownError.set(error); - // If debezium has not shutdown correctly, it can indicate an error with the connector configuration - // or a partial sync success. - // In situations like these, the preference is to fail loud and clear. - if (thrownError.get() != null && !success) { - thrownError.set(new RuntimeException(message)); + if (!success) { + if (error != null) { + thrownError.set(error); + } else { + // There are cases where Debezium doesn't succeed but only fills the message field. + // In that case, we still want to fail loud and clear + thrownError.set(new RuntimeException(message)); + } } engineLatch.countDown(); }) diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java index bade687e5cea..71dfee610a3e 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java @@ -5,15 +5,12 @@ package io.airbyte.cdk.integrations.debezium.internals; import com.google.common.collect.AbstractIterator; -import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; import io.airbyte.cdk.integrations.debezium.CdcStateHandler; import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.DebeziumConnectorType; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import java.time.Duration; -import java.time.Instant; import java.time.OffsetDateTime; import java.util.HashMap; import java.util.Iterator; @@ -37,9 +34,6 @@ public class DebeziumStateDecoratingIterator extends AbstractIterator extends AbstractIterator extends AbstractIterator previousCheckpointOffset; - private final DebeziumConnectorType debeziumConnectorType; - private final ConfiguredAirbyteCatalog configuredAirbyteCatalog; + private final HashMap initialOffset, previousCheckpointOffset; + + private final DebeziumEventConverter eventConverter; /** * @param changeEventIterator Base iterator that we want to enrich with checkpoint messages * @param cdcStateHandler Handler to save the offset and schema history * @param offsetManager Handler to read and write debezium offset file + * @param eventConverter Handler to transform debezium events into Airbyte messages. * @param trackSchemaHistory Set true if the schema needs to be tracked * @param schemaHistoryManager Handler to write schema. Needs to be initialized if * trackSchemaHistory is set to true * @param checkpointDuration Duration object with time between syncs * @param checkpointRecords Number of records between syncs - * @param configuredAirbyteCatalog The {@link ConfiguredAirbyteCatalog} that contains the stream - * @param debeziumConnectorType type of connector that debezium will be capturing changes from */ public DebeziumStateDecoratingIterator(final Iterator changeEventIterator, final CdcStateHandler cdcStateHandler, final CdcTargetPosition targetPosition, - final CdcMetadataInjector cdcMetadataInjector, - final Instant emittedAt, + final DebeziumEventConverter eventConverter, final AirbyteFileOffsetBackingStore offsetManager, final boolean trackSchemaHistory, final AirbyteSchemaHistoryStorage schemaHistoryManager, final Duration checkpointDuration, - final Long checkpointRecords, - final ConfiguredAirbyteCatalog configuredAirbyteCatalog, - final DebeziumConnectorType debeziumConnectorType) { + final Long checkpointRecords) { this.changeEventIterator = changeEventIterator; this.cdcStateHandler = cdcStateHandler; this.targetPosition = targetPosition; - this.cdcMetadataInjector = cdcMetadataInjector; - this.emittedAt = emittedAt; + this.eventConverter = eventConverter; this.offsetManager = offsetManager; this.trackSchemaHistory = trackSchemaHistory; this.schemaHistoryManager = schemaHistoryManager; - this.configuredAirbyteCatalog = configuredAirbyteCatalog; - this.syncCheckpointDuration = checkpointDuration; this.syncCheckpointRecords = checkpointRecords; this.previousCheckpointOffset = (HashMap) offsetManager.read(); - this.debeziumConnectorType = debeziumConnectorType; + this.initialOffset = new HashMap<>(this.previousCheckpointOffset); resetCheckpointValues(); } @@ -140,7 +128,7 @@ protected AirbyteMessage computeNext() { if (cdcStateHandler.isCdcCheckpointEnabled() && sendCheckpointMessage) { LOGGER.info("Sending CDC checkpoint state message."); - final AirbyteMessage stateMessage = createStateMessage(checkpointOffsetToSend); + final AirbyteMessage stateMessage = createStateMessage(checkpointOffsetToSend, recordsLastSync); previousCheckpointOffset.clear(); previousCheckpointOffset.putAll(checkpointOffsetToSend); resetCheckpointValues(); @@ -151,7 +139,7 @@ protected AirbyteMessage computeNext() { final ChangeEventWithMetadata event = changeEventIterator.next(); if (cdcStateHandler.isCdcCheckpointEnabled()) { - if (checkpointOffsetToSend.size() == 0 && + if (checkpointOffsetToSend.isEmpty() && (recordsLastSync >= syncCheckpointRecords || Duration.between(dateTimeLastSync, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0)) { // Using temporal variable to avoid reading teh offset twice, one in the condition and another in @@ -174,11 +162,23 @@ protected AirbyteMessage computeNext() { } } recordsLastSync++; - return DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, debeziumConnectorType); + recordsAllSyncs++; + return eventConverter.toAirbyteMessage(event); } isSyncFinished = true; - return createStateMessage(offsetManager.read()); + final var syncFinishedOffset = (HashMap) offsetManager.read(); + if (recordsAllSyncs == 0L && targetPosition.isSameOffset(initialOffset, syncFinishedOffset)) { + // Edge case where no progress has been made: wrap up the + // sync by returning the initial offset instead of the + // current offset. We do this because we found that + // for some databases, heartbeats will cause Debezium to + // overwrite the offset file with a state which doesn't + // include all necessary data such as snapshot completion. + // This is the case for MS SQL Server, at least. + return createStateMessage(initialOffset, 0); + } + return createStateMessage(syncFinishedOffset, recordsLastSync); } /** @@ -197,7 +197,7 @@ private void resetCheckpointValues() { * * @return {@link AirbyteStateMessage} which includes offset and schema history if used. */ - private AirbyteMessage createStateMessage(final Map offset) { + private AirbyteMessage createStateMessage(final Map offset, final long recordCount) { if (trackSchemaHistory && schemaHistoryManager == null) { throw new RuntimeException("Schema History Tracking is true but manager is not initialised"); } @@ -205,7 +205,9 @@ private AirbyteMessage createStateMessage(final Map offset) { throw new RuntimeException("Offset can not be null"); } - return cdcStateHandler.saveState(offset, schemaHistoryManager != null ? schemaHistoryManager.read() : null); + final AirbyteMessage message = cdcStateHandler.saveState(offset, schemaHistoryManager != null ? schemaHistoryManager.read() : null); + message.getState().withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + return message; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumEventConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumEventConverter.java new file mode 100644 index 000000000000..4003007ba807 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumEventConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import java.time.Instant; + +public class RelationalDbDebeziumEventConverter implements DebeziumEventConverter { + + private final CdcMetadataInjector cdcMetadataInjector; + private final Instant emittedAt; + + public RelationalDbDebeziumEventConverter(CdcMetadataInjector cdcMetadataInjector, Instant emittedAt) { + this.cdcMetadataInjector = cdcMetadataInjector; + this.emittedAt = emittedAt; + } + + @Override + public AirbyteMessage toAirbyteMessage(ChangeEventWithMetadata event) { + final JsonNode debeziumEvent = event.eventValueAsJson(); + final JsonNode before = debeziumEvent.get(DebeziumEventConverter.BEFORE_EVENT); + final JsonNode after = debeziumEvent.get(DebeziumEventConverter.AFTER_EVENT); + final JsonNode source = debeziumEvent.get(DebeziumEventConverter.SOURCE_EVENT); + + final ObjectNode baseNode = (ObjectNode) (after.isNull() ? before : after); + final JsonNode data = DebeziumEventConverter.addCdcMetadata(baseNode, source, cdcMetadataInjector, after.isNull()); + return DebeziumEventConverter.buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java index 1d4a81376ac2..53af1cf72656 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java @@ -10,7 +10,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; import java.util.Iterator; -import java.util.Optional; import java.util.Properties; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -21,10 +20,8 @@ public class RelationalDbDebeziumPropertiesManager extends DebeziumPropertiesMan public RelationalDbDebeziumPropertiesManager(final Properties properties, final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager) { - super(properties, config, catalog, offsetManager, schemaHistoryManager); + final ConfiguredAirbyteCatalog catalog) { + super(properties, config, catalog); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MSSQLConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MSSQLConverter.java deleted file mode 100644 index 293dbc6b4790..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MSSQLConverter.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals.mssql; - -import com.microsoft.sqlserver.jdbc.Geography; -import com.microsoft.sqlserver.jdbc.Geometry; -import com.microsoft.sqlserver.jdbc.SQLServerException; -import io.airbyte.cdk.db.DataTypeUtils; -import io.airbyte.cdk.db.jdbc.DateTimeConverter; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumConverterUtils; -import io.debezium.spi.converter.CustomConverter; -import io.debezium.spi.converter.RelationalColumn; -import java.math.BigDecimal; -import java.nio.charset.Charset; -import java.sql.Timestamp; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import microsoft.sql.DateTimeOffset; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MSSQLConverter implements CustomConverter { - - private final Logger LOGGER = LoggerFactory.getLogger(MSSQLConverter.class); - - private final Set BINARY = Set.of("VARBINARY", "BINARY"); - private final Set DATETIME_TYPES = Set.of("DATETIME", "DATETIME2", "SMALLDATETIME"); - private final String DATE = "DATE"; - private static final String DATETIMEOFFSET = "DATETIMEOFFSET"; - private static final String TIME_TYPE = "TIME"; - private static final String SMALLMONEY_TYPE = "SMALLMONEY"; - private static final String GEOMETRY = "GEOMETRY"; - private static final String GEOGRAPHY = "GEOGRAPHY"; - private static final String DEBEZIUM_DATETIMEOFFSET_FORMAT = "yyyy-MM-dd HH:mm:ss XXX"; - - private static final String DATETIME_FORMAT_MICROSECONDS = "yyyy-MM-dd'T'HH:mm:ss[.][SSSSSS]"; - - @Override - public void configure(Properties props) {} - - @Override - public void converterFor(final RelationalColumn field, - final ConverterRegistration registration) { - if (DATE.equalsIgnoreCase(field.typeName())) { - registerDate(field, registration); - } else if (DATETIME_TYPES.contains(field.typeName().toUpperCase())) { - registerDatetime(field, registration); - } else if (SMALLMONEY_TYPE.equalsIgnoreCase(field.typeName())) { - registerMoney(field, registration); - } else if (BINARY.contains(field.typeName().toUpperCase())) { - registerBinary(field, registration); - } else if (GEOMETRY.equalsIgnoreCase(field.typeName())) { - registerGeometry(field, registration); - } else if (GEOGRAPHY.equalsIgnoreCase(field.typeName())) { - registerGeography(field, registration); - } else if (TIME_TYPE.equalsIgnoreCase(field.typeName())) { - registerTime(field, registration); - } else if (DATETIMEOFFSET.equalsIgnoreCase(field.typeName())) { - registerDateTimeOffSet(field, registration); - } - } - - private void registerGeometry(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - try { - return Geometry.deserialize((byte[]) input).toString(); - } catch (SQLServerException e) { - LOGGER.error(e.getMessage()); - } - } - - LOGGER.warn("Uncovered Geometry class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerGeography(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - try { - return Geography.deserialize((byte[]) input).toString(); - } catch (SQLServerException e) { - LOGGER.error(e.getMessage()); - } - } - - LOGGER.warn("Uncovered Geography class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerDate(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - if (field.typeName().equalsIgnoreCase("DATE")) { - return DateTimeConverter.convertToDate(input); - } - return DateTimeConverter.convertToTimestamp(input); - }); - } - - private void registerDatetime(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), - input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - final LocalDateTime localDateTime = ((Timestamp) input).toLocalDateTime(); - return localDateTime.format(DateTimeFormatter.ofPattern(DATETIME_FORMAT_MICROSECONDS)); - }); - - } - - private void registerDateTimeOffSet(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof DateTimeOffset) { - return DataTypeUtils.toISO8601String( - OffsetDateTime.parse(input.toString(), - DateTimeFormatter.ofPattern(DEBEZIUM_DATETIMEOFFSET_FORMAT))); - } - - LOGGER.warn("Uncovered DateTimeOffSet class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerTime(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof Timestamp) { - return DataTypeUtils.toISOTimeString(((Timestamp) input).toLocalDateTime()); - } - - LOGGER.warn("Uncovered time class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerMoney(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.float64(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof BigDecimal) { - return ((BigDecimal) input).doubleValue(); - } - - LOGGER.warn("Uncovered money class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerBinary(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - return new String((byte[]) input, Charset.defaultCharset()); - } - - LOGGER.warn("Uncovered binary class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java deleted file mode 100644 index 49d1523ffdd8..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals.mssql; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; -import io.debezium.connector.sqlserver.Lsn; -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MssqlCdcTargetPosition implements CdcTargetPosition { - - private static final Logger LOGGER = LoggerFactory.getLogger(MssqlCdcTargetPosition.class); - public final Lsn targetLsn; - - public MssqlCdcTargetPosition(final Lsn targetLsn) { - this.targetLsn = targetLsn; - } - - @Override - public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { - if (changeEventWithMetadata.isSnapshotEvent()) { - return false; - } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { - LOGGER.info("Signalling close because Snapshot is complete"); - return true; - } else { - final Lsn recordLsn = extractLsn(changeEventWithMetadata.eventValueAsJson()); - final boolean isEventLSNAfter = targetLsn.compareTo(recordLsn) <= 0; - if (isEventLSNAfter) { - LOGGER.info("Signalling close because record's LSN : " + recordLsn + " is after target LSN : " + targetLsn); - } - return isEventLSNAfter; - } - } - - @Override - public Lsn extractPositionFromHeartbeatOffset(final Map sourceOffset) { - throw new RuntimeException("Heartbeat is not supported for MSSQL"); - } - - private Lsn extractLsn(final JsonNode valueAsJson) { - return Optional.ofNullable(valueAsJson.get("source")) - .flatMap(source -> Optional.ofNullable(source.get("commit_lsn").asText())) - .map(Lsn::valueOf) - .orElseThrow(() -> new IllegalStateException("Could not find LSN")); - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final MssqlCdcTargetPosition that = (MssqlCdcTargetPosition) o; - return targetLsn.equals(that.targetLsn); - } - - @Override - public int hashCode() { - return targetLsn.hashCode(); - } - - public static MssqlCdcTargetPosition getTargetPosition(final JdbcDatabase database, final String dbName) { - try { - final List jsonNodes = database - .bufferedResultSetQuery(connection -> connection.createStatement().executeQuery( - "USE [" + dbName + "]; SELECT sys.fn_cdc_get_max_lsn() AS max_lsn;"), JdbcUtils.getDefaultSourceOperations()::rowToJson); - Preconditions.checkState(jsonNodes.size() == 1); - if (jsonNodes.get(0).get("max_lsn") != null) { - final Lsn maxLsn = Lsn.valueOf(jsonNodes.get(0).get("max_lsn").binaryValue()); - LOGGER.info("identified target lsn: " + maxLsn); - return new MssqlCdcTargetPosition(maxLsn); - } else { - throw new RuntimeException("SQL returned max LSN as null, this might be because the SQL Server Agent is not running. " + - "Please enable the Agent and try again (https://docs.microsoft.com/en-us/sql/ssms/agent/start-stop-or-pause-the-sql-server-agent-service?view=sql-server-ver15)"); - } - } catch (final SQLException | IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java index 0630c945804e..42d480007285 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java @@ -89,7 +89,6 @@ public abstract class AbstractJdbcSource extends AbstractDbSource streamingQueryConfigProvider; protected final JdbcCompatibleSourceOperations sourceOperations; @@ -99,7 +98,7 @@ public abstract class AbstractJdbcSource extends AbstractDbSource streamingQueryConfigProvider, final JdbcCompatibleSourceOperations sourceOperations) { - this.driverClass = driverClass; + super(driverClass); this.streamingQueryConfigProvider = streamingQueryConfigProvider; this.sourceOperations = sourceOperations; } @@ -427,14 +426,20 @@ protected long getActualCursorRecordCount(final Connection connection, @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { + return createDatabase(sourceConfig, JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER); + } + + public JdbcDatabase createDatabase(final JsonNode sourceConfig, String delimiter) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + Map connectionProperties = JdbcDataSourceUtils.getConnectionProperties(sourceConfig, delimiter); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcDataSourceUtils.getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties)); // Record the data source so that it can be closed. dataSources.add(dataSource); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java index 479214ea0960..f11193178ec4 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java @@ -38,7 +38,11 @@ public static void assertCustomParametersDontOverwriteDefaultParameters(final Ma * @return A mapping of connection properties */ public static Map getConnectionProperties(final JsonNode config) { - final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY); + return getConnectionProperties(config, DEFAULT_JDBC_PARAMETERS_DELIMITER); + } + + public static Map getConnectionProperties(final JsonNode config, String parameterDelimiter) { + final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY, parameterDelimiter); final Map defaultProperties = JdbcDataSourceUtils.getDefaultConnectionProperties(config); assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); return MoreMaps.merge(customProperties, defaultProperties); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java index 0d604dce7518..26d04bed4b6b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java @@ -13,7 +13,7 @@ import io.airbyte.cdk.db.AbstractDatabase; import io.airbyte.cdk.db.IncrementalUtils; import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.source.relationaldb.InvalidCursorInfoUtil.InvalidCursorInfo; @@ -69,7 +69,7 @@ * source of both non-relational and relational type */ public abstract class AbstractDbSource extends - BaseConnector implements Source, AutoCloseable { + JdbcConnector implements Source, AutoCloseable { public static final String CHECK_TRACE_OPERATION_NAME = "check-operation"; public static final String DISCOVER_TRACE_OPERATION_NAME = "discover-operation"; @@ -80,6 +80,10 @@ public abstract class AbstractDbSource read(final JsonNode config, final AirbyteStateType supportedStateType = getSupportedStateType(config); final StateManager stateManager = StateManagerFactory.createStateManager(supportedStateType, - StateGeneratorUtils.deserializeInitialState(state, featureFlags.useStreamCapableState(), supportedStateType), catalog); + StateGeneratorUtils.deserializeInitialState(state, supportedStateType), catalog); final Instant emittedAt = Instant.now(); final Database database = createDatabase(config); @@ -685,7 +689,7 @@ protected int getStateEmissionFrequency() { * @return A {@link AirbyteStateType} representing the state supported by this connector. */ protected AirbyteStateType getSupportedStateType(final JsonNode config) { - return AirbyteStateType.LEGACY; + return AirbyteStateType.STREAM; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java index bffed2b6d040..fd66d1a43b35 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java @@ -10,8 +10,10 @@ import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.List; import java.util.StringJoiner; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +25,8 @@ public class RelationalDbQueryUtils { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalDbQueryUtils.class); + public record TableSizeInfo(Long tableSize, Long avgRowLength) {} + public static String getIdentifierWithQuoting(final String identifier, final String quoteString) { // double-quoted values within a database name or column name should be wrapped with extra // quoteString @@ -79,4 +83,17 @@ public static AutoCloseableIterator que }, airbyteStreamNameNamespacePair); } + public static void logStreamSyncStatus(final List streams, final String syncType) { + if (streams.isEmpty()) { + LOGGER.info("No Streams will be synced via {}.", syncType); + } else { + LOGGER.info("Streams to be synced via {} : {}", syncType, streams.size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(streams)); + } + } + + public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { + return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbReadUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbReadUtil.java new file mode 100644 index 000000000000..9e1b8464e06a --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbReadUtil.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb; + +import com.google.common.collect.Sets; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class RelationalDbReadUtil { + + public static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public static List identifyStreamsForCursorBased(final ConfiguredAirbyteCatalog catalog, + final List streamsForInitialLoad) { + + final Set initialLoadStreamsNamespacePairs = + streamsForInitialLoad.stream().map(stream -> AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream())) + .collect( + Collectors.toSet()); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> !initialLoadStreamsNamespacePairs.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public static AirbyteStreamNameNamespacePair convertNameNamespacePairFromV0(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair v1NameNamespacePair) { + return new AirbyteStreamNameNamespacePair(v1NameNamespacePair.getName(), v1NameNamespacePair.getNamespace()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java index 5c8f7d638ebb..919d38b3bb50 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java @@ -11,6 +11,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Iterator; import java.util.Objects; @@ -53,6 +54,8 @@ public class StateDecoratingIterator extends AbstractIterator im */ private final int stateEmissionFrequency; private int totalRecordCount = 0; + // In between each state message, recordCountInStateMessage will be reset to 0. + private int recordCountInStateMessage = 0; private boolean emitIntermediateState = false; private AirbyteMessage intermediateStateMessage = null; private boolean hasCaughtException = false; @@ -128,6 +131,7 @@ protected AirbyteMessage computeNext() { } totalRecordCount++; + recordCountInStateMessage++; // Use try-catch to catch Exception that could occur when connection to the database fails try { final AirbyteMessage message = messageIterator.next(); @@ -139,7 +143,7 @@ protected AirbyteMessage computeNext() { if (stateEmissionFrequency > 0 && !Objects.equals(currentMaxCursor, initialCursor) && messageIterator.hasNext()) { // Only create an intermediate state when it is not the first or last record message. // The last state message will be processed seperately. - intermediateStateMessage = createStateMessage(false, totalRecordCount); + intermediateStateMessage = createStateMessage(false, recordCountInStateMessage); } currentMaxCursor = cursorCandidate; currentMaxCursorRecordCount = 1L; @@ -164,7 +168,7 @@ protected AirbyteMessage computeNext() { return optionalIntermediateMessage.orElse(endOfData()); } } else if (!hasEmittedFinalState) { - return createStateMessage(true, totalRecordCount); + return createStateMessage(true, recordCountInStateMessage); } else { return endOfData(); } @@ -184,7 +188,12 @@ protected AirbyteMessage computeNext() { protected final Optional getIntermediateMessage() { if (emitIntermediateState && intermediateStateMessage != null) { final AirbyteMessage message = intermediateStateMessage; + if (message.getState() != null) { + message.getState().setSourceStats(new AirbyteStateStats().withRecordCount((double) recordCountInStateMessage)); + } + intermediateStateMessage = null; + recordCountInStateMessage = 0; emitIntermediateState = false; return Optional.of(message); } @@ -196,14 +205,15 @@ protected final Optional getIntermediateMessage() { * read up so far * * @param isFinalState marker for if the final state of the iterator has been reached - * @param totalRecordCount count of read messages + * @param recordCount count of read messages * @return AirbyteMessage which includes information on state of records read so far */ - public AirbyteMessage createStateMessage(final boolean isFinalState, final int totalRecordCount) { + public AirbyteMessage createStateMessage(final boolean isFinalState, final int recordCount) { final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, currentMaxCursor, currentMaxCursorRecordCount); final Optional cursorInfo = stateManager.getCursorInfo(pair); + // logging once every 100 messages to reduce log verbosity - if (totalRecordCount % 100 == 0) { + if (recordCount % 100 == 0) { LOGGER.info("State report for stream {} - original: {} = {} (count {}) -> latest: {} = {} (count {})", pair, cursorInfo.map(CursorInfo::getOriginalCursorField).orElse(null), @@ -213,6 +223,10 @@ public AirbyteMessage createStateMessage(final boolean isFinalState, final int t cursorInfo.map(CursorInfo::getCursor).orElse(null), cursorInfo.map(CursorInfo::getCursorRecordCount).orElse(null)); } + + if (stateMessage != null) { + stateMessage.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + } if (isFinalState) { hasEmittedFinalState = true; if (stateManager.getCursor(pair).isEmpty()) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java new file mode 100644 index 000000000000..5166ae2898ae --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import com.google.common.collect.AbstractIterator; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import java.time.Instant; +import java.util.Iterator; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SourceStateIterator extends AbstractIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(SourceStateIterator.class); + private final Iterator messageIterator; + private boolean hasEmittedFinalState = false; + private long recordCount = 0L; + private Instant lastCheckpoint = Instant.now(); + + private final SourceStateIteratorManager sourceStateIteratorManager; + + public SourceStateIterator(final Iterator messageIterator, + final SourceStateIteratorManager sourceStateIteratorManager) { + this.messageIterator = messageIterator; + this.sourceStateIteratorManager = sourceStateIteratorManager; + } + + @CheckForNull + @Override + protected AirbyteMessage computeNext() { + + boolean iteratorHasNextValue = false; + try { + iteratorHasNextValue = messageIterator.hasNext(); + } catch (final Exception ex) { + // If the initial snapshot is incomplete for this stream, throw an exception failing the sync. This + // will ensure the platform retry logic + // kicks in and keeps retrying the sync until the initial snapshot is complete. + throw new RuntimeException(ex); + } + if (iteratorHasNextValue) { + if (sourceStateIteratorManager.shouldEmitStateMessage(recordCount, lastCheckpoint)) { + final AirbyteStateMessage stateMessage = sourceStateIteratorManager.generateStateMessageAtCheckpoint(); + stateMessage.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + + recordCount = 0L; + lastCheckpoint = Instant.now(); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateMessage); + } + // Use try-catch to catch Exception that could occur when connection to the database fails + try { + final T message = messageIterator.next(); + final AirbyteMessage processedMessage = sourceStateIteratorManager.processRecordMessage(message); + recordCount++; + return processedMessage; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } else if (!hasEmittedFinalState) { + hasEmittedFinalState = true; + final AirbyteStateMessage finalStateMessageForStream = sourceStateIteratorManager.createFinalStateMessage(); + finalStateMessageForStream.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + recordCount = 0L; + return new AirbyteMessage() + .withType(Type.STATE) + .withState(finalStateMessageForStream); + } else { + return endOfData(); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java new file mode 100644 index 000000000000..a76b0256be2f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Instant; + +public interface SourceStateIteratorManager { + + /** + * Returns a state message that should be emitted at checkpoint. + */ + AirbyteStateMessage generateStateMessageAtCheckpoint(); + + /** + * For the incoming record message, this method defines how the connector will consume it. + */ + AirbyteMessage processRecordMessage(final T message); + + /** + * At the end of the iteration, this method will be called and it will generate the final state + * message. + * + * @return + */ + AirbyteStateMessage createFinalStateMessage(); + + /** + * Determines if the iterator has reached checkpoint or not, based on the time and number of record + * messages it has been processed since the last checkpoint. + */ + boolean shouldEmitStateMessage(final long recordCount, final Instant lastCheckpoint); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java index f177ae310809..4c272190946b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java @@ -225,10 +225,8 @@ public static AirbyteStateMessage convertStateMessage(final io.airbyte.protocol. * @return The deserialized object representation of the state. */ public static List deserializeInitialState(final JsonNode initialStateJson, - final boolean useStreamCapableState, final AirbyteStateType supportedStateType) { - final Optional typedState = StateMessageHelper.getTypedState(initialStateJson, - useStreamCapableState); + final Optional typedState = StateMessageHelper.getTypedState(initialStateJson); return typedState .map(state -> switch (state.getStateType()) { case GLOBAL -> List.of(StateGeneratorUtils.convertStateMessage(state.getGlobal())); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/db_models/internal_models.yaml b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/db_models/internal_models.yaml new file mode 100644 index 000000000000..cb462b9cffcc --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/db_models/internal_models.yaml @@ -0,0 +1,49 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +title: DbSource Models +type: object +description: DbSource Models +properties: + state_type: + "$ref": "#/definitions/StateType" + ordered_column_state: + "$ref": "#/definitions/OrderedColumnLoadStatus" + cursor_based_state: + "$ref": "#/definitions/CursorBasedStatus" +definitions: + StateType: + description: Enum to define the sync mode of stream state. + type: string + enum: + - cursor_based + - ordered_column + - cdc + CursorBasedStatus: + type: object + extends: + type: object + existingJavaType: "io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState" + properties: + state_type: + "$ref": "#/definitions/StateType" + version: + description: Version of state. + type: integer + OrderedColumnLoadStatus: + type: object + properties: + version: + description: Version of state. + type: integer + state_type: + "$ref": "#/definitions/StateType" + ordered_col: + description: ordered column name + type: string + ordered_col_val: + description: ordered column high watermark + type: string + incremental_state: + description: State to switch to after completion of the ordered column initial sync + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json deleted file mode 100644 index 892b30269c60..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "username": "default", - "jdbc_url": "default" -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json deleted file mode 100644 index df19d77048be..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "JDBC Source Spec", - "type": "object", - "required": ["username", "jdbc_url"], - "properties": { - "username": { - "title": "Username", - "description": "The username which is used to access the database.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "The password associated with this username.", - "type": "string", - "airbyte_secret": true - }, - "jdbc_url": { - "title": "JDBC URL", - "description": "JDBC formatted URL. See the standard here.", - "type": "string" - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string" - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java index 268c11a6012e..482936bd54aa 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java @@ -19,8 +19,8 @@ public class AirbyteSchemaHistoryStorageTest { @Test - public void testForContentBiggerThan3MBLimit() throws IOException { - final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_greater_than_3_mb.dat"); + public void testForContentBiggerThan1MBLimit() throws IOException { + final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_greater_than_1_mb.dat"); final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)), @@ -46,14 +46,14 @@ public void testForContentBiggerThan3MBLimit() throws IOException { @Test public void sizeTest() throws IOException { assertEquals(5.881045341491699, - AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_greater_than_3_mb.dat"))); + AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_greater_than_1_mb.dat"))); assertEquals(0.0038671493530273438, - AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_less_than_3_mb.dat"))); + AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_less_than_1_mb.dat"))); } @Test - public void testForContentLessThan3MBLimit() throws IOException { - final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_less_than_3_mb.dat"); + public void testForContentLessThan1MBLimit() throws IOException { + final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_less_than_1_mb.dat"); final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)), diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java deleted file mode 100644 index be46d4a561cf..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals; - -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.DebeziumConnectorType; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.debezium.engine.ChangeEvent; -import java.io.IOException; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; - -class DebeziumEventUtilsTest { - - @Test - void testConvertRelationalDbChangeEvent() throws IOException { - final String stream = "names"; - final Instant emittedAt = Instant.now(); - final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); - final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("insert_change_event.json", ""); - final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("update_change_event.json", ""); - final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("delete_change_event.json", ""); - final ConfiguredAirbyteCatalog configuredAirbyteCatalog = mock(ConfiguredAirbyteCatalog.class); - - final AirbyteMessage actualInsert = - DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, - DebeziumConnectorType.RELATIONALDB); - final AirbyteMessage actualUpdate = - DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, - DebeziumConnectorType.RELATIONALDB); - final AirbyteMessage actualDelete = - DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, - DebeziumConnectorType.RELATIONALDB); - - final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "insert_message.json"); - final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "update_message.json"); - final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "delete_message.json"); - - deepCompare(expectedInsert, actualInsert); - deepCompare(expectedUpdate, actualUpdate); - deepCompare(expectedDelete, actualDelete); - } - - @Test - void testConvertMongoDbChangeEvent() throws IOException { - final String objectId = "64f24244f95155351c4185b1"; - final String stream = "names"; - final Instant emittedAt = Instant.now(); - final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); - final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("mongodb/change_event_insert.json", ""); - final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("mongodb/change_event_update.json", ""); - final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("mongodb/change_event_delete.json", ""); - final ChangeEventWithMetadata deleteChangeEventNoBefore = mockChangeEvent("mongodb/change_event_delete_no_before.json", - "{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + objectId + "\\\"}"); - - final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "mongodb/insert_airbyte_message.json"); - final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "mongodb/update_airbyte_message.json"); - final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "mongodb/delete_airbyte_message.json"); - final AirbyteMessage expectedDeleteNoBefore = createAirbyteMessage(stream, emittedAt, "mongodb/delete_no_before_airbyte_message.json"); - - final ConfiguredAirbyteCatalog insertConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedInsert); - final ConfiguredAirbyteCatalog updateConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedUpdate); - final ConfiguredAirbyteCatalog deleteConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDelete); - final ConfiguredAirbyteCatalog deleteNoBeforeConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDeleteNoBefore); - - final AirbyteMessage actualInsert = - DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, insertConfiguredAirbyteCatalog, emittedAt, - DebeziumConnectorType.MONGODB); - final AirbyteMessage actualUpdate = - DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, updateConfiguredAirbyteCatalog, emittedAt, - DebeziumConnectorType.MONGODB); - final AirbyteMessage actualDelete = - DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, deleteConfiguredAirbyteCatalog, emittedAt, - DebeziumConnectorType.MONGODB); - final AirbyteMessage actualDeleteNoBefore = - DebeziumEventUtils.toAirbyteMessage(deleteChangeEventNoBefore, cdcMetadataInjector, deleteNoBeforeConfiguredAirbyteCatalog, emittedAt, - DebeziumConnectorType.MONGODB); - - deepCompare(expectedInsert, actualInsert); - deepCompare(expectedUpdate, actualUpdate); - deepCompare(expectedDelete, actualDelete); - deepCompare(expectedDeleteNoBefore, actualDeleteNoBefore); - } - - @Test - void testConvertMongoDbChangeEventUnsupportedOperation() throws IOException { - final Instant emittedAt = Instant.now(); - final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); - final ChangeEventWithMetadata unsupportedOperationEvent = mockChangeEvent("mongodb/change_event_unsupported.json", ""); - final ConfiguredAirbyteCatalog configuredAirbyteCatalog = mock(ConfiguredAirbyteCatalog.class); - assertThrows(IllegalArgumentException.class, - () -> DebeziumEventUtils.toAirbyteMessage(unsupportedOperationEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, - DebeziumConnectorType.MONGODB)); - } - - private ConfiguredAirbyteCatalog buildFromAirbyteMessage(final AirbyteMessage airbyteMessage) { - final ConfiguredAirbyteCatalog configuredAirbyteCatalog = new ConfiguredAirbyteCatalog(); - final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream(); - final AirbyteStream airbyteStream = new AirbyteStream(); - airbyteStream.setName(airbyteMessage.getRecord().getStream()); - airbyteStream.setNamespace(airbyteMessage.getRecord().getNamespace()); - airbyteStream.setJsonSchema(Jsons.jsonNode(Map.of("properties", airbyteMessage.getRecord().getData()))); - configuredAirbyteStream.setStream(airbyteStream); - configuredAirbyteCatalog.setStreams(List.of(configuredAirbyteStream)); - return configuredAirbyteCatalog; - } - - private static ChangeEventWithMetadata mockChangeEvent(final String resourceName, final String idValue) throws IOException { - final ChangeEvent mocked = mock(ChangeEvent.class); - final String resource = MoreResources.readResource(resourceName); - final String key = "{\"" + ID_FIELD + "\":\"" + idValue + "\"}"; - when(mocked.key()).thenReturn(key); - when(mocked.value()).thenReturn(resource); - - return new ChangeEventWithMetadata(mocked); - } - - private static AirbyteMessage createAirbyteMessage(final String stream, final Instant emittedAt, final String resourceName) throws IOException { - final String data = MoreResources.readResource(resourceName); - - final AirbyteRecordMessage recordMessage = new AirbyteRecordMessage() - .withStream(stream) - .withNamespace("public") - .withData(Jsons.deserialize(data)) - .withEmittedAt(emittedAt.toEpochMilli()); - - return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(recordMessage); - } - - private static void deepCompare(final Object expected, final Object actual) { - assertEquals(Jsons.deserialize(Jsons.serialize(expected)), Jsons.deserialize(Jsons.serialize(actual))); - } - - public static class DummyMetadataInjector implements CdcMetadataInjector { - - @Override - public void addMetaData(final ObjectNode event, final JsonNode source) { - if (source.has("lsn")) { - final long lsn = source.get("lsn").asLong(); - event.put("_ab_cdc_lsn", lsn); - } - } - - @Override - public String namespace(final JsonNode source) { - return source.has("schema") ? source.get("schema").asText() : source.get("db").asText(); - } - - @Override - public String name(final JsonNode source) { - return source.has("table") ? source.get("table").asText() : source.get("collection").asText(); - } - - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java deleted file mode 100644 index 75146837ca89..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals.mongodb; - -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.DOCUMENT_OBJECT_ID_FIELD; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD_PATTERN; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.db.DataTypeUtils; -import io.airbyte.commons.json.Jsons; -import java.nio.charset.Charset; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import org.apache.commons.codec.binary.Base64; -import org.bson.BsonBinary; -import org.bson.BsonBoolean; -import org.bson.BsonDateTime; -import org.bson.BsonDecimal128; -import org.bson.BsonDocument; -import org.bson.BsonDouble; -import org.bson.BsonInt32; -import org.bson.BsonInt64; -import org.bson.BsonJavaScript; -import org.bson.BsonJavaScriptWithScope; -import org.bson.BsonNull; -import org.bson.BsonObjectId; -import org.bson.BsonRegularExpression; -import org.bson.BsonString; -import org.bson.BsonSymbol; -import org.bson.BsonTimestamp; -import org.bson.Document; -import org.bson.UuidRepresentation; -import org.bson.types.Decimal128; -import org.bson.types.ObjectId; -import org.junit.jupiter.api.Test; - -class MongoDbCdcEventUtilsTest { - - private static final String OBJECT_ID = "64f24244f95155351c4185b1"; - - @Test - void testGenerateObjectIdDocument() { - final String key = "{\"" + OBJECT_ID_FIELD + "\": \"" + OBJECT_ID + "\"}"; - JsonNode debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, key)); - - String updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); - - assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); - assertEquals(key.replaceAll(OBJECT_ID_FIELD_PATTERN, DOCUMENT_OBJECT_ID_FIELD), updated); - - debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, "\"" + OBJECT_ID + "\"")); - updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); - assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); - assertEquals(Jsons.serialize(debeziumEventKey).replaceAll(ID_FIELD, DOCUMENT_OBJECT_ID_FIELD), updated); - } - - @Test - void testNormalizeObjectId() { - - final JsonNode data = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( - Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID)))); - assertEquals(OBJECT_ID, data.get(DOCUMENT_OBJECT_ID_FIELD).asText()); - - final JsonNode dataWithoutObjectId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( - Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of()))); - assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); - - final JsonNode dataWithoutId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode(Map.of())); - assertNull(dataWithoutId.get(DOCUMENT_OBJECT_ID_FIELD)); - } - - @Test - void testTransformDataTypes() { - final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); - final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); - final UUID standardUuid = UUID.randomUUID(); - final UUID legacyUuid = UUID.randomUUID(); - - final Document document = new Document("field1", new BsonBoolean(true)) - .append("field2", new BsonInt32(1)) - .append("field3", new BsonInt64(2)) - .append("field4", new BsonDouble(3.0)) - .append("field5", new BsonDecimal128(new Decimal128(4))) - .append("field6", bsonTimestamp) - .append("field7", new BsonDateTime(bsonTimestamp.getValue())) - .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) - .append("field9", new BsonSymbol("test2")) - .append("field10", new BsonString("test3")) - .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) - .append("field12", new BsonJavaScript("code")) - .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) - .append("field14", new BsonRegularExpression("pattern")) - .append("field15", new BsonNull()) - .append("field16", new Document("key", "value")) - .append("field17", new BsonBinary(standardUuid, UuidRepresentation.STANDARD)) - .append("field18", new BsonBinary(legacyUuid, UuidRepresentation.JAVA_LEGACY)); - - final String documentAsJson = document.toJson(); - final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, document.keySet()); - - assertNotNull(transformed); - assertNotEquals(documentAsJson, Jsons.serialize(transformed)); - assertEquals(true, transformed.get("field1").asBoolean()); - assertEquals(1, transformed.get("field2").asInt()); - assertEquals(2, transformed.get("field3").asInt()); - assertEquals(3.0, transformed.get("field4").asDouble()); - assertEquals(4.0, transformed.get("field5").asDouble()); - assertEquals(expectedTimestamp, transformed.get("field6").asText()); - assertEquals(expectedTimestamp, transformed.get("field7").asText()); - assertEquals(Base64.encodeBase64String("test".getBytes(Charset.defaultCharset())), transformed.get("field8").asText()); - assertEquals("test2", transformed.get("field9").asText()); - assertEquals("test3", transformed.get("field10").asText()); - assertEquals(OBJECT_ID, transformed.get("field11").asText()); - assertEquals("code", transformed.get("field12").asText()); - assertEquals("code2", transformed.get("field13").get("code").asText()); - assertEquals("scope", transformed.get("field13").get("scope").get("scope").asText()); - assertEquals("pattern", transformed.get("field14").asText()); - assertFalse(transformed.has("field15")); - assertEquals("value", transformed.get("field16").get("key").asText()); - // Assert that UUIDs can be serialized. Currently, they will be represented as base 64 encoded - // strings. Since the original mongo source - // may have these UUIDs written by a variety of sources, each with different encodings - we cannot - // decode these back to the original UUID. - assertTrue(transformed.has("field17")); - assertTrue(transformed.has("field18")); - } - - @Test - void testTransformDataTypesWithFilteredFields() { - final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); - final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); - - final Document document = new Document("field1", new BsonBoolean(true)) - .append("field2", new BsonInt32(1)) - .append("field3", new BsonInt64(2)) - .append("field4", new BsonDouble(3.0)) - .append("field5", new BsonDecimal128(new Decimal128(4))) - .append("field6", bsonTimestamp) - .append("field7", new BsonDateTime(bsonTimestamp.getValue())) - .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) - .append("field9", new BsonSymbol("test2")) - .append("field10", new BsonString("test3")) - .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) - .append("field12", new BsonJavaScript("code")) - .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) - .append("field14", new BsonRegularExpression("pattern")) - .append("field15", new BsonNull()) - .append("field16", new Document("key", "value")); - - final String documentAsJson = document.toJson(); - final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, Set.of("field1", "field2", "field3")); - - assertNotNull(transformed); - assertNotEquals(documentAsJson, Jsons.serialize(transformed)); - assertEquals(true, transformed.get("field1").asBoolean()); - assertEquals(1, transformed.get("field2").asInt()); - assertEquals(2, transformed.get("field3").asInt()); - assertFalse(transformed.has("field4")); - assertFalse(transformed.has("field5")); - assertFalse(transformed.has("field6")); - assertFalse(transformed.has("field7")); - assertFalse(transformed.has("field8")); - assertFalse(transformed.has("field9")); - assertFalse(transformed.has("field10")); - assertFalse(transformed.has("field11")); - assertFalse(transformed.has("field12")); - assertFalse(transformed.has("field13")); - assertFalse(transformed.has("field14")); - assertFalse(transformed.has("field15")); - assertFalse(transformed.has("field16")); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java deleted file mode 100644 index 91b5c2334aca..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals.mongodb; - -import static com.mongodb.assertions.Assertions.assertNotNull; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.mongodb.client.ChangeStreamIterable; -import com.mongodb.client.MongoChangeStreamCursor; -import com.mongodb.client.MongoClient; -import com.mongodb.client.model.changestream.ChangeStreamDocument; -import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.Jsons; -import io.debezium.connector.mongodb.ResumeTokens; -import io.debezium.engine.ChangeEvent; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.junit.jupiter.api.Test; - -class MongoDbCdcTargetPositionTest { - - private static final String OBJECT_ID = "64f24244f95155351c4185b1"; - private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; - private static final String OTHER_RESUME_TOKEN = "8264BEB9F3000000012B0229296E05"; - - @Test - void testCreateTargetPosition() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - assertNotNull(targetPosition); - assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); - } - - @Test - void testReachedTargetPosition() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - - when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\": \\d+,", "\"ts_ms\": 1590221043000,")); - final ChangeEventWithMetadata changeEventWithMetadata2 = new ChangeEventWithMetadata(changeEvent); - assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata2)); - } - - @Test - void testReachedTargetPositionSnapshotEvent() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - } - - @Test - void testReachedTargetPositionSnapshotLastEvent() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot_last.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - } - - @Test - void testReachedTargetPositionFromHeartbeat() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( - Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), - 0); - - assertTrue(targetPosition.reachedTargetPosition(heartbeatTimestamp)); - assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); - } - - @Test - void testIsHeartbeatSupported() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - - assertTrue(targetPosition.isHeartbeatSupported()); - } - - @Test - void testExtractPositionFromHeartbeatOffset() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final BsonTimestamp resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeTokenDocument); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - - final Map sourceOffset = Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), - MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, resumeTokenTimestamp.getInc(), - MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN, RESUME_TOKEN); - - final BsonTimestamp timestamp = targetPosition.extractPositionFromHeartbeatOffset(sourceOffset); - assertEquals(resumeTokenTimestamp, timestamp); - } - - @Test - void testIsEventAheadOfOffset() throws IOException { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final Map offset = - Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - final boolean result = targetPosition.isEventAheadOffset(offset, changeEventWithMetadata); - assertTrue(result); - } - - @Test - void testIsSameOffset() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final Map offsetA = - Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); - final Map offsetB = - Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); - final Map offsetC = - Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, OTHER_RESUME_TOKEN), new TypeReference<>() {}); - - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - - assertTrue(targetPosition.isSameOffset(offsetA, offsetA)); - assertTrue(targetPosition.isSameOffset(offsetA, offsetB)); - assertTrue(targetPosition.isSameOffset(offsetB, offsetA)); - assertFalse(targetPosition.isSameOffset(offsetA, offsetC)); - assertFalse(targetPosition.isSameOffset(offsetB, offsetC)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java deleted file mode 100644 index 62b9fecf1f07..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.debezium.internals.mongodb; - -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.mock; - -import com.mongodb.ConnectionString; -import io.debezium.connector.mongodb.MongoDbConnectorConfig; -import io.debezium.connector.mongodb.MongoDbOffsetContext; -import io.debezium.connector.mongodb.ReplicaSets; -import io.debezium.connector.mongodb.ResumeTokens; -import io.debezium.connector.mongodb.connection.ReplicaSet; -import java.util.HashMap; -import java.util.Map; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.junit.jupiter.api.Test; - -class MongoDbCustomLoaderTest { - - private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; - - @Test - void testLoadOffsets() { - final String replicaSet = "replica-set"; - final BsonDocument resumeToken = ResumeTokens.fromData(RESUME_TOKEN); - final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeToken); - final Map key = Map.of(KEY_REPLICA_SET, replicaSet); - final Map value = new HashMap<>(); - value.put(SOURCE_SECONDS, timestamp.getTime()); - value.put(SOURCE_ORDER, timestamp.getInc()); - value.put(SOURCE_RESUME_TOKEN, RESUME_TOKEN); - value.put(VALUE_TRANSACTION_ID, null); - final Map, Map> offsets = Map.of(key, value); - final MongoDbConnectorConfig mongoDbConnectorConfig = mock(MongoDbConnectorConfig.class); - final ReplicaSets replicaSets = ReplicaSets.of( - new ReplicaSet(new ConnectionString("mongodb://localhost:1234/?replicaSet=" + replicaSet))); - final MongoDbCustomLoader loader = new MongoDbCustomLoader(mongoDbConnectorConfig, replicaSets); - - final MongoDbOffsetContext context = loader.loadOffsets(offsets); - final Map offset = context.getReplicaSetOffsetContext(replicaSets.all().get(0)).getOffset(); - - assertNotNull(offset); - assertEquals(value, offset); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java index 15bca65f35a1..5d4dcb3e68d0 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java @@ -17,8 +17,6 @@ import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.cdk.testutils.TestDatabase; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import java.sql.JDBCType; @@ -59,9 +57,7 @@ protected JsonNode config() { @Override protected PostgresTestSource source() { - final var source = new PostgresTestSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new PostgresTestSource(); } @Override @@ -86,11 +82,6 @@ public JsonNode getConfigWithConnectionProperties(final PostgreSQLContainer p .build()); } - @Override - protected boolean supportsPerStream() { - return true; - } - @AfterAll static void cleanUp() { PSQL_CONTAINER.close(); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java index 5118e0c7b644..9e7bab7177f2 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java @@ -5,12 +5,12 @@ package io.airbyte.cdk.integrations.source.relationaldb; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.withSettings; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -34,13 +34,13 @@ public class AbstractDbSourceTest { @Test void testDeserializationOfLegacyState() throws IOException { - final AbstractDbSource dbSource = spy(AbstractDbSource.class); + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); final JsonNode config = mock(JsonNode.class); final String legacyStateJson = MoreResources.readResource("states/legacy.json"); final JsonNode legacyState = Jsons.deserialize(legacyStateJson); - final List result = StateGeneratorUtils.deserializeInitialState(legacyState, false, + final List result = StateGeneratorUtils.deserializeInitialState(legacyState, dbSource.getSupportedStateType(config)); assertEquals(1, result.size()); assertEquals(AirbyteStateType.LEGACY, result.get(0).getType()); @@ -48,40 +48,38 @@ void testDeserializationOfLegacyState() throws IOException { @Test void testDeserializationOfGlobalState() throws IOException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final AbstractDbSource dbSource = spy(AbstractDbSource.class); + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); final JsonNode config = mock(JsonNode.class); final String globalStateJson = MoreResources.readResource("states/global.json"); final JsonNode globalState = Jsons.deserialize(globalStateJson); final List result = - StateGeneratorUtils.deserializeInitialState(globalState, true, dbSource.getSupportedStateType(config)); + StateGeneratorUtils.deserializeInitialState(globalState, dbSource.getSupportedStateType(config)); assertEquals(1, result.size()); assertEquals(AirbyteStateType.GLOBAL, result.get(0).getType()); } @Test void testDeserializationOfStreamState() throws IOException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final AbstractDbSource dbSource = spy(AbstractDbSource.class); + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); final JsonNode config = mock(JsonNode.class); final String streamStateJson = MoreResources.readResource("states/per_stream.json"); final JsonNode streamState = Jsons.deserialize(streamStateJson); final List result = - StateGeneratorUtils.deserializeInitialState(streamState, true, dbSource.getSupportedStateType(config)); + StateGeneratorUtils.deserializeInitialState(streamState, dbSource.getSupportedStateType(config)); assertEquals(2, result.size()); assertEquals(AirbyteStateType.STREAM, result.get(0).getType()); } @Test void testDeserializationOfNullState() throws IOException { - final AbstractDbSource dbSource = spy(AbstractDbSource.class); + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); final JsonNode config = mock(JsonNode.class); - final List result = StateGeneratorUtils.deserializeInitialState(null, false, dbSource.getSupportedStateType(config)); + final List result = StateGeneratorUtils.deserializeInitialState(null, dbSource.getSupportedStateType(config)); assertEquals(1, result.size()); assertEquals(dbSource.getSupportedStateType(config), result.get(0).getType()); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java index 8e6448b78d89..d927faa3f502 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -6,12 +6,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; @@ -19,11 +19,16 @@ import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.sql.SQLException; import java.util.Collections; import java.util.Iterator; -import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -36,27 +41,22 @@ class StateDecoratingIteratorTest { private static final AirbyteStreamNameNamespacePair NAME_NAMESPACE_PAIR = new AirbyteStreamNameNamespacePair(STREAM_NAME, NAMESPACE); private static final String UUID_FIELD_NAME = "ascending_inventory_uuid"; - private static final AirbyteMessage EMPTY_STATE_MESSAGE = new AirbyteMessage().withType(Type.STATE); + private static final AirbyteMessage EMPTY_STATE_MESSAGE = createEmptyStateMessage(0.0); private static final String RECORD_VALUE_1 = "abc"; private static final AirbyteMessage RECORD_MESSAGE_1 = createRecordMessage(RECORD_VALUE_1); - private static final AirbyteMessage STATE_MESSAGE_1 = createStateMessage(RECORD_VALUE_1); private static final String RECORD_VALUE_2 = "def"; private static final AirbyteMessage RECORD_MESSAGE_2 = createRecordMessage(RECORD_VALUE_2); - private static final AirbyteMessage STATE_MESSAGE_2 = createStateMessage(RECORD_VALUE_2); private static final String RECORD_VALUE_3 = "ghi"; private static final AirbyteMessage RECORD_MESSAGE_3 = createRecordMessage(RECORD_VALUE_3); - private static final AirbyteMessage STATE_MESSAGE_3 = createStateMessage(RECORD_VALUE_3); private static final String RECORD_VALUE_4 = "jkl"; private static final AirbyteMessage RECORD_MESSAGE_4 = createRecordMessage(RECORD_VALUE_4); - private static final AirbyteMessage STATE_MESSAGE_4 = createStateMessage(RECORD_VALUE_4); private static final String RECORD_VALUE_5 = "xyz"; private static final AirbyteMessage RECORD_MESSAGE_5 = createRecordMessage(RECORD_VALUE_5); - private static final AirbyteMessage STATE_MESSAGE_5 = createStateMessage(RECORD_VALUE_5); private static AirbyteMessage createRecordMessage(final String recordValue) { return new AirbyteMessage() @@ -65,11 +65,43 @@ private static AirbyteMessage createRecordMessage(final String recordValue) { .withData(Jsons.jsonNode(ImmutableMap.of(UUID_FIELD_NAME, recordValue)))); } - private static AirbyteMessage createStateMessage(final String recordValue) { + private static AirbyteMessage createStateMessage(final String recordValue, final long cursorRecordCount, final double statsRecordCount) { + final DbStreamState dbStreamState = new DbStreamState() + .withCursorField(Collections.singletonList(UUID_FIELD_NAME)) + .withCursor(recordValue) + .withStreamName(STREAM_NAME) + .withStreamNamespace(NAMESPACE); + if (cursorRecordCount > 0) { + dbStreamState.withCursorRecordCount(cursorRecordCount); + } + final DbState dbState = new DbState().withCdc(false).withStreams(Collections.singletonList(dbStreamState)); return new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(ImmutableMap.of("cursor", recordValue)))); + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(dbStreamState))) + .withData(Jsons.jsonNode(dbState)) + .withSourceStats(new AirbyteStateStats().withRecordCount(statsRecordCount))); + } + + private static AirbyteMessage createEmptyStateMessage(final double statsRecordCount) { + final DbStreamState dbStreamState = new DbStreamState() + .withCursorField(Collections.singletonList(UUID_FIELD_NAME)) + .withStreamName(STREAM_NAME) + .withStreamNamespace(NAMESPACE); + + final DbState dbState = new DbState().withCdc(false).withStreams(Collections.singletonList(dbStreamState)); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(dbStreamState))) + .withData(Jsons.jsonNode(dbState)) + .withSourceStats(new AirbyteStateStats().withRecordCount(statsRecordCount))); } private Iterator createExceptionIterator() { @@ -103,15 +135,13 @@ public AirbyteMessage next() { @BeforeEach void setup() { - stateManager = mock(StateManager.class); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, null, 0)).thenReturn(EMPTY_STATE_MESSAGE.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_1, 1L)).thenReturn(STATE_MESSAGE_1.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2, 1L)).thenReturn(STATE_MESSAGE_2.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3, 1L)).thenReturn(STATE_MESSAGE_3.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_4, 1L)).thenReturn(STATE_MESSAGE_4.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5, 1L)).thenReturn(STATE_MESSAGE_5.getState()); - - when(stateManager.getCursorInfo(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); + final AirbyteStream airbyteStream = new AirbyteStream().withNamespace(NAMESPACE).withName(STREAM_NAME); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream() + .withStream(airbyteStream) + .withCursorField(Collections.singletonList(UUID_FIELD_NAME)); + + stateManager = new StreamStateManager(Collections.emptyList(), + new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(configuredAirbyteStream))); } @Test @@ -128,7 +158,7 @@ void testWithoutInitialCursor() { assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - assertEquals(STATE_MESSAGE_2, iterator.next()); + assertEquals(createStateMessage(RECORD_VALUE_2, 1, 2.0), iterator.next()); assertFalse(iterator.hasNext()); } @@ -136,7 +166,6 @@ void testWithoutInitialCursor() { void testWithInitialCursor() { // record 1 and 2 has smaller cursor value, so at the end, the initial cursor is emitted with 0 // record count - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5, 0L)).thenReturn(STATE_MESSAGE_5.getState()); messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2); final StateDecoratingIterator iterator = new StateDecoratingIterator( @@ -150,7 +179,7 @@ void testWithInitialCursor() { assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - assertEquals(STATE_MESSAGE_5, iterator.next()); + assertEquals(createStateMessage(RECORD_VALUE_5, 0, 2.0), iterator.next()); assertFalse(iterator.hasNext()); } @@ -171,19 +200,13 @@ void testCursorFieldIsEmpty() { assertEquals(recordMessage, iterator.next()); // null because no records with a cursor field were replicated for the stream. - assertNull(iterator.next().getState()); + assertEquals(createEmptyStateMessage(1.0), iterator.next()); assertFalse(iterator.hasNext()); } @Test void testIteratorCatchesExceptionWhenEmissionFrequencyNonZero() { final Iterator exceptionIterator = createExceptionIterator(); - - // The mock record count matches the number of records returned by the exception iterator. - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_1, 1L)).thenReturn(STATE_MESSAGE_1.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2, 2L)).thenReturn(STATE_MESSAGE_2.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3, 1L)).thenReturn(STATE_MESSAGE_3.getState()); - final StateDecoratingIterator iterator = new StateDecoratingIterator( exceptionIterator, stateManager, @@ -200,7 +223,7 @@ void testIteratorCatchesExceptionWhenEmissionFrequencyNonZero() { assertEquals(RECORD_MESSAGE_3, iterator.next()); // emits the first state message since the iterator has changed cursorFields (2 -> 3) and met the // frequency minimum of 1 record - assertEquals(STATE_MESSAGE_2, iterator.next()); + assertEquals(createStateMessage(RECORD_VALUE_2, 2, 4.0), iterator.next()); // no further records to read since Exception was caught above and marked iterator as endOfData() assertFalse(iterator.hasNext()); } @@ -248,8 +271,6 @@ void testUnicodeNull() { final AirbyteMessage recordMessageWithNull = createRecordMessage(recordValueWithNull); // UTF8 null \u0000 is removed from the cursor value in the state message - final AirbyteMessage stateMessageWithNull = STATE_MESSAGE_1; - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, recordValueWithNull, 1L)).thenReturn(stateMessageWithNull.getState()); messageIterator = MoreIterators.of(recordMessageWithNull); @@ -263,7 +284,7 @@ void testUnicodeNull() { 0); assertEquals(recordMessageWithNull, iterator.next()); - assertEquals(stateMessageWithNull, iterator.next()); + assertEquals(createStateMessage(RECORD_VALUE_1, 1, 1.0), iterator.next()); assertFalse(iterator.hasNext()); } @@ -285,16 +306,16 @@ void testStateEmissionFrequency1() { // records with the same cursor value, so no state is ready for emission assertEquals(RECORD_MESSAGE_2, iterator1.next()); // emit state 1 because it is the latest state ready for emission - assertEquals(STATE_MESSAGE_1, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_1, 1, 2.0), iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_2, 1, 1.0), iterator1.next()); assertEquals(RECORD_MESSAGE_4, iterator1.next()); - assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_3, 1, 1.0), iterator1.next()); assertEquals(RECORD_MESSAGE_5, iterator1.next()); // state 4 is not emitted because there is no more record and only // the final state should be emitted at this point; also the final // state should only be emitted once - assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_5, 1, 1.0), iterator1.next()); assertFalse(iterator1.hasNext()); } @@ -314,13 +335,13 @@ void testStateEmissionFrequency2() { assertEquals(RECORD_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_2, iterator1.next()); // emit state 1 because it is the latest state ready for emission - assertEquals(STATE_MESSAGE_1, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_1, 1, 2.0), iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); assertEquals(RECORD_MESSAGE_4, iterator1.next()); // emit state 3 because it is the latest state ready for emission - assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_3, 1, 2.0), iterator1.next()); assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_5, 1, 1.0), iterator1.next()); assertFalse(iterator1.hasNext()); } @@ -339,11 +360,11 @@ void testStateEmissionWhenInitialCursorIsNotNull() { assertEquals(RECORD_MESSAGE_2, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_2, 1, 2.0), iterator1.next()); assertEquals(RECORD_MESSAGE_4, iterator1.next()); - assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_3, 1, 1.0), iterator1.next()); assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_5, 1, 1.0), iterator1.next()); assertFalse(iterator1.hasNext()); } @@ -377,10 +398,6 @@ void testStateEmissionWhenInitialCursorIsNotNull() { @Test @DisplayName("When there are multiple records with the same cursor value") void testStateEmissionForRecordsSharingSameCursorValue() { - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2, 2L)).thenReturn(STATE_MESSAGE_2.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3, 3L)).thenReturn(STATE_MESSAGE_3.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_4, 1L)).thenReturn(STATE_MESSAGE_4.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5, 2L)).thenReturn(STATE_MESSAGE_5.getState()); messageIterator = MoreIterators.of( RECORD_MESSAGE_2, RECORD_MESSAGE_2, @@ -401,15 +418,50 @@ void testStateEmissionForRecordsSharingSameCursorValue() { assertEquals(RECORD_MESSAGE_3, iterator1.next()); // state 2 is the latest state ready for emission because // all records with the same cursor value have been emitted - assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_2, 2, 3.0), iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); assertEquals(RECORD_MESSAGE_4, iterator1.next()); - assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_3, 3, 3.0), iterator1.next()); assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_4, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_4, 1, 1.0), iterator1.next()); assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_5, 2, 1.0), iterator1.next()); + assertFalse(iterator1.hasNext()); + } + + @Test + void testStateEmissionForRecordsSharingSameCursorValueButDifferentStatsCount() { + messageIterator = MoreIterators.of( + RECORD_MESSAGE_2, RECORD_MESSAGE_2, + RECORD_MESSAGE_2, RECORD_MESSAGE_2, + RECORD_MESSAGE_3, RECORD_MESSAGE_3, RECORD_MESSAGE_3, + RECORD_MESSAGE_3, + RECORD_MESSAGE_3, RECORD_MESSAGE_3, RECORD_MESSAGE_3); + final StateDecoratingIterator iterator1 = new StateDecoratingIterator( + messageIterator, + stateManager, + NAME_NAMESPACE_PAIR, + UUID_FIELD_NAME, + RECORD_VALUE_1, + JsonSchemaPrimitive.STRING, + 10); + + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + // state 2 is the latest state ready for emission because + // all records with the same cursor value have been emitted + assertEquals(createStateMessage(RECORD_VALUE_2, 4, 10.0), iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(createStateMessage(RECORD_VALUE_3, 7, 1.0), iterator1.next()); assertFalse(iterator1.hasNext()); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java new file mode 100644 index 000000000000..34560be119d9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import java.util.Iterator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SourceStateIteratorTest { + + SourceStateIteratorManager mockProcessor; + Iterator messageIterator; + + SourceStateIterator sourceStateIterator; + + @BeforeEach + void setup() { + mockProcessor = mock(SourceStateIteratorManager.class); + messageIterator = mock(Iterator.class); + sourceStateIterator = new SourceStateIterator(messageIterator, mockProcessor); + } + + // Provides a way to generate a record message and will verify corresponding spied functions have + // been called. + void processRecordMessage() { + doReturn(true).when(messageIterator).hasNext(); + doReturn(false).when(mockProcessor).shouldEmitStateMessage(anyLong(), any()); + AirbyteMessage message = new AirbyteMessage().withType(Type.RECORD).withRecord(new AirbyteRecordMessage()); + doReturn(message).when(mockProcessor).processRecordMessage(any()); + doReturn(message).when(messageIterator).next(); + + assertEquals(message, sourceStateIterator.computeNext()); + verify(mockProcessor, atLeastOnce()).processRecordMessage(message); + verify(mockProcessor, atLeastOnce()).shouldEmitStateMessage(eq(0L), any()); + } + + @Test + void testShouldProcessRecordMessage() { + processRecordMessage(); + } + + @Test + void testShouldEmitStateMessage() { + processRecordMessage(); + doReturn(true).when(mockProcessor).shouldEmitStateMessage(anyLong(), any()); + final AirbyteStateMessage stateMessage = new AirbyteStateMessage(); + doReturn(stateMessage).when(mockProcessor).generateStateMessageAtCheckpoint(); + AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + expectedMessage.getState().withSourceStats(new AirbyteStateStats().withRecordCount(1.0)); + assertEquals(expectedMessage, sourceStateIterator.computeNext()); + } + + @Test + void testShouldEmitFinalStateMessage() { + processRecordMessage(); + processRecordMessage(); + doReturn(false).when(messageIterator).hasNext(); + final AirbyteStateMessage stateMessage = new AirbyteStateMessage(); + doReturn(stateMessage).when(mockProcessor).createFinalStateMessage(); + AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + expectedMessage.getState().withSourceStats(new AirbyteStateStats().withRecordCount(2.0)); + assertEquals(expectedMessage, sourceStateIterator.computeNext()); + } + + @Test + void testShouldSendEndOfData() { + processRecordMessage(); + doReturn(false).when(messageIterator).hasNext(); + doReturn(new AirbyteStateMessage()).when(mockProcessor).createFinalStateMessage(); + sourceStateIterator.computeNext(); + + // After sending the final state, if iterator was called again, we will return null. + assertEquals(null, sourceStateIterator.computeNext()); + } + + @Test + void testShouldRethrowExceptions() { + processRecordMessage(); + doThrow(new ArrayIndexOutOfBoundsException("unexpected error")).when(messageIterator).hasNext(); + assertThrows(RuntimeException.class, () -> sourceStateIterator.computeNext()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_3_mb.dat b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_1_mb.dat similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_3_mb.dat rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_1_mb.dat diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_3_mb.dat b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_1_mb.dat similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_3_mb.dat rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_1_mb.dat diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/spec.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/spec.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java index a0ee71a226d0..26544fe47fbd 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java @@ -219,6 +219,19 @@ protected void writeRecords( recordJson.get(modelCol).asText()); } + protected void deleteMessageOnIdCol(final String streamName, final String idCol, final int idValue) { + testdb.with("DELETE FROM %s.%s WHERE %s = %s", modelsSchema(), streamName, idCol, idValue); + } + + protected void deleteCommand(final String streamName) { + testdb.with("DELETE FROM %s.%s", modelsSchema(), streamName); + } + + protected void updateCommand(final String streamName, final String modelCol, final String modelVal, final String idCol, final int idValue) { + testdb.with("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", modelsSchema(), streamName, + modelCol, modelVal, COL_ID, 11); + } + static protected Set removeDuplicates(final Set messages) { final Set existingDataRecordsWithoutUpdated = new HashSet<>(); final Set output = new HashSet<>(); @@ -346,7 +359,7 @@ void testDelete() throws Exception { final List stateMessages1 = extractStateMessages(actualRecords1); assertExpectedStateMessages(stateMessages1); - testdb.with("DELETE FROM %s.%s WHERE %s = %s", modelsSchema(), MODELS_STREAM_NAME, COL_ID, 11); + deleteMessageOnIdCol(MODELS_STREAM_NAME, COL_ID, 11); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); final AutoCloseableIterator read2 = source() @@ -375,8 +388,7 @@ void testUpdate() throws Exception { final List stateMessages1 = extractStateMessages(actualRecords1); assertExpectedStateMessages(stateMessages1); - testdb.with("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", modelsSchema(), MODELS_STREAM_NAME, - COL_MODEL, updatedModel, COL_ID, 11); + updateCommand(MODELS_STREAM_NAME, COL_MODEL, updatedModel, COL_ID, 11); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); final AutoCloseableIterator read2 = source() @@ -536,8 +548,7 @@ void testCdcAndFullRefreshInSameSync() throws Exception { @DisplayName("When no records exist, no records are returned.") void testNoData() throws Exception { - testdb.with("DELETE FROM %s.%s", modelsSchema(), MODELS_STREAM_NAME); - + deleteCommand(MODELS_STREAM_NAME); final AutoCloseableIterator read = source().read(config(), getConfiguredCatalog(), null); final List actualRecords = AutoCloseableIterators.toListAndClose(read); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java index bc773832465c..836f6cf50347 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java @@ -26,7 +26,12 @@ public class DebugUtil { public static void debug(final Source debugSource) throws Exception { final JsonNode debugConfig = DebugUtil.getConfig(); final ConfiguredAirbyteCatalog configuredAirbyteCatalog = DebugUtil.getCatalog(); - final JsonNode state = DebugUtil.getState(); + JsonNode state; + try { + state = DebugUtil.getState(); + } catch (final Exception e) { + state = null; + } debugSource.check(debugConfig); debugSource.discover(debugConfig); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index fea6f5709024..aac25c5d87b0 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -36,6 +36,7 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -222,7 +223,7 @@ void testCheckSuccess() throws Exception { } @Test - void testCheckFailure() throws Exception { + protected void testCheckFailure() throws Exception { final var config = config(); maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); @@ -287,7 +288,7 @@ protected AirbyteCatalog filterOutOtherSchemas(final AirbyteCatalog catalog) { } @Test - void testDiscoverWithMultipleSchemas() throws Exception { + protected void testDiscoverWithMultipleSchemas() throws Exception { // clickhouse and mysql do not have a concept of schemas, so this test does not make sense for them. switch (testdb.getDatabaseDriver()) { case MYSQL, CLICKHOUSE, TERADATA: @@ -336,7 +337,7 @@ void testReadSuccess() throws Exception { } @Test - void testReadOneColumn() throws Exception { + protected void testReadOneColumn() throws Exception { final ConfiguredAirbyteCatalog catalog = CatalogHelpers .createConfiguredAirbyteCatalog(streamName(), getDefaultNamespace(), Field.of(COL_ID, JsonSchemaType.NUMBER)); final List actualMessages = MoreIterators @@ -364,7 +365,7 @@ protected List getAirbyteMessagesReadOneColumn() { } @Test - void testReadMultipleTables() throws Exception { + protected void testReadMultipleTables() throws Exception { final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithOneStream( getDefaultNamespace()); final List expectedMessages = new ArrayList<>(getTestMessages()); @@ -411,7 +412,7 @@ protected List getAirbyteMessagesSecondSync(final String streamN } @Test - void testTablesWithQuoting() throws Exception { + protected void testTablesWithQuoting() throws Exception { final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() @@ -542,7 +543,7 @@ void testIncrementalCursorChanges() throws Exception { } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + protected void testReadOneTableIncrementallyTwice() throws Exception { final var config = config(); final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); @@ -577,8 +578,8 @@ void testReadOneTableIncrementallyTwice() throws Exception { protected void executeStatementReadIncrementallyTwice() { testdb - .with("INSERT INTO %s(id, name, updated_at) VALUES (4, 'riker', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)) - .with("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)); + .with("INSERT INTO %s (id, name, updated_at) VALUES (4, 'riker', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)) + .with("INSERT INTO %s (id, name, updated_at) VALUES (5, 'data', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)); } protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { @@ -601,12 +602,12 @@ protected List getExpectedAirbyteMessagesSecondSync(final String .withCursorField(List.of(COL_ID)) .withCursor("5") .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2L)); return expectedMessages; } @Test - void testReadMultipleTablesIncrementally() throws Exception { + protected void testReadMultipleTablesIncrementally() throws Exception { final String tableName2 = TABLE_NAME + 2; final String streamName2 = streamName() + 2; final String fqTableName2 = getFullyQualifiedTableName(tableName2); @@ -671,9 +672,9 @@ void testReadMultipleTablesIncrementally() throws Exception { .withCursorRecordCount(1L)); final List expectedMessagesFirstSync = new ArrayList<>(getTestMessages()); - expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams1.get(0), expectedStateStreams1)); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams1.get(0), expectedStateStreams1, 3L)); expectedMessagesFirstSync.addAll(secondStreamExpectedMessages); - expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams2.get(1), expectedStateStreams2)); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams2.get(1), expectedStateStreams2, 3L)); setEmittedAtToNull(actualMessagesFirstSync); @@ -748,8 +749,8 @@ public void testIncrementalWithConcurrentInsertion() throws Exception { .filter(r -> r.getType() == Type.RECORD) .map(r -> r.getRecord().getData().get(COL_NAME).asText()) .toList(); - // teradata doesn't make insertion order guarantee when equal ordering value - if (testdb.getDatabaseDriver().equals(DatabaseDriver.TERADATA)) { + // some databases don't make insertion order guarantee when equal ordering value + if (testdb.getDatabaseDriver().equals(DatabaseDriver.TERADATA) || testdb.getDatabaseDriver().equals(DatabaseDriver.ORACLE)) { assertThat(List.of("a", "b"), Matchers.containsInAnyOrder(firstSyncNames.toArray())); } else { assertEquals(List.of("a", "b"), firstSyncNames); @@ -854,7 +855,7 @@ protected void incrementalCursorCheck( final List expectedStreams = List.of(buildStreamState(airbyteStream, cursorField, endCursorValue)); final List expectedMessages = new ArrayList<>(expectedRecordMessages); - expectedMessages.addAll(createExpectedTestMessages(expectedStreams)); + expectedMessages.addAll(createExpectedTestMessages(expectedStreams, expectedRecordMessages.size())); assertEquals(expectedMessages.size(), actualMessages.size()); assertTrue(expectedMessages.containsAll(actualMessages)); @@ -934,32 +935,28 @@ protected List getTestMessages() { COL_UPDATED_AT, "2006-10-19"))))); } - protected List createExpectedTestMessages(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s))) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))) - .collect( - Collectors.toList()) - : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))); + protected List createExpectedTestMessages(final List states, final long numRecords) { + return states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) numRecords)))) + .collect( + Collectors.toList()); } protected List createState(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s)))) - .collect( - Collectors.toList()) - : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(new DbState().withStreams(states)))); + return states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()); } protected ConfiguredAirbyteStream createTableWithSpaces() throws SQLException { @@ -1011,14 +1008,6 @@ protected void createSchemas() { } } - protected void dropSchemas() { - if (supportsSchemas()) { - for (final String schemaName : TEST_SCHEMAS) { - testdb.with(DROP_SCHEMA_QUERY, schemaName); - } - } - } - private JsonNode convertIdBasedOnDatabase(final int idValue) { return switch (testdb.getDatabaseDriver()) { case ORACLE, SNOWFLAKE -> Jsons.jsonNode(BigDecimal.valueOf(idValue)); @@ -1046,18 +1035,6 @@ protected static void setEmittedAtToNull(final Iterable messages } } - /** - * Tests whether the connector under test supports the per-stream state format or should use the - * legacy format for data generated by this test. - * - * @return {@code true} if the connector supports the per-stream state format or {@code false} if it - * does not support the per-stream state format (e.g. legacy format supported). Default - * value is {@code false}. - */ - protected boolean supportsPerStream() { - return false; - } - /** * Creates empty state with the provided stream name and namespace. * @@ -1066,69 +1043,37 @@ protected boolean supportsPerStream() { * @return {@link JsonNode} representation of the generated empty state. */ protected JsonNode createEmptyState(final String streamName, final String streamNamespace) { - if (supportsPerStream()) { - final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace))); - return Jsons.jsonNode(List.of(airbyteStateMessage)); - } else { - final DbState dbState = new DbState() - .withStreams(List.of(new DbStreamState().withStreamName(streamName).withStreamNamespace(streamNamespace))); - return Jsons.jsonNode(dbState); - } + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace))); + return Jsons.jsonNode(List.of(airbyteStateMessage)); + } protected JsonNode createState(final String streamName, final String streamNamespace, final JsonNode stateData) { - if (supportsPerStream()) { - final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream( - new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace)) - .withStreamState(stateData)); - return Jsons.jsonNode(List.of(airbyteStateMessage)); - } else { - final List cursorFields = MoreIterators.toList(stateData.get("cursor_field").elements()).stream().map(JsonNode::asText).toList(); - final DbState dbState = new DbState().withStreams(List.of( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(streamNamespace) - .withCursor(stateData.get("cursor").asText()) - .withCursorField(cursorFields) - .withCursorRecordCount(stateData.get("cursor_record_count").asLong()))); - return Jsons.jsonNode(dbState); - } + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream( + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace)) + .withStreamState(stateData)); + return Jsons.jsonNode(List.of(airbyteStateMessage)); } - /** - * Extracts the state component from the provided {@link AirbyteMessage} based on the value returned - * by {@link #supportsPerStream()}. - * - * @param airbyteMessage An {@link AirbyteMessage} that contains state. - * @return A {@link JsonNode} representation of the state contained in the {@link AirbyteMessage}. - */ protected JsonNode extractState(final AirbyteMessage airbyteMessage) { - if (supportsPerStream()) { - return Jsons.jsonNode(List.of(airbyteMessage.getState())); - } else { - return airbyteMessage.getState().getData(); - } + return Jsons.jsonNode(List.of(airbyteMessage.getState())); } - protected AirbyteMessage createStateMessage(final DbStreamState dbStreamState, final List legacyStates) { - if (supportsPerStream()) { - return new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(dbStreamState.getStreamNamespace()) - .withName(dbStreamState.getStreamName())) - .withStreamState(Jsons.jsonNode(dbStreamState))) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); - } else { - return new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); - } + protected AirbyteMessage createStateMessage(final DbStreamState dbStreamState, final List legacyStates, final long recordCount) { + return new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(dbStreamState.getStreamNamespace()) + .withName(dbStreamState.getStreamName())) + .withStreamState(Jsons.jsonNode(dbStreamState))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount))); } protected List extractSpecificFieldFromCombinedMessages(final List messages, diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java index 8b4ce9950bd7..9e77e0037d35 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -111,18 +112,6 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest { */ protected abstract JsonNode getState() throws Exception; - /** - * Tests whether the connector under test supports the per-stream state format or should use the - * legacy format for data generated by this test. - * - * @return {@code true} if the connector supports the per-stream state format or {@code false} if it - * does not support the per-stream state format (e.g. legacy format supported). Default - * value is {@code false}. - */ - protected boolean supportsPerStream() { - return false; - } - /** * Verify that a spec operation issued to the connector returns a valid spec. */ @@ -264,7 +253,20 @@ public void testIncrementalSyncWithState() throws Exception { // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); + JsonNode latestState = null; + for (final AirbyteStateMessage stateMessage : stateMessages) { + if (stateMessage.getType().equals(AirbyteStateMessage.AirbyteStateType.STREAM)) { + latestState = Jsons.jsonNode(stateMessages); + break; + } else if (stateMessage.getType().equals(AirbyteStateMessage.AirbyteStateType.GLOBAL)) { + latestState = Jsons.jsonNode(List.of(Iterables.getLast(stateMessages))); + break; + } else { + throw new RuntimeException("Unknown state type " + stateMessage.getType()); + } + } + + assert Objects.nonNull(latestState); final List secondSyncRecords = filterRecords(runRead(configuredCatalog, latestState)); assertTrue( secondSyncRecords.isEmpty(), diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java deleted file mode 100644 index 4735716dc05e..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.testutils; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.utility.DockerImageName; - -/** - * ContainerFactory is the companion interface to {@link TestDatabase} for providing it with - * suitable testcontainer instances. - */ -public interface ContainerFactory> { - - /** - * Creates a new, unshared testcontainer instance. This usually wraps the default constructor for - * the testcontainer type. - */ - C createNewContainer(DockerImageName imageName); - - /** - * Returns the class object of the testcontainer. - */ - Class getContainerClass(); - - /** - * Returns a shared instance of the testcontainer. - */ - default C shared(String imageName, String... methods) { - final String mapKey = Stream.concat( - Stream.of(imageName, this.getClass().getCanonicalName()), - Stream.of(methods)) - .collect(Collectors.joining("+")); - return Singleton.getOrCreate(mapKey, this); - } - - /** - * This class is exclusively used by {@link #shared(String, String...)}. It wraps a specific shared - * testcontainer instance, which is created exactly once. - */ - class Singleton> { - - static private final Logger LOGGER = LoggerFactory.getLogger(Singleton.class); - static private final ConcurrentHashMap> LAZY = new ConcurrentHashMap<>(); - - @SuppressWarnings("unchecked") - static private > C getOrCreate(String mapKey, ContainerFactory factory) { - final Singleton singleton = LAZY.computeIfAbsent(mapKey, Singleton::new); - return ((Singleton) singleton).getOrCreate(factory); - } - - final private String imageName; - final private List methodNames; - - private C sharedContainer; - private RuntimeException containerCreationError; - - private Singleton(String imageNamePlusMethods) { - final String[] parts = imageNamePlusMethods.split("\\+"); - this.imageName = parts[0]; - this.methodNames = Arrays.stream(parts).skip(2).toList(); - } - - private synchronized C getOrCreate(ContainerFactory factory) { - if (sharedContainer == null && containerCreationError == null) { - try { - create(imageName, factory, methodNames); - } catch (RuntimeException e) { - sharedContainer = null; - containerCreationError = e; - } - } - if (containerCreationError != null) { - throw new RuntimeException( - "Error during container creation for imageName=" + imageName - + ", factory=" + factory.getClass().getName() - + ", methods=" + methodNames, - containerCreationError); - } - return sharedContainer; - } - - private void create(String imageName, ContainerFactory factory, List methodNames) { - LOGGER.info("Creating new shared container based on {} with {}.", imageName, methodNames); - try { - final var parsed = DockerImageName.parse(imageName); - final var methods = new ArrayList(); - for (String methodName : methodNames) { - methods.add(factory.getClass().getMethod(methodName, factory.getContainerClass())); - } - sharedContainer = factory.createNewContainer(parsed); - sharedContainer.withLogConsumer(new Slf4jLogConsumer(LOGGER)); - for (Method method : methods) { - LOGGER.info("Calling {} in {} on new shared container based on {}.", - method.getName(), factory.getClass().getName(), imageName); - method.invoke(factory, sharedContainer); - } - sharedContainer.start(); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - throw new RuntimeException(e); - } - } - - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle new file mode 100644 index 000000000000..786d31082c79 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle @@ -0,0 +1,220 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + id "com.github.eirnym.js2p" version "1.0" + id "de.undercouch.download" version "5.4.0" + id "java-library" + id "org.openapi.generator" version "6.2.1" +} + +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-varargs,-try,-deprecation,-unchecked,-this-escape" + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-try" + } +} + +String specFile = "$projectDir/src/main/openapi/config.yaml" +String serverOutputDir = "$buildDir/generated/api/server" +String clientOutputDir = "$buildDir/generated/api/client" +String docsOutputDir = "$buildDir/generated/api/docs" +Map schemaMappingsValue = [ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', +] + +def generate = tasks.register('generate') + +def generateApiServer = tasks.register('generateApiServer', GenerateTask) { + + inputs.file specFile + outputs.dir serverOutputDir + + generatorName = "jaxrs-spec" + inputSpec = specFile + outputDir = serverOutputDir + + apiPackage = "io.airbyte.api.generated" + invokerPackage = "io.airbyte.api.invoker.generated" + modelPackage = "io.airbyte.api.model.generated" + + schemaMappings.set(schemaMappingsValue) + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true", + /* + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation is made to only keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + ]) +} +generate.configure { + dependsOn generateApiServer +} + +def generateApiClient = tasks.register('generateApiClient', GenerateTask) { + + inputs.file specFile + outputs.dir clientOutputDir + + generatorName = "java" + inputSpec = specFile + outputDir = clientOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings.set(schemaMappingsValue) + + library = "native" + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true" + ]) +} +generate.configure { + dependsOn generateApiClient +} + +def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { + + generatorName = "html" + inputSpec = specFile + outputDir = docsOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings.set(schemaMappingsValue) + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true" + ]) +} +def deleteExistingDocs = tasks.register('deleteOldApiDocs', Delete) { + delete rootProject.file("docs/reference/api/generated-api-html") +} +deleteExistingDocs.configure { + dependsOn generateApiDocs +} +def copyApiDocs = tasks.register('copyApiDocs', Copy) { + from(docsOutputDir) { + include "**/*.html" + } + into rootProject.file("docs/reference/api/generated-api-html") + includeEmptyDirs = false +} +copyApiDocs.configure { + dependsOn deleteExistingDocs +} +generate.configure { + dependsOn copyApiDocs +} + +dependencies { + api platform('com.fasterxml.jackson:jackson-bom:2.15.2') + api 'com.fasterxml.jackson.core:jackson-annotations' + api 'com.fasterxml.jackson.core:jackson-databind' + api 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + api 'com.google.guava:guava:33.0.0-jre' + api 'commons-io:commons-io:2.15.1' + api ('io.airbyte.airbyte-protocol:protocol-models:0.5.0') { exclude group: 'com.google.api-client', module: 'google-api-client' } + api 'javax.annotation:javax.annotation-api:1.3.2' + api 'org.apache.commons:commons-compress:1.25.0' + api 'org.apache.commons:commons-lang3:3.14.0' + api 'org.apache.logging.log4j:log4j-api:2.21.1' + api 'org.apache.logging.log4j:log4j-core:2.21.1' + api 'org.apache.logging.log4j:log4j-slf4j-impl:2.21.1' + api 'org.apache.logging.log4j:log4j-slf4j2-impl:2.21.1' + api 'org.slf4j:log4j-over-slf4j:2.0.11' + api 'org.slf4j:slf4j-api:2.0.11' + + implementation 'com.jayway.jsonpath:json-path:2.7.0' + implementation 'com.networknt:json-schema-validator:1.0.72' + implementation 'commons-cli:commons-cli:1.4' + implementation 'io.swagger:swagger-annotations:1.6.2' + implementation 'javax.validation:validation-api:2.0.1.Final' + implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' + implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' // needed so that we can follow $ref when parsing json + implementation 'org.openapitools:jackson-databind-nullable:0.2.1' + + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.2' + testFixturesImplementation 'org.apache.ant:ant:1.10.11' + + testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' +} + + +jsonSchema2Pojo { + sourceType = org.jsonschema2pojo.SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/types") + targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') + + targetPackage = 'io.airbyte.configoss' + useLongIntegers = true + + removeOldOutput = true + + generateBuilders = true + includeConstructors = false + includeSetters = true + serializable = true +} + +generate.configure { + dependsOn tasks.named('generateJsonSchema2Pojo') +} + + +sourceSets { + main { + java { + srcDirs([ + "$projectDir/src/main/java", + "${serverOutputDir}/src/gen/java", + "${clientOutputDir}/src/main/java", + ]) + } + resources { + srcDir "$projectDir/src/main/openapi/" + } + } +} + +tasks.named('compileJava').configure { + dependsOn generate +} +tasks.named('compileTestJava').configure { + dependsOn generate +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/AirbyteApiClient.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/AirbyteApiClient.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/PatchedLogsApi.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/PatchedLogsApi.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/cli/Clis.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/cli/Clis.java diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java new file mode 100644 index 000000000000..d7330332ca57 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.concurrency; + +import io.airbyte.commons.functional.Either; +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicInteger; + +public class CompletableFutures { + + /** + * Non-blocking implementation which does not use join. and returns an aggregated future. The order + * of results is preserved from the original list of futures. + * + * @param futures list of futures + * @param type of result + * @return a future that completes when all the input futures have completed + */ + public static CompletionStage>> allOf(final List> futures) { + CompletableFuture>> result = new CompletableFuture<>(); + final int size = futures.size(); + final AtomicInteger counter = new AtomicInteger(); + @SuppressWarnings("unchecked") + final Either[] results = (Either[]) Array.newInstance(Either.class, size); + // attach a whenComplete to all futures + for (int i = 0; i < size; i++) { + final int currentIndex = i; + futures.get(i).whenComplete((value, exception) -> { + // if exception is null, then the future completed successfully + // maybe synchronization is unnecessary here, but it's better to be safe + synchronized (results) { + if (exception == null) { + results[currentIndex] = Either.right(value); + } else { + if (exception instanceof Exception) { + results[currentIndex] = Either.left((Exception) exception); + } else { + // this should never happen + throw new RuntimeException("Unexpected exception in a future completion.", exception); + } + } + } + int completedCount = counter.incrementAndGet(); + if (completedCount == size) { + result.complete(Arrays.asList(results)); + } + }); + } + return result; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/enums/Enums.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/enums/Enums.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java new file mode 100644 index 000000000000..02d8f8311c2a --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.exceptions; + +import java.sql.SQLException; + +/** + * Wrapper unchecked exception for {@link SQLException}. This can be used in functional interfaces + * that do not allow checked exceptions without the generic RuntimeException. + */ +public class SQLRuntimeException extends RuntimeException { + + public SQLRuntimeException(final SQLException cause) { + super(cause); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index 06caa5bd6387..e2e86d1c2688 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -12,7 +12,6 @@ public class EnvVariableFeatureFlags implements FeatureFlags { private static final Logger log = LoggerFactory.getLogger(EnvVariableFeatureFlags.class); - public static final String USE_STREAM_CAPABLE_STATE = "USE_STREAM_CAPABLE_STATE"; public static final String AUTO_DETECT_SCHEMA = "AUTO_DETECT_SCHEMA"; // Set this value to true to see all messages from the source to destination, set to one second // emission @@ -24,11 +23,6 @@ public class EnvVariableFeatureFlags implements FeatureFlags { public static final String STRICT_COMPARISON_NORMALIZATION_TAG = "STRICT_COMPARISON_NORMALIZATION_TAG"; public static final String DEPLOYMENT_MODE = "DEPLOYMENT_MODE"; - @Override - public boolean useStreamCapableState() { - return getEnvOrDefault(USE_STREAM_CAPABLE_STATE, false, Boolean::parseBoolean); - } - @Override public boolean autoDetectSchema() { return getEnvOrDefault(AUTO_DETECT_SCHEMA, true, Boolean::parseBoolean); diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlags.java similarity index 97% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlags.java index cf35d83a4ff6..fa55fbd9484c 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlags.java @@ -10,8 +10,6 @@ */ public interface FeatureFlags { - boolean useStreamCapableState(); - boolean autoDetectSchema(); boolean logConnectorMessages(); diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java new file mode 100644 index 000000000000..17cdfa91dcbf --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.features; + +public class FeatureFlagsWrapper implements FeatureFlags { + + /** + * Overrides the {@link FeatureFlags#deploymentMode} method in the feature flags. + */ + static public FeatureFlags overridingDeploymentMode( + final FeatureFlags wrapped, + final String deploymentMode) { + return new FeatureFlagsWrapper(wrapped) { + + @Override + public String deploymentMode() { + return deploymentMode; + } + + }; + } + + private final FeatureFlags wrapped; + + public FeatureFlagsWrapper(FeatureFlags wrapped) { + this.wrapped = wrapped; + } + + @Override + public boolean autoDetectSchema() { + return wrapped.autoDetectSchema(); + } + + @Override + public boolean logConnectorMessages() { + return wrapped.logConnectorMessages(); + } + + @Override + public boolean concurrentSourceStreamRead() { + return wrapped.concurrentSourceStreamRead(); + } + + @Override + public boolean applyFieldSelection() { + return wrapped.applyFieldSelection(); + } + + @Override + public String fieldSelectionWorkspaces() { + return wrapped.fieldSelectionWorkspaces(); + } + + @Override + public String strictComparisonNormalizationWorkspaces() { + return wrapped.strictComparisonNormalizationWorkspaces(); + } + + @Override + public String strictComparisonNormalizationTag() { + return wrapped.strictComparisonNormalizationTag(); + } + + @Override + public String deploymentMode() { + return wrapped.deploymentMode(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedFunction.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedFunction.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/Either.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/Either.java new file mode 100644 index 000000000000..187b109e42f2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/Either.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.functional; + +import java.util.Objects; + +/** + * A class that represents a value of one of two possible types (a disjoint union). An instance of + * Either is an instance of Left or Right. + * + * A common use of Either is for error handling in functional programming. By convention, Left is + * failure and Right is success. + * + * @param the type of the left value + * @param the type of the right value + */ +public class Either { + + private final Error left; + private final Result right; + + private Either(Error left, Result right) { + this.left = left; + this.right = right; + } + + public boolean isLeft() { + return left != null; + } + + public boolean isRight() { + return right != null; + } + + public Error getLeft() { + return left; + } + + public Result getRight() { + return right; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Either either = (Either) o; + return Objects.equals(left, either.left) && Objects.equals(right, either.right); + } + + @Override + public int hashCode() { + return Objects.hash(left, right); + } + + public static Either left(Error error) { + return new Either<>(error, null); + } + + public static Either right(Result result) { + return new Either<>(null, result); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/IOs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/IOs.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/LineGobbler.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/LineGobbler.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/jackson/MoreMappers.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/jackson/MoreMappers.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonPaths.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonPaths.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonSchemas.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonSchemas.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/Jsons.java similarity index 87% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/Jsons.java index 2d0420bf63f0..e6475509b5aa 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/Jsons.java @@ -36,10 +36,14 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressWarnings({"PMD.AvoidReassigningParameters", "PMD.AvoidCatchingThrowable"}) public class Jsons { + private static final Logger LOGGER = LoggerFactory.getLogger(Jsons.class); + // Object Mapper is thread-safe private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper(); // sort of a hotfix; I don't know how bad the performance hit is so not turning this on by default @@ -117,7 +121,7 @@ public static Optional tryDeserialize(final String jsonString, final Clas try { return Optional.of(OBJECT_MAPPER.readValue(jsonString, klass)); } catch (final Throwable e) { - return Optional.empty(); + return handleDeserThrowable(e); } } @@ -125,11 +129,26 @@ public static Optional tryDeserializeExact(final String jsonString, final try { return Optional.of(OBJECT_MAPPER_EXACT.readValue(jsonString, klass)); } catch (final Throwable e) { - return Optional.empty(); + return handleDeserThrowable(e); } } public static Optional tryDeserialize(final String jsonString) { + try { + return Optional.of(OBJECT_MAPPER.readTree(jsonString)); + } catch (final Throwable e) { + return handleDeserThrowable(e); + } + } + + /** + * This method does not generate deserialization warn log on why serialization failed. See also + * {@link #tryDeserialize(String)}. + * + * @param jsonString + * @return + */ + public static Optional tryDeserializeWithoutWarn(final String jsonString) { try { return Optional.of(OBJECT_MAPPER.readTree(jsonString)); } catch (final Throwable e) { @@ -377,4 +396,33 @@ public DefaultPrettyPrinter withSeparators(final Separators separators) { } + /** + * Simple utility method to log a semi-useful message when deserialization fails. Intentionally + * don't log the actual exception object, because it probably contains some/all of the inputString + * (e.g. `[Source: (String)"{"foo": "bar"; line: 1, column: 13]`). Logging the class name + * can at least help narrow down the problem, without leaking potentially-sensitive information. + */ + private static Optional handleDeserThrowable(Throwable t) { + // Manually build the stacktrace, excluding the top-level exception object + // so that we don't accidentally include the exception message. + // Otherwise we could just do ExceptionUtils.getStackTrace(t). + final StringBuilder sb = new StringBuilder(); + sb.append(t.getClass()); + for (final StackTraceElement traceElement : t.getStackTrace()) { + sb.append("\n\tat "); + sb.append(traceElement.toString()); + } + while (t.getCause() != null) { + t = t.getCause(); + sb.append("\nCaused by "); + sb.append(t.getClass()); + for (final StackTraceElement traceElement : t.getStackTrace()) { + sb.append("\n\tat "); + sb.append(traceElement.toString()); + } + } + LOGGER.warn("Failed to deserialize json due to {}", sb); + return Optional.empty(); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableQueue.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableQueue.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/Exceptions.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/Exceptions.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/MoreBooleans.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/MoreBooleans.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/LoggingHelper.java similarity index 84% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/LoggingHelper.java index 8250449bc77f..d9cca1b78089 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/LoggingHelper.java @@ -22,6 +22,7 @@ public enum Color { YELLOW_BACKGROUND("\u001b[43m"), // destination GREEN_BACKGROUND("\u001b[42m"), // normalization CYAN_BACKGROUND("\u001b[46m"), // container runner + RED_BACKGROUND("\u001b[41m"), // testcontainers PURPLE_BACKGROUND("\u001b[45m"); // dbt private final String ansi; @@ -40,9 +41,10 @@ public String getCode() { @VisibleForTesting public static final String RESET = "\u001B[0m"; + public static final String PREPARE_COLOR_CHAR = "\u001b[m"; public static String applyColor(final Color color, final String msg) { - return color.getCode() + msg + RESET; + return PREPARE_COLOR_CHAR + color.getCode() + msg + PREPARE_COLOR_CHAR + RESET; } } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java similarity index 86% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java index 21264d559ef3..5f49f00b1251 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java @@ -8,6 +8,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.function.BiConsumer; import org.slf4j.MDC; /** @@ -28,7 +29,7 @@ */ public class MdcScope implements AutoCloseable { - public final static MdcScope.Builder DEFAULT_BUILDER = new Builder(); + public final static Builder DEFAULT_BUILDER = new Builder(); private final Map originalContextMap; @@ -70,22 +71,24 @@ public Builder setSimple(final boolean simple) { return this; } - public MdcScope build() { - final Map extraMdcEntries = new HashMap<>(); - + public void produceMappings(final BiConsumer mdcConsumer) { maybeLogPrefix.ifPresent(logPrefix -> { final String potentiallyColoredLog = maybePrefixColor .map(color -> LoggingHelper.applyColor(color, logPrefix)) .orElse(logPrefix); - extraMdcEntries.put(LoggingHelper.LOG_SOURCE_MDC_KEY, potentiallyColoredLog); + mdcConsumer.accept(LoggingHelper.LOG_SOURCE_MDC_KEY, potentiallyColoredLog); if (simple) { // outputs much less information for this line. see log4j2.xml to see exactly what this does - extraMdcEntries.put("simple", "true"); + mdcConsumer.accept("simple", "true"); } }); + } + public MdcScope build() { + final Map extraMdcEntries = new HashMap<>(); + produceMappings(extraMdcEntries::put); return new MdcScope(extraMdcEntries); } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/map/MoreMaps.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/map/MoreMaps.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/resources/MoreResources.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/resources/MoreResources.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/MoreStreams.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/MoreStreams.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/string/Strings.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/string/Strings.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Names.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Names.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Sqls.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Sqls.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/CompositeIterator.java similarity index 84% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/CompositeIterator.java index 7c5997d344c6..c8a3030bb92d 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/CompositeIterator.java @@ -10,9 +10,11 @@ import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +47,7 @@ public final class CompositeIterator extends AbstractIterator implements A private final List> iterators; private int i; - private boolean firstRead; + private final Set> seenIterators; private boolean hasClosed; CompositeIterator(final List> iterators, final Consumer airbyteStreamStatusConsumer) { @@ -54,7 +56,7 @@ public final class CompositeIterator extends AbstractIterator implements A this.airbyteStreamStatusConsumer = Optional.ofNullable(airbyteStreamStatusConsumer); this.iterators = iterators; this.i = 0; - this.firstRead = true; + this.seenIterators = new HashSet>(); this.hasClosed = false; } @@ -72,6 +74,7 @@ protected T computeNext() { while (!currentIterator().hasNext()) { try { currentIterator().close(); + emitStartStreamStatus(currentIterator().getAirbyteStream()); StreamStatusUtils.emitCompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } catch (final Exception e) { StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); @@ -80,26 +83,21 @@ protected T computeNext() { if (i + 1 < iterators.size()) { i++; - StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); - firstRead = true; } else { return endOfData(); } } try { - if (isFirstStream()) { - StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); + final boolean isFirstRun = emitStartStreamStatus(currentIterator().getAirbyteStream()); + final T next = currentIterator().next(); + if (isFirstRun) { + StreamStatusUtils.emitRunningStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } - return currentIterator().next(); + return next; } catch (final RuntimeException e) { StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); throw e; - } finally { - if (firstRead) { - StreamStatusUtils.emitRunningStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); - firstRead = false; - } } } @@ -107,8 +105,13 @@ private AutoCloseableIterator currentIterator() { return iterators.get(i); } - private boolean isFirstStream() { - return i == 0 && firstRead; + private boolean emitStartStreamStatus(final Optional airbyteStream) { + if (airbyteStream.isPresent() && !seenIterators.contains(airbyteStream)) { + seenIterators.add(airbyteStream); + StreamStatusUtils.emitStartStreamStatus(airbyteStream, airbyteStreamStatusConsumer); + return true; + } + return false; } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/MoreIterators.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/MoreIterators.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteVersion.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteVersion.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/Version.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/Version.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionDeserializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionDeserializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/yaml/Yamls.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/yaml/Yamls.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/ConfigSchema.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/ConfigSchema.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/Configs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/Configs.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/EnvConfigs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/EnvConfigs.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java similarity index 82% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java index 04b54e013eb0..22274b2dadff 100644 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Iterables; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.State; import io.airbyte.configoss.StateType; @@ -29,7 +28,7 @@ public static class AirbyteStateMessageListTypeReference extends TypeReference getTypedState(final JsonNode state, final boolean useStreamCapableState) { + public static Optional getTypedState(final JsonNode state) { if (state == null) { return Optional.empty(); } else { @@ -49,10 +48,10 @@ public static Optional getTypedState(final JsonNode state, final b } else { switch (stateMessages.get(0).getType()) { case GLOBAL -> { - return Optional.of(provideGlobalState(stateMessages.get(0), useStreamCapableState)); + return Optional.of(provideGlobalState(stateMessages.get(0))); } case STREAM -> { - return Optional.of(provideStreamState(stateMessages, useStreamCapableState)); + return Optional.of(provideStreamState(stateMessages)); } case LEGACY -> { return Optional.of(getLegacyStateWrapper(stateMessages.get(0).getData())); @@ -65,7 +64,7 @@ public static Optional getTypedState(final JsonNode state, final b } } else { if (stateMessages.stream().allMatch(stateMessage -> stateMessage.getType() == AirbyteStateType.STREAM)) { - return Optional.of(provideStreamState(stateMessages, useStreamCapableState)); + return Optional.of(provideStreamState(stateMessages)); } if (stateMessages.stream().allMatch(stateMessage -> stateMessage.getType() == null)) { return Optional.of(getLegacyStateWrapper(state)); @@ -104,16 +103,10 @@ public static Boolean isMigration(final StateType currentStateType, final @Nulla return previousStateType == StateType.LEGACY && currentStateType != StateType.LEGACY; } - private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMessages, final boolean useStreamCapableState) { - if (useStreamCapableState) { - return new StateWrapper() - .withStateType(StateType.GLOBAL) - .withGlobal(stateMessages); - } else { - return new StateWrapper() - .withStateType(StateType.LEGACY) - .withLegacyState(stateMessages.getData()); - } + private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMessages) { + return new StateWrapper() + .withStateType(StateType.GLOBAL) + .withGlobal(stateMessages); } /** @@ -123,16 +116,11 @@ private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMe * @param useStreamCapableState - a flag that indicates whether to return the new format * @return a wrapped state */ - private static StateWrapper provideStreamState(final List stateMessages, final boolean useStreamCapableState) { - if (useStreamCapableState) { - return new StateWrapper() - .withStateType(StateType.STREAM) - .withStateMessages(stateMessages); - } else { - return new StateWrapper() - .withStateType(StateType.LEGACY) - .withLegacyState(Iterables.getLast(stateMessages).getData()); - } + private static StateWrapper provideStreamState(final List stateMessages) { + return new StateWrapper() + .withStateType(StateType.STREAM) + .withStateMessages(stateMessages); + } private static StateWrapper getLegacyStateWrapper(final JsonNode state) { diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonValidationException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonValidationException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/openapi/config.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/openapi/config.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml new file mode 100644 index 000000000000..9c9e72c2c33f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml @@ -0,0 +1,88 @@ + + + + + %d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%t`%T`%highlight{%p}`%C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n + + %d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%replace{%X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n + + ${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}} + build/test-logs/${date:yyyy-MM-dd'T'HH:mm:ss} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorDefinitionResourceRequirements.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorDefinitionResourceRequirements.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/AllowedHosts.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/AllowedHosts.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/CombinedConnectorCatalog.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/CombinedConnectorCatalog.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ConnectorJobOutput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ConnectorJobOutput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DataType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DataType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationConnection.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationConnection.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationOAuthParameter.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationOAuthParameter.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationSyncMode.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationSyncMode.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DockerImageSpec.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DockerImageSpec.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/FailureReason.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/FailureReason.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobGetSpecConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobGetSpecConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobSyncConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobSyncConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NamespaceDefinitionType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NamespaceDefinitionType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NotificationType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NotificationType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorDbt.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorDbt.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorNormalization.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorNormalization.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorWebhook.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorWebhook.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ReplicationStatus.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ReplicationStatus.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ResourceRequirements.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ResourceRequirements.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceConnection.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceConnection.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceOAuthParameter.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceOAuthParameter.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionOutput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionOutput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDestinationDefinition.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDestinationDefinition.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDiscoverCatalogInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDiscoverCatalogInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSourceDefinition.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSourceDefinition.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncOperation.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncOperation.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/State.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/State.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateWrapper.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateWrapper.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SuggestedStreams.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SuggestedStreams.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncMode.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncMode.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncStats.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncStats.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationConfigs.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationConfigs.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationSummary.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationSummary.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerDestinationConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerDestinationConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerSourceConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerSourceConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/cli/ClisTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/cli/ClisTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/CompletableFuturesTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/CompletableFuturesTest.java new file mode 100644 index 000000000000..def67f8e5916 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/CompletableFuturesTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.concurrency; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.commons.functional.Either; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import org.junit.jupiter.api.Test; + +class CompletableFuturesTest { + + @Test + public void testAllOf() { + // Complete in random order + final List> futures = Arrays.asList( + returnSuccessWithDelay(1, 2000), + returnSuccessWithDelay(2, 200), + returnSuccessWithDelay(3, 500), + returnSuccessWithDelay(4, 100), + returnFailureWithDelay("Fail 5", 2000), + returnFailureWithDelay("Fail 6", 300)); + + final CompletableFuture>> allOfResult = CompletableFutures.allOf(futures).toCompletableFuture(); + final List> result = allOfResult.join(); + List> success = result.stream().filter(Either::isRight).toList(); + assertEquals(success, Arrays.asList( + Either.right(1), + Either.right(2), + Either.right(3), + Either.right(4))); + // Extract wrapped CompletionException messages. + final List failureMessages = result.stream().filter(Either::isLeft).map(either -> either.getLeft().getCause().getMessage()).toList(); + assertEquals(failureMessages, Arrays.asList("Fail 5", "Fail 6")); + } + + private CompletableFuture returnSuccessWithDelay(final int value, final long delayMs) { + return CompletableFuture.supplyAsync(() -> { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return value; + }); + } + + private CompletableFuture returnFailureWithDelay(final String message, final long delayMs) { + return CompletableFuture.supplyAsync(() -> { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException(message); + }); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/enums/EnumsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/enums/EnumsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/IOsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/IOsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/LineGobblerTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/LineGobblerTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonPathsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonPathsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/map/MoreMapsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/map/MoreMapsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/string/StringsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/string/StringsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/NamesTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/NamesTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/SqlsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/SqlsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/VersionTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/VersionTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/yaml/YamlsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/yaml/YamlsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java similarity index 99% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java index ca6c193ea348..3986e6e5ac36 100644 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java @@ -83,8 +83,6 @@ void testStartsWait() throws InterruptedException { while (recordedBeats.get() < 3) { Thread.sleep(10); } - - thread.stop(); } @Test diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/composite_json_schema.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/composite_json_schema.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_all_types.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_all_types.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test_a similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test_a diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_a similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_a diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub_2 similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub_2 diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-with-version.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-with-version.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-spec-message.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-spec-message.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-version.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-version.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarnessUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarnessUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/WorkerConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/WorkerConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/exception/TestHarnessException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/exception/TestHarnessException.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/CheckConnectionTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/CheckConnectionTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DbtTransformationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DbtTransformationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/GetSpecTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/GetSpecTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/CatalogClientConverters.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/CatalogClientConverters.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/EntrypointEnvChecker.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/EntrypointEnvChecker.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/FailureHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/FailureHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteDestination.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteDestination.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteSource.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteSource.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteStreamFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteStreamFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteSource.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteSource.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java index e0b16aaf6aa5..2badef87b11a 100644 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java @@ -15,9 +15,9 @@ import java.nio.charset.StandardCharsets; import java.text.CharacterIterator; import java.text.StringCharacterIterator; +import java.time.Instant; import java.util.Optional; import java.util.stream.Stream; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +91,7 @@ public Stream create(final BufferedReader bufferedReader) { try { final String errorMessage = String.format( "Airbyte has received a message at %s UTC which is larger than %s (size: %s). The sync has been failed to prevent running out of memory.", - DateTime.now(), + Instant.now(), humanReadableByteCountSI(maxMemory), humanReadableByteCountSI(messageSize)); throw exceptionClass.get().getConstructor(String.class).newInstance(errorMessage); @@ -108,7 +108,7 @@ public Stream create(final BufferedReader bufferedReader) { } protected Stream parseJson(final String line) { - final Optional jsonLine = Jsons.tryDeserialize(line); + final Optional jsonLine = Jsons.tryDeserializeWithoutWarn(line); if (jsonLine.isEmpty()) { // we log as info all the lines that are not valid json // some sources actually log their process on stdout, we diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/HeartbeatMonitor.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/HeartbeatMonitor.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java index ef176060d43b..b8592a8e6c19 100644 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java @@ -216,7 +216,6 @@ private Map getWorkerMetadata() { .put("WORKER_CONNECTOR_IMAGE", imageName) .put("WORKER_JOB_ID", jobId) .put("WORKER_JOB_ATTEMPT", String.valueOf(attempt)) - .put(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(featureFlags.useStreamCapableState())) .put(EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, String.valueOf(featureFlags.autoDetectSchema())) .put(EnvVariableFeatureFlags.APPLY_FIELD_SELECTION, String.valueOf(featureFlags.applyFieldSelection())) .put(EnvVariableFeatureFlags.FIELD_SELECTION_WORKSPACES, featureFlags.fieldSelectionWorkspaces()) diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/DockerProcessFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/DockerProcessFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/IntegrationLauncher.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/IntegrationLauncher.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/Metadata.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/Metadata.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/ProcessFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/ProcessFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/TestConfigHelpers.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/TestConfigHelpers.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/dbt_transformation_entrypoint.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/dbt_transformation_entrypoint.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/check.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/check.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/init.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/init.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/main.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/main.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/image_exists.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/image_exists.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/sshtunneling.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/sshtunneling.sh diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle new file mode 100644 index 000000000000..2f86620d7ec6 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle @@ -0,0 +1,20 @@ +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-deprecation" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-deprecation" + } +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + api project(':airbyte-cdk:java:airbyte-cdk:s3-destinations') + + api 'com.google.cloud:google-cloud-storage:2.32.1' + + testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:s3-destinations') + testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:s3-destinations')) +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java new file mode 100644 index 000000000000..4dedbba45ab0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3ConsumerFactory; +import io.airbyte.cdk.integrations.destination.s3.SerializedBufferFactory; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseGcsDestination extends BaseConnector implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseGcsDestination.class); + public static final String EXPECTED_ROLES = "storage.multipartUploads.abort, storage.multipartUploads.create, " + + "storage.objects.create, storage.objects.delete, storage.objects.get, storage.objects.list"; + + private final NamingConventionTransformer nameTransformer; + + public BaseGcsDestination() { + this.nameTransformer = new GcsNameTransformer(); + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) { + try { + final GcsDestinationConfig destinationConfig = GcsDestinationConfig.getGcsDestinationConfig(config); + final AmazonS3 s3Client = destinationConfig.getS3Client(); + + // Test single upload (for small files) permissions + S3BaseChecks.testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); + + // Test multipart upload with stream transfer manager + S3BaseChecks.testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); + + return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + } catch (final AmazonS3Exception e) { + LOGGER.error("Exception attempting to access the Gcs bucket", e); + final String message = getErrorMessage(e.getErrorCode(), 0, e.getMessage(), e); + AirbyteTraceMessageUtility.emitConfigErrorTrace(e, message); + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage(message); + } catch (final Exception e) { + LOGGER.error("Exception attempting to access the Gcs bucket: {}. Please make sure you account has all of these roles: " + EXPECTED_ROLES, e); + AirbyteTraceMessageUtility.emitConfigErrorTrace(e, e.getMessage()); + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage("Could not connect to the Gcs bucket with the provided configuration. \n" + e + .getMessage()); + } + } + + @Override + public AirbyteMessageConsumer getConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog configuredCatalog, + final Consumer outputRecordCollector) { + final GcsDestinationConfig gcsConfig = GcsDestinationConfig.getGcsDestinationConfig(config); + return new S3ConsumerFactory().create( + outputRecordCollector, + new GcsStorageOperations(nameTransformer, gcsConfig.getS3Client(), gcsConfig), + nameTransformer, + SerializedBufferFactory.getCreateFunction(gcsConfig, FileBuffer::new), + gcsConfig, + configuredCatalog); + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java similarity index 90% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java index 23a72598e1cd..2f7865057584 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -10,14 +10,14 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfigs; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfigs; import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfigs; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; /** * Currently we always reuse the S3 client for GCS. So the GCS config extends from the S3 config. diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java similarity index 79% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java index af2146bd742b..c2cce517070c 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java index 453b4d60e7df..7233ac4fdb69 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java index e63b436a732f..bd3b5a1813de 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -10,15 +10,15 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.GcsUtils; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.GcsUtils; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; @@ -26,7 +26,6 @@ import java.util.UUID; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumWriter; import org.slf4j.Logger; @@ -40,7 +39,7 @@ public class GcsAvroWriter extends BaseGcsWriter implements DestinationFileWrite private final AvroRecordFactory avroRecordFactory; private final StreamTransferManager uploadManager; private final MultiPartOutputStream outputStream; - private final DataFileWriter dataFileWriter; + private final DataFileWriter dataFileWriter; private final String gcsFileLocation; private final String objectKey; @@ -98,7 +97,7 @@ public void write(final UUID id, final AirbyteRecordMessage recordMessage) throw @Override public void write(final JsonNode formattedData) throws IOException { - final GenericData.Record record = avroRecordFactory.getAvroRecord(formattedData); + final Record record = avroRecordFactory.getAvroRecord(formattedData); dataFileWriter.append(record); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java similarity index 86% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java index f8465486a9f8..0166337f33c6 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import io.airbyte.cdk.integrations.destination.s3.credential.BlobStorageCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialConfig; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java index 8ce44901743a..9241a6af94f2 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java new file mode 100644 index 000000000000..a44f77241e39 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs.credential; + +public enum GcsCredentialType { + HMAC_KEY +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java similarity index 95% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java index e1521ad34bf0..18bc1da6df61 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java similarity index 95% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java index 1e76838b8a85..097c457e7d2d 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.csv; +package io.airbyte.cdk.integrations.destination.gcs.csv; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -10,13 +10,13 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator; import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java similarity index 94% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java index 1d56bd357d1d..23113125f0f5 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.jsonl; +package io.airbyte.cdk.integrations.destination.gcs.jsonl; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; @@ -11,13 +11,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java similarity index 90% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java index b77aec84d515..022355661772 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java @@ -2,18 +2,19 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.parquet; +package io.airbyte.cdk.integrations.destination.gcs.parquet; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.GcsS3FileSystem; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; @@ -22,7 +23,6 @@ import java.sql.Timestamp; import java.util.UUID; import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -64,7 +64,7 @@ public GcsParquetWriter(final GcsDestinationConfig config, final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); final Configuration hadoopConfig = getHadoopConfig(config); - this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) + this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) .withSchema(schema) .withCompressionCodec(formatConfig.getCompressionCodec()) .withRowGroupSize(formatConfig.getBlockSize()) @@ -81,7 +81,7 @@ public static Configuration getHadoopConfig(final GcsDestinationConfig config) { final Configuration hadoopConfig = new Configuration(); // the default org.apache.hadoop.fs.s3a.S3AFileSystem does not work for GCS - hadoopConfig.set("fs.s3a.impl", "io.airbyte.integrations.destination.gcs.util.GcsS3FileSystem"); + hadoopConfig.set("fs.s3a.impl", GcsS3FileSystem.class.getCanonicalName()); // https://stackoverflow.com/questions/64141204/process-data-in-google-storage-on-an-aws-emr-cluster-in-spark hadoopConfig.set("fs.s3a.access.key", hmacKeyCredential.getHmacKeyAccessId()); diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java similarity index 90% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java index 1fa384557fc4..eb0978cb2c06 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import java.io.IOException; import org.apache.hadoop.fs.s3a.Retries; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java index 6c305c177ec7..1bb7606d096e 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java index 225f85851fd7..fdac8772e48c 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.writer; +package io.airbyte.cdk.integrations.destination.gcs.writer; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.HeadBucketRequest; import com.amazonaws.services.s3.model.S3ObjectSummary; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.util.S3OutputPathHelper; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java similarity index 88% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java index bbecfdd52e43..d4252764384e 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import java.io.IOException; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java similarity index 96% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java index ae91d60910be..a535a4679c56 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -11,12 +11,12 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; import java.util.List; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileConstants; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java similarity index 88% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java index bec533af2b54..dca5629d5319 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; @@ -12,10 +12,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java similarity index 93% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java index 5521f04a5dcf..d58946d0f8ba 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.csv; +package io.airbyte.cdk.integrations.destination.gcs.csv; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -11,12 +11,12 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java similarity index 92% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java index 3db5d455daff..577a810dc72d 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.jsonl; +package io.airbyte.cdk.integrations.destination.gcs.jsonl; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -10,11 +10,11 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java similarity index 89% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java index 078a78c5aec9..ef47148b802f 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; public class ConfigTestUtils { diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/resources/test_config.json b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/resources/test_config.json similarity index 100% rename from airbyte-integrations/connectors/destination-gcs/src/test/resources/test_config.json rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/resources/test_config.json diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java similarity index 99% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java index f5178562d19b..f34a2d21b77f 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java index 2dcb585cd770..97793a57758d 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java new file mode 100644 index 000000000000..f855843de3eb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.avro.Schema.Type; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.SeekableByteArrayInput; +import org.apache.avro.generic.GenericData.Record; +import org.apache.avro.generic.GenericDatumReader; + +public abstract class GcsBaseAvroDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { + + public GcsBaseAvroDestinationAcceptanceTest() { + super(S3Format.AVRO); + } + + @Override + protected JsonNode getFormatConfig() { + return Jsons.deserialize("{\n" + + " \"format_type\": \"Avro\",\n" + + " \"compression_codec\": { \"codec\": \"no compression\", \"compression_level\": 5, \"include_checksum\": true }\n" + + "}"); + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new GcsAvroTestDataComparator(); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + final JsonFieldNameUpdater nameUpdater = AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema); + + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + final List jsonRecords = new LinkedList<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + try (final DataFileReader dataFileReader = new DataFileReader<>( + new SeekableByteArrayInput(object.getObjectContent().readAllBytes()), + new GenericDatumReader<>())) { + final ObjectReader jsonReader = MAPPER.reader(); + while (dataFileReader.hasNext()) { + final Record record = dataFileReader.next(); + final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); + JsonNode jsonRecord = jsonReader.readTree(jsonBytes); + jsonRecord = nameUpdater.getJsonWithOriginalFieldNames(jsonRecord); + jsonRecords.add(AvroRecordHelper.pruneAirbyteJson(jsonRecord)); + } + } + } + + return jsonRecords; + } + + @Override + protected Map> retrieveDataTypesFromPersistedFiles(final String streamName, final String namespace) throws Exception { + + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + final Map> resultDataTypes = new HashMap<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + try (final DataFileReader dataFileReader = new DataFileReader<>( + new SeekableByteArrayInput(object.getObjectContent().readAllBytes()), + new GenericDatumReader<>())) { + while (dataFileReader.hasNext()) { + final Record record = dataFileReader.next(); + final Map> actualDataTypes = getTypes(record); + resultDataTypes.putAll(actualDataTypes); + } + } + } + return resultDataTypes; + } + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java new file mode 100644 index 000000000000..5b55c80c7191 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.StreamSupport; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; + +public abstract class GcsBaseCsvDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { + + public GcsBaseCsvDestinationAcceptanceTest() { + super(S3Format.CSV); + } + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + + @Override + protected JsonNode getFormatConfig() { + return Jsons.jsonNode(Map.of( + "format_type", outputFormat, + "flattening", Flattening.ROOT_LEVEL.getValue(), + "compression", Jsons.jsonNode(Map.of("compression_type", "No Compression")))); + } + + /** + * Convert json_schema to a map from field name to field types. + */ + private static Map getFieldTypes(final JsonNode streamSchema) { + final Map fieldTypes = new HashMap<>(); + final JsonNode fieldDefinitions = streamSchema.get("properties"); + final Iterator> iterator = fieldDefinitions.fields(); + while (iterator.hasNext()) { + final Entry entry = iterator.next(); + JsonNode fieldValue = entry.getValue(); + JsonNode typeValue = fieldValue.get("type") == null ? fieldValue.get("$ref") : fieldValue.get("type"); + fieldTypes.put(entry.getKey(), typeValue.asText()); + } + return fieldTypes; + } + + private static JsonNode getJsonNode(final Map input, final Map fieldTypes) { + final ObjectNode json = MAPPER.createObjectNode(); + + if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) { + return Jsons.deserialize(input.get(JavaBaseConstants.COLUMN_NAME_DATA)); + } + + for (final Entry entry : input.entrySet()) { + final String key = entry.getKey(); + if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key + .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { + continue; + } + final String value = entry.getValue(); + if (value == null || value.equals("")) { + continue; + } + final String type = fieldTypes.get(key); + switch (type) { + case "boolean" -> json.put(key, Boolean.valueOf(value)); + case "integer" -> json.put(key, Integer.valueOf(value)); + case "number" -> json.put(key, Double.valueOf(value)); + case "" -> addNoTypeValue(json, key, value); + default -> json.put(key, value); + } + } + return json; + } + + private static void addNoTypeValue(final ObjectNode json, final String key, final String value) { + if (value != null && (value.matches("^\\[.*\\]$")) || value.matches("^\\{.*\\}$")) { + final var newNode = Jsons.deserialize(value); + json.set(key, newNode); + } else { + json.put(key, value); + } + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws IOException { + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + + final Map fieldTypes = getFieldTypes(streamSchema); + final List jsonRecords = new LinkedList<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + try (final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + final Reader in = getReader(object)) { + final Iterable records = CSVFormat.DEFAULT + .withQuoteMode(QuoteMode.NON_NUMERIC) + .withFirstRecordAsHeader() + .parse(in); + StreamSupport.stream(records.spliterator(), false) + .forEach(r -> jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))); + } + } + + return jsonRecords; + } + + protected Reader getReader(final S3Object s3Object) throws IOException { + return new InputStreamReader(s3Object.getObjectContent(), StandardCharsets.UTF_8); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java new file mode 100644 index 000000000000..6bf3d81ea48d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.zip.GZIPInputStream; + +public abstract class GcsBaseCsvGzipDestinationAcceptanceTest extends GcsBaseCsvDestinationAcceptanceTest { + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + + @Override + protected JsonNode getFormatConfig() { + // config without compression defaults to GZIP + return Jsons.jsonNode(Map.of( + "format_type", outputFormat, + "flattening", Flattening.ROOT_LEVEL.getValue())); + } + + protected Reader getReader(final S3Object s3Object) throws IOException { + return new InputStreamReader(new GZIPInputStream(s3Object.getObjectContent()), StandardCharsets.UTF_8); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java new file mode 100644 index 000000000000..40e7bae0051f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.commons.json.Jsons; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public abstract class GcsBaseJsonlDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { + + public GcsBaseJsonlDestinationAcceptanceTest() { + super(S3Format.JSONL); + } + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + + @Override + protected JsonNode getFormatConfig() { + return Jsons.jsonNode(Map.of( + "format_type", outputFormat, + "compression", Jsons.jsonNode(Map.of("compression_type", "No Compression")))); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws IOException { + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + final List jsonRecords = new LinkedList<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + try (final BufferedReader reader = getReader(object)) { + String line; + while ((line = reader.readLine()) != null) { + jsonRecords.add(Jsons.deserialize(line).get(JavaBaseConstants.COLUMN_NAME_DATA)); + } + } + } + + return jsonRecords; + } + + protected BufferedReader getReader(final S3Object s3Object) throws IOException { + return new BufferedReader(new InputStreamReader(s3Object.getObjectContent(), StandardCharsets.UTF_8)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java new file mode 100644 index 000000000000..2924aecc8d1e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.commons.json.Jsons; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.zip.GZIPInputStream; + +public abstract class GcsBaseJsonlGzipDestinationAcceptanceTest extends GcsBaseJsonlDestinationAcceptanceTest { + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + + @Override + protected JsonNode getFormatConfig() { + // config without compression defaults to GZIP + return Jsons.jsonNode(Map.of("format_type", outputFormat)); + } + + protected BufferedReader getReader(final S3Object s3Object) throws IOException { + return new BufferedReader(new InputStreamReader(new GZIPInputStream(s3Object.getObjectContent()), StandardCharsets.UTF_8)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java new file mode 100644 index 000000000000..3725e76e02ea --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.gcs; + +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.gcs.parquet.GcsParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.avro.Schema.Type; +import org.apache.avro.generic.GenericData.Record; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.avro.AvroReadSupport; +import org.apache.parquet.hadoop.ParquetReader; + +public abstract class GcsBaseParquetDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { + + public GcsBaseParquetDestinationAcceptanceTest() { + super(S3Format.PARQUET); + } + + @Override + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V1; + } + + @Override + protected JsonNode getFormatConfig() { + return Jsons.jsonNode(Map.of( + "format_type", "Parquet", + "compression_codec", "GZIP")); + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new GcsAvroTestDataComparator(); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws IOException, URISyntaxException { + final JsonFieldNameUpdater nameUpdater = AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema); + + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + final List jsonRecords = new LinkedList<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + final URI uri = new URI(String.format("s3a://%s/%s", object.getBucketName(), object.getKey())); + final var path = new org.apache.hadoop.fs.Path(uri); + final Configuration hadoopConfig = GcsParquetWriter.getHadoopConfig(config); + + try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) + .withConf(hadoopConfig) + .build()) { + final ObjectReader jsonReader = MAPPER.reader(); + Record record; + while ((record = parquetReader.read()) != null) { + final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); + JsonNode jsonRecord = jsonReader.readTree(jsonBytes); + jsonRecord = nameUpdater.getJsonWithOriginalFieldNames(jsonRecord); + jsonRecords.add(AvroRecordHelper.pruneAirbyteJson(jsonRecord)); + } + } + } + + return jsonRecords; + } + + @Override + protected Map> retrieveDataTypesFromPersistedFiles(final String streamName, final String namespace) throws Exception { + + final List objectSummaries = getAllSyncedObjects(streamName, namespace); + final Map> resultDataTypes = new HashMap<>(); + + for (final S3ObjectSummary objectSummary : objectSummaries) { + final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + final URI uri = new URI(String.format("s3a://%s/%s", object.getBucketName(), object.getKey())); + final var path = new org.apache.hadoop.fs.Path(uri); + final Configuration hadoopConfig = S3ParquetWriter.getHadoopConfig(config); + + try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) + .withConf(hadoopConfig) + .build()) { + Record record; + while ((record = parquetReader.read()) != null) { + final Map> actualDataTypes = getTypes(record); + resultDataTypes.putAll(actualDataTypes); + } + } + } + + return resultDataTypes; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java similarity index 95% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java index b78e0ef089d4..a92ee08a4e15 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -46,8 +46,7 @@ *
  • Implement {@link #getFormatConfig} that returns a {@link S3FormatConfig}
    • *
  • Implement {@link #retrieveRecords} that returns the Json records for the test
    • * - * Under the hood, a {@link io.airbyte.integrations.destination.gcs.GcsDestinationConfig} is - * constructed as follows: + * Under the hood, a {@link GcsDestinationConfig} is constructed as follows: *
  • Retrieve the secrets from "secrets/config.json"
    • *
  • Get the GCS bucket path from the constructor
    • *
  • Get the format config from {@link #getFormatConfig}
    • @@ -79,11 +78,6 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } - @Override - protected String getImageName() { - return "airbyte/destination-gcs:dev"; - } - @Override protected JsonNode getConfig() { return configJson; @@ -240,7 +234,7 @@ public void testCheckIncorrectHmacKeyAccessIdCredential() { ((ObjectNode) baseJson).put("credential", credential); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: SignatureDoesNotMatch;")); @@ -258,7 +252,7 @@ public void testCheckIncorrectHmacKeySecretCredential() { ((ObjectNode) baseJson).put("credential", credential); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: SignatureDoesNotMatch;")); @@ -270,7 +264,7 @@ public void testCheckIncorrectBucketCredential() { ((ObjectNode) baseJson).put("gcs_bucket_name", "fake_bucket"); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: NoSuchKey;")); diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg deleted file mode 100644 index 554d68224850..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle deleted file mode 100644 index c76851730e11..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -plugins { - id 'java-library' - id "de.undercouch.download" version "5.4.0" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation 'commons-cli:commons-cli:1.4' - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - implementation libs.lombok - implementation libs.micronaut.cache.caffeine - - testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' -} - -def downloadConnectorRegistry = tasks.register('downloadConnectorRegistry', Download) { - src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' - dest new File(projectDir, 'src/main/resources/seed/oss_registry.json') - overwrite true -} -tasks.named('processResources')configure { - dependsOn downloadConnectorRegistry -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md b/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md deleted file mode 100644 index f9b088d33d6f..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md +++ /dev/null @@ -1,12 +0,0 @@ -# airbyte-config:init - -This module fulfills two responsibilities: -1. It is where we declare what connectors should ship with the Platform. See below for more instruction on how it works. -2. It contains the scripts and Dockerfile that allow the `docker-compose` version of Airbyte to mount the local filesystem. This is helpful in cases where a user wants to use a connector that interacts with (reads data from or writes data to) the local filesystem. e.g. `destination-local-json`. - -## The Connector Registry and Spec Secret Masks -The connector registry (list of available connectors) is downloaded at runtime from https://connectors.airbyte.com/files/registries/v0/oss_registry.json - -The spec secret mask (one of the multiple systems that hide your secrets from the logs) is also downloaded at runtime from https://connectors.airbyte.com/files/registries/v0/spec_secret_mask.json - -The logic inside the folder is responsible for downloading these files and making them available to the platform. \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh b/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh deleted file mode 100755 index c27ee40a1c0c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env sh - -set -e - -# hack: attempt to get local mounts to work properly -# constraints: ROOT_PARENT better exist on the local filesystem. -# check that the given directory (ROOT) that we plan to use as a mount -# in other containers is a in fact a directory within the parent. if it -# is, then we make sure it is created. we do this by removing the common -# part of the path from the root and appending it to the mount. then we -# make the directories. -# e.g. ROOT_PARENT=/tmp, ROOT=/tmp/airbyte_local MOUNT=/local_parent. -# We create MOUNT_ROOT which will look like /local_parent/airbyte_local. -# Because it is using the mount name, we can create it on the local -# fileystem from within the container. -MOUNT=$1; echo "MOUNT: $MOUNT" -ROOT_PARENT=$2; echo "ROOT_PARENT: $ROOT_PARENT" -ROOT=$3; echo "ROOT: $ROOT" - -[[ "${ROOT}"="${ROOT_PARENT}"* ]] || (echo "ROOT ${ROOT} is not a child of ROOT_PARENT ${ROOT_PARENT}." && exit 1) -MOUNT_ROOT=${MOUNT}/$(echo $ROOT | sed -e "s|${ROOT_PARENT}||g") -echo "MOUNT_ROOT: ${MOUNT_ROOT}" -mkdir -p ${MOUNT_ROOT} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java deleted file mode 100644 index 66033a188e8c..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -public class ConfigNotFoundException extends Exception { - - private static final long serialVersionUID = 836273627; - private final String type; - private final String configId; - - public ConfigNotFoundException(final String type, final String configId) { - super(String.format("config type: %s id: %s", type, configId)); - this.type = type; - this.configId = configId; - } - - public String getType() { - return type; - } - - public String getConfigId() { - return configId; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java deleted file mode 100644 index 34bd03a801c6..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.util.List; -import java.util.UUID; - -public interface DefinitionsProvider { - - StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException; - - List getSourceDefinitions(); - - StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException; - - List getDestinationDefinitions(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java deleted file mode 100644 index 2aca382f70ba..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.BooleanNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -public class JsonDefinitionsHelper { - - public static JsonNode addMissingTombstoneField(final JsonNode definitionJson) { - final JsonNode currTombstone = definitionJson.get("tombstone"); - if (currTombstone == null || currTombstone.isNull()) { - ((ObjectNode) definitionJson).set("tombstone", BooleanNode.FALSE); - } - return definitionJson; - } - - public static JsonNode addMissingPublicField(final JsonNode definitionJson) { - final JsonNode currPublic = definitionJson.get("public"); - if (currPublic == null || currPublic.isNull()) { - // definitions loaded from seed yamls are by definition public - ((ObjectNode) definitionJson).set("public", BooleanNode.TRUE); - } - return definitionJson; - } - - public static JsonNode addMissingCustomField(final JsonNode definitionJson) { - final JsonNode currCustom = definitionJson.get("custom"); - if (currCustom == null || currCustom.isNull()) { - // definitions loaded from seed yamls are by definition not custom - ((ObjectNode) definitionJson).set("custom", BooleanNode.FALSE); - } - return definitionJson; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java deleted file mode 100644 index 33966a42ad58..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.google.common.io.Resources; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.configoss.CatalogDefinitionsConfig; -import io.airbyte.configoss.CombinedConnectorCatalog; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -/** - * This provider contains all definitions according to the local catalog json files. - */ -final public class LocalDefinitionsProvider implements DefinitionsProvider { - - private static final String LOCAL_CONNECTOR_REGISTRY_PATH = CatalogDefinitionsConfig.getLocalConnectorCatalogPath(); - - public CombinedConnectorCatalog getLocalDefinitionCatalog() { - try { - final URL url = Resources.getResource(LOCAL_CONNECTOR_REGISTRY_PATH); - final String jsonString = Resources.toString(url, StandardCharsets.UTF_8); - final CombinedConnectorCatalog catalog = Jsons.deserialize(jsonString, CombinedConnectorCatalog.class); - return catalog; - - } catch (final Exception e) { - throw new RuntimeException("Failed to fetch local catalog definitions", e); - } - } - - public Map getSourceDefinitionsMap() { - final CombinedConnectorCatalog catalog = getLocalDefinitionCatalog(); - return catalog.getSources().stream().collect(Collectors.toMap( - StandardSourceDefinition::getSourceDefinitionId, - source -> source.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(source.getSpec() != null ? source.getSpec().getProtocolVersion() : null).serialize()))); - } - - public Map getDestinationDefinitionsMap() { - final CombinedConnectorCatalog catalog = getLocalDefinitionCatalog(); - return catalog.getDestinations().stream().collect( - Collectors.toMap( - StandardDestinationDefinition::getDestinationDefinitionId, - destination -> destination.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault( - destination.getSpec() != null - ? destination.getSpec().getProtocolVersion() - : null) - .serialize()))); - } - - @Override - public StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardSourceDefinition definition = getSourceDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("local_registry:source_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getSourceDefinitions() { - return new ArrayList<>(getSourceDefinitionsMap().values()); - } - - @Override - public StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardDestinationDefinition definition = getDestinationDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("local_registry:destination_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getDestinationDefinitions() { - return new ArrayList<>(getDestinationDefinitionsMap().values()); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java deleted file mode 100644 index 4d6145a5f35e..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -/** - * Defines any additional tasks that should be executed after successful boostrapping of the Airbyte - * environment. - */ -public interface PostLoadExecutor { - - /** - * Executes the additional post bootstrapping tasks. - * - * @throws Exception if unable to perform the additional tasks. - */ - void execute() throws Exception; - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java deleted file mode 100644 index c1e4816367a3..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.configoss.CombinedConnectorCatalog; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import io.micronaut.cache.annotation.CacheConfig; -import io.micronaut.cache.annotation.Cacheable; -import io.micronaut.context.annotation.Primary; -import io.micronaut.context.annotation.Requires; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; - -/** - * This provider pulls the definitions from a remotely hosted catalog. - */ -@Singleton -@Primary -@Requires(property = "airbyte.platform.remote-connector-catalog.url", - notEquals = "") -@CacheConfig("remote-definitions-provider") -@Slf4j -public class RemoteDefinitionsProvider implements DefinitionsProvider { - - private static final HttpClient httpClient = HttpClient.newHttpClient(); - private final URI remoteDefinitionCatalogUrl; - private final Duration timeout; - - public RemoteDefinitionsProvider(@Value("${airbyte.platform.remote-connector-catalog.url}") final String remoteCatalogUrl, - @Value("${airbyte.platform.remote-connector-catalog.timeout-ms}") final long remoteCatalogTimeoutMs) - throws URISyntaxException { - log.info("Creating remote definitions provider for URL '{}'...", remoteCatalogUrl); - remoteDefinitionCatalogUrl = new URI(remoteCatalogUrl); - timeout = Duration.ofMillis(remoteCatalogTimeoutMs); - } - - private Map getSourceDefinitionsMap() { - final CombinedConnectorCatalog catalog = getRemoteDefinitionCatalog(); - return catalog.getSources().stream().collect(Collectors.toMap( - StandardSourceDefinition::getSourceDefinitionId, - source -> source.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(source.getSpec() != null ? source.getSpec().getProtocolVersion() : null).serialize()))); - } - - private Map getDestinationDefinitionsMap() { - final CombinedConnectorCatalog catalog = getRemoteDefinitionCatalog(); - return catalog.getDestinations().stream().collect(Collectors.toMap( - StandardDestinationDefinition::getDestinationDefinitionId, - destination -> destination.withProtocolVersion( - AirbyteProtocolVersion.getWithDefault(destination.getSpec() != null ? destination.getSpec().getProtocolVersion() : null).serialize()))); - } - - @Override - public StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardSourceDefinition definition = getSourceDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("remote_registry:source_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getSourceDefinitions() { - return new ArrayList<>(getSourceDefinitionsMap().values()); - } - - @Override - public StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException { - final StandardDestinationDefinition definition = getDestinationDefinitionsMap().get(definitionId); - if (definition == null) { - throw new ConfigNotFoundException("remote_registry:destination_def", definitionId.toString()); - } - return definition; - } - - @Override - public List getDestinationDefinitions() { - return new ArrayList<>(getDestinationDefinitionsMap().values()); - } - - @Cacheable - public CombinedConnectorCatalog getRemoteDefinitionCatalog() { - try { - final HttpRequest request = HttpRequest.newBuilder(remoteDefinitionCatalogUrl).timeout(timeout).header("accept", "application/json").build(); - - final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (errorStatusCode(response)) { - throw new IOException( - "getRemoteDefinitionCatalog request ran into status code error: " + response.statusCode() + " with message: " + response.getClass()); - } - - log.info("Fetched latest remote definitions ({})", response.body().hashCode()); - return Jsons.deserialize(response.body(), CombinedConnectorCatalog.class); - } catch (final Exception e) { - throw new RuntimeException("Failed to fetch remote definitions", e); - } - } - - private static Boolean errorStatusCode(final HttpResponse response) { - return response.statusCode() >= 400; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json deleted file mode 100644 index 75a2b6869422..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "2817b3f0-04e4-4c7a-9f32-7a5e8a83db95", - "name": "PagerDuty", - "dockerRepository": "farosai/airbyte-pagerduty-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pagerduty", - "icon": "pagerduty.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json deleted file mode 100644 index 80c10bf0b680..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "6fe89830-d04d-401b-aad6-6552ffa5c4af", - "name": "Harness", - "dockerRepository": "farosai/airbyte-harness-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/harness", - "icon": "harness.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json deleted file mode 100644 index 1ec101ea71ec..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "7e20ce3e-d820-4327-ad7a-88f3927fd97a", - "name": "VictorOps", - "dockerRepository": "farosai/airbyte-victorops-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/victorops", - "icon": "victorops.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json deleted file mode 100644 index ada2c5380217..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "sourceDefinitionId": "c47d6804-8b98-449f-970a-5ddb5cb5d7aa", - "name": "Customer.io", - "dockerRepository": "farosai/airbyte-customer-io-source", - "dockerImageTag": "0.1.23", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/customer-io", - "icon": "customer-io.svg" -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg deleted file mode 100644 index 36c7f62a33c1..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg deleted file mode 100644 index 01e394f5b30f..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java deleted file mode 100644 index 768ea436aa53..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -class LocalDefinitionsProviderTest { - - private static LocalDefinitionsProvider localDefinitionsProvider; - - @BeforeAll - static void setup() throws IOException { - localDefinitionsProvider = new LocalDefinitionsProvider(); - } - - @Test - void testGetSourceDefinition() throws Exception { - // source - final UUID stripeSourceId = UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de"); - final StandardSourceDefinition stripeSource = localDefinitionsProvider.getSourceDefinition(stripeSourceId); - assertEquals(stripeSourceId, stripeSource.getSourceDefinitionId()); - assertEquals("Stripe", stripeSource.getName()); - assertEquals("airbyte/source-stripe", stripeSource.getDockerRepository()); - assertEquals("https://docs.airbyte.com/integrations/sources/stripe", stripeSource.getDocumentationUrl()); - assertEquals("stripe.svg", stripeSource.getIcon()); - assertEquals(URI.create("https://docs.airbyte.com/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); - assertEquals(false, stripeSource.getTombstone()); - assertEquals("0.2.0", stripeSource.getProtocolVersion()); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetDestinationDefinition() throws Exception { - final UUID s3DestinationId = UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362"); - final StandardDestinationDefinition s3Destination = localDefinitionsProvider - .getDestinationDefinition(s3DestinationId); - assertEquals(s3DestinationId, s3Destination.getDestinationDefinitionId()); - assertEquals("S3", s3Destination.getName()); - assertEquals("airbyte/destination-s3", s3Destination.getDockerRepository()); - assertEquals("https://docs.airbyte.com/integrations/destinations/s3", s3Destination.getDocumentationUrl()); - assertEquals(URI.create("https://docs.airbyte.com/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); - assertEquals(false, s3Destination.getTombstone()); - assertEquals("0.2.0", s3Destination.getProtocolVersion()); - } - - @Test - void testGetInvalidDefinitionId() { - final UUID invalidDefinitionId = UUID.fromString("1a7c360c-1289-4b96-a171-2ac1c86fb7ca"); - - assertThrows( - ConfigNotFoundException.class, - () -> localDefinitionsProvider.getSourceDefinition(invalidDefinitionId)); - assertThrows( - ConfigNotFoundException.class, - () -> localDefinitionsProvider.getDestinationDefinition(invalidDefinitionId)); - } - - @Test - void testGetSourceDefinitions() { - final List sourceDefinitions = localDefinitionsProvider.getSourceDefinitions(); - assertFalse(sourceDefinitions.isEmpty()); - assertTrue(sourceDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - - @Test - void testGetDestinationDefinitions() { - final List destinationDefinitions = localDefinitionsProvider.getDestinationDefinitions(); - assertFalse(destinationDefinitions.isEmpty()); - assertTrue(destinationDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java deleted file mode 100644 index 3cfcc905947b..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.io.Resources; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.configoss.StandardDestinationDefinition; -import io.airbyte.configoss.StandardSourceDefinition; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.net.http.HttpTimeoutException; -import java.nio.charset.Charset; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class RemoteDefinitionsProviderTest { - - private MockWebServer webServer; - private MockResponse validCatalogResponse; - private String catalogUrl; - private JsonNode jsonCatalog; - - @BeforeEach - void setup() throws IOException { - webServer = new MockWebServer(); - catalogUrl = webServer.url("/connector_catalog.json").toString(); - - final URL testCatalog = Resources.getResource("connector_catalog.json"); - final String jsonBody = Resources.toString(testCatalog, Charset.defaultCharset()); - jsonCatalog = Jsons.deserialize(jsonBody); - validCatalogResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") - .setBody(jsonBody); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetSourceDefinition() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID stripeSourceId = UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de"); - final StandardSourceDefinition stripeSource = remoteDefinitionsProvider.getSourceDefinition(stripeSourceId); - assertEquals(stripeSourceId, stripeSource.getSourceDefinitionId()); - assertEquals("Stripe", stripeSource.getName()); - assertEquals("airbyte/source-stripe", stripeSource.getDockerRepository()); - assertEquals("https://docs.airbyte.io/integrations/sources/stripe", stripeSource.getDocumentationUrl()); - assertEquals("stripe.svg", stripeSource.getIcon()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); - assertEquals(false, stripeSource.getTombstone()); - assertEquals("0.2.1", stripeSource.getProtocolVersion()); - } - - @Test - @SuppressWarnings({"PMD.AvoidDuplicateLiterals"}) - void testGetDestinationDefinition() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID s3DestinationId = UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362"); - final StandardDestinationDefinition s3Destination = remoteDefinitionsProvider - .getDestinationDefinition(s3DestinationId); - assertEquals(s3DestinationId, s3Destination.getDestinationDefinitionId()); - assertEquals("S3", s3Destination.getName()); - assertEquals("airbyte/destination-s3", s3Destination.getDockerRepository()); - assertEquals("https://docs.airbyte.io/integrations/destinations/s3", s3Destination.getDocumentationUrl()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); - assertEquals(false, s3Destination.getTombstone()); - assertEquals("0.2.2", s3Destination.getProtocolVersion()); - } - - @Test - void testGetInvalidDefinitionId() throws Exception { - webServer.enqueue(validCatalogResponse); - webServer.enqueue(validCatalogResponse); - - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final UUID invalidDefinitionId = UUID.fromString("1a7c360c-1289-4b96-a171-2ac1c86fb7ca"); - - assertThrows( - ConfigNotFoundException.class, - () -> remoteDefinitionsProvider.getSourceDefinition(invalidDefinitionId)); - assertThrows( - ConfigNotFoundException.class, - () -> remoteDefinitionsProvider.getDestinationDefinition(invalidDefinitionId)); - } - - @Test - void testGetSourceDefinitions() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final List sourceDefinitions = remoteDefinitionsProvider.getSourceDefinitions(); - final int expectedNumberOfSources = MoreIterators.toList(jsonCatalog.get("sources").elements()).size(); - assertEquals(expectedNumberOfSources, sourceDefinitions.size()); - assertTrue(sourceDefinitions.stream().allMatch(sourceDef -> sourceDef.getProtocolVersion().length() > 0)); - } - - @Test - void testGetDestinationDefinitions() throws Exception { - webServer.enqueue(validCatalogResponse); - final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(30)); - final List destinationDefinitions = remoteDefinitionsProvider.getDestinationDefinitions(); - final int expectedNumberOfDestinations = MoreIterators.toList(jsonCatalog.get("destinations").elements()).size(); - assertEquals(expectedNumberOfDestinations, destinationDefinitions.size()); - assertTrue(destinationDefinitions.stream().allMatch(destDef -> destDef.getProtocolVersion().length() > 0)); - } - - @Test - void testBadResponseStatus() { - webServer.enqueue(new MockResponse().setResponseCode(404)); - final RuntimeException ex = assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - - assertTrue(ex.getMessage().contains("Failed to fetch remote definitions")); - assertTrue(ex.getCause() instanceof IOException); - } - - @Test - void testTimeOut() { - // No request enqueued -> Timeout - final RuntimeException ex = assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - - assertTrue(ex.getMessage().contains("Failed to fetch remote definitions")); - assertTrue(ex.getCause() instanceof HttpTimeoutException); - } - - @Test - void testNonJson() { - final MockResponse notJson = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") - .setBody("not json"); - webServer.enqueue(notJson); - assertThrows(RuntimeException.class, () -> { - new RemoteDefinitionsProvider(catalogUrl, TimeUnit.SECONDS.toMillis(1)).getDestinationDefinitions(); - }); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java deleted file mode 100644 index 68136d8bfef2..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.configoss.init; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.JsonSchemas; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -@Slf4j -class SpecFormatTest { - - @Test - void testOnAllExistingConfig() throws IOException, JsonValidationException { - final DefinitionsProvider definitionsProvider = new LocalDefinitionsProvider(); - - final List sourceSpecs = definitionsProvider.getSourceDefinitions() - .stream() - .map(standardSourceDefinition -> standardSourceDefinition.getSpec().getConnectionSpecification()) - .toList(); - - final List destinationSpecs = definitionsProvider.getDestinationDefinitions() - .stream() - .map(standardDestinationDefinition -> standardDestinationDefinition.getSpec().getConnectionSpecification()) - .toList(); - - final List allSpecs = new ArrayList<>(); - - allSpecs.addAll(sourceSpecs); - allSpecs.addAll(destinationSpecs); - - Assertions.assertThat(allSpecs) - .flatMap(spec -> { - try { - if (!isValidJsonSchema(spec)) { - throw new RuntimeException("Fail JsonSecretsProcessor validation"); - } - JsonSchemas.traverseJsonSchema(spec, (node, path) -> {}); - return Collections.emptyList(); - } catch (final Exception e) { - log.error("failed on: " + spec.toString(), e); - return List.of(e); - } - }) - .isEmpty(); - } - - private static boolean isValidJsonSchema(final JsonNode schema) { - return schema.isObject() && ((schema.has("properties") && schema.get("properties").isObject()) - || (schema.has("oneOf") && schema.get("oneOf").isArray())); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json deleted file mode 100644 index f4e09b0d95c5..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json +++ /dev/null @@ -1,14185 +0,0 @@ -{ - "destinations": [ - { - "destinationDefinitionId": "0eeee7fb-518f-4045-bacc-9619e31c43ea", - "name": "Amazon SQS", - "dockerRepository": "airbyte/destination-amazon-sqs", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/amazon-sqs", - "icon": "amazonsqs.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/amazon-sqs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Amazon Sqs", - "type": "object", - "required": ["queue_url", "region"], - "additionalProperties": false, - "properties": { - "queue_url": { - "title": "Queue URL", - "description": "URL of the SQS Queue", - "type": "string", - "examples": [ - "https://sqs.eu-west-1.amazonaws.com/1234567890/my-example-queue" - ], - "order": 0 - }, - "region": { - "title": "AWS Region", - "description": "AWS Region of the SQS Queue", - "type": "string", - "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 1 - }, - "message_delay": { - "title": "Message Delay", - "description": "Modify the Message Delay of the individual message from the Queue's default (seconds).", - "type": "integer", - "examples": ["15"], - "order": 2 - }, - "access_key": { - "title": "AWS IAM Access Key ID", - "description": "The Access Key ID of the AWS IAM Role to use for sending messages", - "type": "string", - "examples": ["xxxxxHRNxxx3TBxxxxxx"], - "order": 3, - "airbyte_secret": true - }, - "secret_key": { - "title": "AWS IAM Secret Key", - "description": "The Secret Key of the AWS IAM Role to use for sending messages", - "type": "string", - "examples": ["hu+qE5exxxxT6o/ZrKsxxxxxxBhxxXLexxxxxVKz"], - "order": 4, - "airbyte_secret": true - }, - "message_body_key": { - "title": "Message Body Key", - "description": "Use this property to extract the contents of the named key in the input record to use as the SQS message body. If not set, the entire content of the input record data is used as the message body.", - "type": "string", - "examples": ["myDataPath"], - "order": 5 - }, - "message_group_id": { - "title": "Message Group Id", - "description": "The tag that specifies that a message belongs to a specific message group. This parameter applies only to, and is REQUIRED by, FIFO queues.", - "type": "string", - "examples": ["my-fifo-group"], - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "b4c5d105-31fd-4817-96b6-cb923bfc04cb", - "name": "Azure Blob Storage", - "dockerRepository": "airbyte/destination-azure-blob-storage", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/azureblobstorage", - "icon": "azureblobstorage.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/azureblobstorage", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AzureBlobStorage Destination Spec", - "type": "object", - "required": [ - "azure_blob_storage_account_name", - "azure_blob_storage_account_key", - "format" - ], - "additionalProperties": false, - "properties": { - "azure_blob_storage_endpoint_domain_name": { - "title": "Endpoint Domain Name", - "type": "string", - "default": "blob.core.windows.net", - "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", - "examples": ["blob.core.windows.net"] - }, - "azure_blob_storage_container_name": { - "title": "Azure blob storage container (Bucket) Name", - "type": "string", - "description": "The name of the Azure blob storage container. If not exists - will be created automatically. May be empty, then will be created automatically airbytecontainer+timestamp", - "examples": ["airbytetescontainername"] - }, - "azure_blob_storage_account_name": { - "title": "Azure Blob Storage account name", - "type": "string", - "description": "The account's name of the Azure Blob Storage.", - "examples": ["airbyte5storage"] - }, - "azure_blob_storage_account_key": { - "title": "Azure Blob Storage account key", - "description": "The Azure blob storage account key.", - "airbyte_secret": true, - "type": "string", - "examples": [ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" - ] - }, - "azure_blob_storage_output_buffer_size": { - "title": "Azure Blob Storage output buffer size (Megabytes)", - "type": "integer", - "description": "The amount of megabytes to buffer for the output stream to Azure. This will impact memory footprint on workers, but may need adjustment for performance and appropriate block size in Azure.", - "minimum": 1, - "maximum": 2047, - "default": 5, - "examples": [5] - }, - "format": { - "title": "Output Format", - "type": "object", - "description": "Output data format", - "oneOf": [ - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type", "flattening"], - "properties": { - "format_type": { - "type": "string", - "const": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Flattening)", - "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - } - } - }, - { - "title": "JSON Lines: newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "const": "JSONL" - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", - "name": "BigQuery", - "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "1.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "icon": "bigquery.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Destination Spec", - "type": "object", - "required": ["project_id", "dataset_location", "dataset_id"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", - "title": "Project ID", - "order": 0 - }, - "dataset_location": { - "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. Read more here.", - "title": "Dataset Location", - "order": 1, - "enum": [ - "US", - "EU", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "northamerica-northeast1", - "northamerica-northeast2", - "southamerica-east1", - "southamerica-west1", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4" - ] - }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID", - "order": 2 - }, - "loading_method": { - "type": "object", - "title": "Loading Method", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
    Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
    GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", - "order": 3, - "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "GCS Staging", - "required": [ - "method", - "gcs_bucket_name", - "gcs_bucket_path", - "credential" - ], - "properties": { - "method": { - "type": "string", - "const": "GCS Staging", - "order": 0 - }, - "credential": { - "title": "Credential", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 1, - "oneOf": [ - { - "title": "HMAC key", - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "const": "HMAC_KEY", - "order": 0 - }, - "hmac_key_access_id": { - "type": "string", - "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", - "title": "HMAC Key Access ID", - "airbyte_secret": true, - "examples": ["1234567890abcdefghij1234"], - "order": 1 - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", - "title": "HMAC Key Secret", - "airbyte_secret": true, - "examples": [ - "1234567890abcdefghij1234567890ABCDEFGHIJ" - ], - "order": 2 - } - } - } - ] - }, - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "type": "string", - "description": "The name of the GCS bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "Directory under the GCS bucket where data will be written.", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "keep_files_in_gcs-bucket": { - "type": "string", - "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", - "title": "GCS Tmp Files Afterward Processing (Optional)", - "default": "Delete all tmp files from GCS", - "enum": [ - "Delete all tmp files from GCS", - "Keep all tmp files in GCS" - ], - "order": 4 - } - } - } - ] - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Service Account Key JSON (Required for cloud, optional for open-source)", - "airbyte_secret": true, - "order": 4 - }, - "transformation_priority": { - "type": "string", - "description": "Interactive run type means that the query is executed as soon as possible, and these queries count towards concurrent rate limit and daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources are available in the BigQuery shared resource pool, which usually occurs within a few minutes. Batch queries don’t count towards your concurrent rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly.", - "title": "Transformation Query Run Type (Optional)", - "default": "interactive", - "enum": ["interactive", "batch"], - "order": 5 - }, - "big_query_client_buffer_size_mb": { - "title": "Google BigQuery Client Chunk Size (Optional)", - "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", - "type": "integer", - "minimum": 1, - "maximum": 15, - "default": 15, - "examples": ["15"], - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "079d5540-f236-4294-ba7c-ade8fd918496", - "name": "BigQuery (denormalized typed struct)", - "dockerRepository": "airbyte/destination-bigquery-denormalized", - "dockerImageTag": "1.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "icon": "bigquery.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Denormalized Typed Struct Destination Spec", - "type": "object", - "required": ["project_id", "dataset_id"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", - "title": "Project ID", - "order": 0 - }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID", - "order": 1 - }, - "loading_method": { - "type": "object", - "title": "Loading Method *", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
    Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
    GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", - "order": 2, - "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "GCS Staging", - "type": "object", - "required": [ - "method", - "gcs_bucket_name", - "gcs_bucket_path", - "credential" - ], - "properties": { - "method": { - "type": "string", - "const": "GCS Staging", - "order": 0 - }, - "credential": { - "title": "Credential", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 1, - "oneOf": [ - { - "title": "HMAC key", - "order": 0, - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "const": "HMAC_KEY", - "order": 0 - }, - "hmac_key_access_id": { - "type": "string", - "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", - "title": "HMAC Key Access ID", - "airbyte_secret": true, - "examples": ["1234567890abcdefghij1234"], - "order": 1 - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", - "title": "HMAC Key Secret", - "airbyte_secret": true, - "examples": [ - "1234567890abcdefghij1234567890ABCDEFGHIJ" - ], - "order": 2 - } - } - } - ] - }, - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "type": "string", - "description": "The name of the GCS bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "Directory under the GCS bucket where data will be written. Read more here.", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "keep_files_in_gcs-bucket": { - "type": "string", - "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", - "title": "GCS Tmp Files Afterward Processing (Optional)", - "default": "Delete all tmp files from GCS", - "enum": [ - "Delete all tmp files from GCS", - "Keep all tmp files in GCS" - ], - "order": 4 - } - } - } - ] - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Service Account Key JSON (Required for cloud, optional for open-source)", - "airbyte_secret": true, - "order": 3 - }, - "dataset_location": { - "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. The default \"US\" value is used if not set explicitly. Read more here.", - "title": "Dataset Location (Optional)", - "default": "US", - "order": 4, - "enum": [ - "US", - "EU", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "northamerica-northeast1", - "northamerica-northeast2", - "southamerica-east1", - "southamerica-west1", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4" - ] - }, - "big_query_client_buffer_size_mb": { - "title": "Google BigQuery Client Chunk Size (Optional)", - "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", - "type": "integer", - "minimum": 1, - "maximum": 15, - "default": 15, - "examples": ["15"], - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "707456df-6f4f-4ced-b5c6-03f73bcad1c5", - "name": "Cassandra", - "dockerRepository": "airbyte/destination-cassandra", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/cassandra", - "icon": "cassandra.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/cassandra", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Cassandra Destination Spec", - "type": "object", - "required": ["keyspace", "username", "password", "address", "port"], - "additionalProperties": true, - "properties": { - "keyspace": { - "title": "Keyspace", - "description": "Default Cassandra keyspace to create data in.", - "type": "string", - "order": 0 - }, - "username": { - "title": "Username", - "description": "Username to use to access Cassandra.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with Cassandra.", - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "address": { - "title": "Address", - "description": "Address to connect to.", - "type": "string", - "examples": ["localhost,127.0.0.1"], - "order": 3 - }, - "port": { - "title": "Port", - "description": "Port of Cassandra.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9042, - "order": 4 - }, - "datacenter": { - "title": "Datacenter", - "description": "Datacenter of the cassandra cluster.", - "type": "string", - "default": "datacenter1", - "order": 5 - }, - "replication": { - "title": "Replication factor", - "type": "integer", - "description": "Indicates to how many nodes the data should be replicated to.", - "default": 1, - "order": 6 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "81740ce8-d764-4ea7-94df-16bb41de36ae", - "name": "Chargify (Keen)", - "dockerRepository": "airbyte/destination-keen", - "dockerImageTag": "0.2.4", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/keen", - "icon": "chargify.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/keen", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Keen Spec", - "type": "object", - "required": ["project_id", "api_key"], - "additionalProperties": false, - "properties": { - "project_id": { - "description": "To get Keen Project ID, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", - "title": "Project ID", - "type": "string", - "examples": ["58b4acc22ba938934e888322e"] - }, - "api_key": { - "title": "API Key", - "description": "To get Keen Master API Key, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", - "type": "string", - "examples": ["ABCDEFGHIJKLMNOPRSTUWXYZ"], - "airbyte_secret": true - }, - "infer_timestamp": { - "title": "Infer Timestamp", - "description": "Allow connector to guess keen.timestamp value based on the streamed data.", - "type": "boolean", - "default": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ce0d828e-1dc4-496c-b122-2da42e637e48", - "name": "Clickhouse", - "dockerRepository": "airbyte/destination-clickhouse", - "dockerImageTag": "0.1.11", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ClickHouse Destination Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "HTTP port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 3 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "ssl": { - "title": "SSL Connection", - "description": "Encrypt data using SSL.", - "type": "boolean", - "default": false, - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "072d5540-f236-4294-ba7c-ade8fd918496", - "name": "Databricks Lakehouse", - "dockerRepository": "airbyte/destination-databricks", - "dockerImageTag": "0.2.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/databricks", - "icon": "databricks.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/databricks", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Databricks Lakehouse Destination Spec", - "type": "object", - "required": [ - "accept_terms", - "databricks_server_hostname", - "databricks_http_path", - "databricks_personal_access_token", - "data_source" - ], - "properties": { - "accept_terms": { - "title": "Agree to the Databricks JDBC Driver Terms & Conditions", - "type": "boolean", - "description": "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector.", - "default": false, - "order": 1 - }, - "databricks_server_hostname": { - "title": "Server Hostname", - "type": "string", - "description": "Databricks Cluster Server Hostname.", - "examples": ["abc-12345678-wxyz.cloud.databricks.com"], - "order": 2 - }, - "databricks_http_path": { - "title": "HTTP Path", - "type": "string", - "description": "Databricks Cluster HTTP Path.", - "examples": ["sql/protocolvx/o/1234567489/0000-1111111-abcd90"], - "order": 3 - }, - "databricks_port": { - "title": "Port", - "type": "string", - "description": "Databricks Cluster Port.", - "default": "443", - "examples": ["443"], - "order": 4 - }, - "databricks_personal_access_token": { - "title": "Access Token", - "type": "string", - "description": "Databricks Personal Access Token for making authenticated requests.", - "examples": ["dapi0123456789abcdefghij0123456789AB"], - "airbyte_secret": true, - "order": 5 - }, - "database_schema": { - "title": "Database Schema", - "type": "string", - "description": "The default schema tables are written to if the source does not specify a namespace. Unless specifically configured, the usual value for this field is \"public\".", - "default": "public", - "examples": ["public"], - "order": 6 - }, - "data_source": { - "title": "Data Source", - "type": "object", - "description": "Storage on which the delta lake is built.", - "oneOf": [ - { - "title": "Amazon S3", - "required": [ - "data_source_type", - "s3_bucket_name", - "s3_bucket_path", - "s3_bucket_region", - "s3_access_key_id", - "s3_secret_access_key" - ], - "properties": { - "data_source_type": { - "type": "string", - "enum": ["S3"], - "default": "S3", - "order": 1 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the S3 bucket to use for intermittent staging of the data.", - "examples": ["airbyte.staging"], - "order": 2 - }, - "s3_bucket_path": { - "title": "S3 Bucket Path", - "type": "string", - "description": "The directory under the S3 bucket where data will be written.", - "examples": ["data_sync/test"], - "order": 3 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 4 - }, - "s3_access_key_id": { - "type": "string", - "description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.", - "title": "S3 Access Key ID", - "examples": ["A012345678910EXAMPLE"], - "airbyte_secret": true, - "order": 5 - }, - "s3_secret_access_key": { - "title": "S3 Secret Access Key", - "type": "string", - "description": "The corresponding secret to the above access key id.", - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], - "airbyte_secret": true, - "order": 6 - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 7 - } - } - } - ], - "order": 7 - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables", - "type": "boolean", - "description": "Default to 'true'. Switch it to 'false' for debugging purpose.", - "default": true, - "order": 8 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "8ccd8909-4e99-4141-b48d-4984b70b2d89", - "name": "DynamoDB", - "dockerRepository": "airbyte/destination-dynamodb", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/dynamodb", - "icon": "dynamodb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/dynamodb", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "DynamoDB Destination Spec", - "type": "object", - "required": [ - "dynamodb_table_name_prefix", - "dynamodb_region", - "access_key_id", - "secret_access_key" - ], - "additionalProperties": false, - "properties": { - "dynamodb_endpoint": { - "title": "Endpoint", - "type": "string", - "default": "", - "description": "This is your DynamoDB endpoint url.(if you are working with AWS DynamoDB, just leave empty).", - "examples": ["http://localhost:9000"] - }, - "dynamodb_table_name_prefix": { - "title": "Table name prefix", - "type": "string", - "description": "The prefix to use when naming DynamoDB tables.", - "examples": ["airbyte_sync"] - }, - "dynamodb_region": { - "title": "DynamoDB Region", - "type": "string", - "default": "", - "description": "The region of the DynamoDB.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ] - }, - "access_key_id": { - "type": "string", - "description": "The access key id to access the DynamoDB. Airbyte requires Read and Write permissions to the DynamoDB.", - "title": "DynamoDB Key Id", - "airbyte_secret": true, - "examples": ["A012345678910EXAMPLE"] - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the access key id.", - "title": "DynamoDB Access Key", - "airbyte_secret": true, - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "68f351a7-2745-4bef-ad7f-996b8e51bb8c", - "name": "ElasticSearch", - "dockerRepository": "airbyte/destination-elasticsearch", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/elasticsearch", - "icon": "elasticsearch.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/elasticsearch", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Elasticsearch Connection Configuration", - "type": "object", - "required": ["endpoint"], - "additionalProperties": false, - "properties": { - "endpoint": { - "title": "Server Endpoint", - "type": "string", - "description": "The full url of the Elasticsearch server" - }, - "upsert": { - "type": "boolean", - "title": "Upsert Records", - "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", - "default": true - }, - "authenticationMethod": { - "title": "Authentication Method", - "type": "object", - "description": "The type of authentication to be used", - "oneOf": [ - { - "title": "None", - "additionalProperties": false, - "description": "No authentication will be used", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "none" - } - } - }, - { - "title": "Api Key/Secret", - "additionalProperties": false, - "description": "Use a api key and secret combination to authenticate", - "required": ["method", "apiKeyId", "apiKeySecret"], - "properties": { - "method": { - "type": "string", - "const": "secret" - }, - "apiKeyId": { - "title": "API Key ID", - "description": "The Key ID to used when accessing an enterprise Elasticsearch instance.", - "type": "string" - }, - "apiKeySecret": { - "title": "API Key Secret", - "description": "The secret associated with the API Key ID.", - "type": "string", - "airbyte_secret": true - } - } - }, - { - "title": "Username/Password", - "additionalProperties": false, - "description": "Basic auth header with a username and password", - "required": ["method", "username", "password"], - "properties": { - "method": { - "type": "string", - "const": "basic" - }, - "username": { - "title": "Username", - "description": "Basic auth username to access a secure Elasticsearch server", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Basic auth password to access a secure Elasticsearch server", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], - "supportsNamespaces": true - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3", - "name": "End-to-End Testing (/dev/null)", - "dockerRepository": "airbyte/destination-dev-null", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/e2e-test", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/e2e-test", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "E2E Test (/dev/null) Destination Spec", - "type": "object", - "oneOf": [ - { - "title": "Silent", - "required": ["type"], - "properties": { - "type": { - "type": "string", - "const": "SILENT", - "default": "SILENT" - } - } - } - ] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "18081484-02a5-4662-8dba-b270b582f321", - "name": "Firebolt", - "dockerRepository": "airbyte/destination-firebolt", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firebolt", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firebolt", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Firebolt Spec", - "type": "object", - "required": ["username", "password", "database"], - "additionalProperties": false, - "properties": { - "username": { - "type": "string", - "title": "Username", - "description": "Firebolt email address you use to login.", - "examples": ["username@email.com"], - "order": 0 - }, - "password": { - "type": "string", - "title": "Password", - "description": "Firebolt password.", - "airbyte_secret": true, - "order": 1 - }, - "account": { - "type": "string", - "title": "Account", - "description": "Firebolt account to login." - }, - "host": { - "type": "string", - "title": "Host", - "description": "The host name of your Firebolt database.", - "examples": ["api.app.firebolt.io"] - }, - "database": { - "type": "string", - "title": "Database", - "description": "The database to connect to." - }, - "engine": { - "type": "string", - "title": "Engine", - "description": "Engine name or url to connect to." - }, - "loading_method": { - "type": "object", - "title": "Loading Method", - "description": "Loading method used to select the way data will be uploaded to Firebolt", - "oneOf": [ - { - "title": "SQL Inserts", - "additionalProperties": false, - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "SQL" - } - } - }, - { - "title": "External Table via S3", - "additionalProperties": false, - "required": [ - "method", - "s3_bucket", - "s3_region", - "aws_key_id", - "aws_key_secret" - ], - "properties": { - "method": { - "type": "string", - "const": "S3" - }, - "s3_bucket": { - "type": "string", - "title": "S3 bucket name", - "description": "The name of the S3 bucket." - }, - "s3_region": { - "type": "string", - "title": "S3 region name", - "description": "Region name of the S3 bucket.", - "examples": ["us-east-1"] - }, - "aws_key_id": { - "type": "string", - "title": "AWS Key ID", - "airbyte_secret": true, - "description": "AWS access key granting read and write access to S3." - }, - "aws_key_secret": { - "type": "string", - "title": "AWS Key Secret", - "airbyte_secret": true, - "description": "Corresponding secret part of the AWS Key" - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ca8f6566-e555-4b40-943a-545bf123117a", - "name": "Google Cloud Storage (GCS)", - "dockerRepository": "airbyte/destination-gcs", - "dockerImageTag": "0.2.10", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/gcs", - "icon": "googlecloudstorage.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/gcs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "GCS Destination Spec", - "type": "object", - "required": [ - "gcs_bucket_name", - "gcs_bucket_path", - "credential", - "format" - ], - "properties": { - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "order": 1, - "type": "string", - "description": "You can find the bucket name in the App Engine Admin console Application Settings page, under the label Google Cloud Storage Bucket. Read more here.", - "examples": ["airbyte_sync"] - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "GCS Bucket Path string Subdirectory under the above bucket to sync the data into.", - "order": 2, - "type": "string", - "examples": ["data_sync/test"] - }, - "gcs_bucket_region": { - "title": "GCS Bucket Region (Optional)", - "type": "string", - "order": 3, - "default": "us", - "description": "Select a Region of the GCS Bucket. Read more here.", - "enum": [ - "northamerica-northeast1", - "northamerica-northeast2", - "us-central1", - "us-east1", - "us-east4", - "us-west1", - "us-west2", - "us-west3", - "us-west4", - "southamerica-east1", - "southamerica-west1", - "europe-central2", - "europe-north1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "asia", - "eu", - "us", - "asia1", - "eur4", - "nam4" - ] - }, - "credential": { - "title": "Authentication", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 0, - "oneOf": [ - { - "title": "HMAC Key", - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "enum": ["HMAC_KEY"], - "default": "HMAC_KEY" - }, - "hmac_key_access_id": { - "type": "string", - "description": "When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long. Read more here.", - "title": "Access ID", - "airbyte_secret": true, - "order": 0, - "examples": ["1234567890abcdefghij1234"] - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string. Read more here.", - "title": "Secret", - "airbyte_secret": true, - "order": 1, - "examples": ["1234567890abcdefghij1234567890ABCDEFGHIJ"] - } - } - } - ] - }, - "format": { - "title": "Output Format", - "type": "object", - "description": "Output data format. One of the following formats must be selected - AVRO format, PARQUET format, CSV format, or JSONL format.", - "order": 4, - "oneOf": [ - { - "title": "Avro: Apache Avro", - "required": ["format_type", "compression_codec"], - "properties": { - "format_type": { - "type": "string", - "enum": ["Avro"], - "default": "Avro" - }, - "compression_codec": { - "title": "Compression Codec", - "description": "The compression algorithm used to compress data. Default to no compression.", - "type": "object", - "oneOf": [ - { - "title": "No Compression", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["no compression"], - "default": "no compression" - } - } - }, - { - "title": "Deflate", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["Deflate"], - "default": "Deflate" - }, - "compression_level": { - "title": "Deflate level (Optional)", - "description": "0: no compression & fastest, 9: best compression & slowest.", - "type": "integer", - "default": 0, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "bzip2", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["bzip2"], - "default": "bzip2" - } - } - }, - { - "title": "xz", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["xz"], - "default": "xz" - }, - "compression_level": { - "title": "Compression Level (Optional)", - "description": "The presets 0-3 are fast presets with medium compression. The presets 4-6 are fairly slow presets with high compression. The default preset is 6. The presets 7-9 are like the preset 6 but use bigger dictionaries and have higher compressor and decompressor memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively. Read more here for details.", - "type": "integer", - "default": 6, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "zstandard", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["zstandard"], - "default": "zstandard" - }, - "compression_level": { - "title": "Compression Level (Optional)", - "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", - "type": "integer", - "default": 3, - "minimum": -5, - "maximum": 22 - }, - "include_checksum": { - "title": "Include Checksum (Optional)", - "description": "If true, include a checksum with each data block.", - "type": "boolean", - "default": false - } - } - }, - { - "title": "snappy", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["snappy"], - "default": "snappy" - } - } - } - ] - } - } - }, - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["CSV"], - "default": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Optional)", - "description": "Whether the input JSON data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "JSON Lines: newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["JSONL"], - "default": "JSONL" - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "Parquet: Columnar Storage", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "enum": ["Parquet"], - "default": "Parquet" - }, - "compression_codec": { - "title": "Compression Codec (Optional)", - "description": "The compression algorithm used to compress data pages.", - "type": "string", - "default": "UNCOMPRESSED", - "enum": [ - "UNCOMPRESSED", - "SNAPPY", - "GZIP", - "LZO", - "BROTLI", - "LZ4", - "ZSTD" - ] - }, - "block_size_mb": { - "title": "Block Size (Row Group Size) (MB) (Optional)", - "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", - "type": "integer", - "default": 128, - "examples": [128] - }, - "max_padding_size_mb": { - "title": "Max Padding Size (MB) (Optional)", - "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", - "type": "integer", - "default": 8, - "examples": [8] - }, - "page_size_kb": { - "title": "Page Size (KB) (Optional)", - "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_page_size_kb": { - "title": "Dictionary Page Size (KB) (Optional)", - "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_encoding": { - "title": "Dictionary Encoding (Optional)", - "description": "Default: true.", - "type": "boolean", - "default": true - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], - "$schema": "http://json-schema.org/draft-07/schema#" - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4", - "name": "Google Firestore", - "dockerRepository": "airbyte/destination-firestore", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firestore", - "icon": "firestore.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/firestore", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Google Firestore", - "type": "object", - "required": ["project_id"], - "additionalProperties": false, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset.", - "title": "Project ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append", "overwrite"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "356668e2-7e34-47f3-a3b0-67a8a481b692", - "name": "Google PubSub", - "dockerRepository": "airbyte/destination-pubsub", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pubsub", - "icon": "googlepubsub.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pubsub", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google PubSub Destination Spec", - "type": "object", - "required": ["project_id", "topic_id", "credentials_json"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target PubSub.", - "title": "Project ID" - }, - "topic_id": { - "type": "string", - "description": "The PubSub topic ID in the given GCP project ID.", - "title": "PubSub Topic ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a", - "name": "Google Sheets", - "dockerRepository": "airbyte/destination-google-sheets", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/google-sheets", - "icon": "google-sheets.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Google Sheets", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": false, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "The link to your spreadsheet. See this guide for more details.", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit" - ] - }, - "credentials": { - "type": "object", - "title": "* Authentication via Google (OAuth)", - "description": "Google API Credentials for connecting to Google Sheets and Google Drive APIs", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Sheets developer application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Sheets developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining new access token.", - "airbyte_secret": true - } - } - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "d4353156-9217-4cad-8dd7-c108fd4f74cf", - "name": "MS SQL Server", - "dockerRepository": "airbyte/destination-mssql-strict-encrypt", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "icon": "mssql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MS SQL Server Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database", "schema"], - "properties": { - "host": { - "title": "Host", - "description": "The host name of the MSSQL database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The port of the MSSQL database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1433, - "examples": ["1433"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "The name of the MSSQL database.", - "type": "string", - "order": 2 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "order": 3 - }, - "username": { - "title": "User", - "description": "The username which is used to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "The password associated with this username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 6 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used to communicate with the database.", - "order": 7, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "type": "object", - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate", - "enum": ["encrypted_trust_server_certificate"], - "default": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": [ - "ssl_method", - "trustStoreName", - "trustStorePassword" - ], - "type": "object", - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 8 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "294a4790-429b-40ae-9516-49826b9702e1", - "name": "MariaDB ColumnStore", - "dockerRepository": "airbyte/destination-mariadb-columnstore", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore", - "icon": "mariadb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MariaDB Columnstore Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "The Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "Username", - "description": "The Username which is used to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "The Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", - "name": "MeiliSearch", - "dockerRepository": "airbyte/destination-meilisearch", - "dockerImageTag": "0.2.13", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch", - "icon": "meilisearch.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MeiliSearch Destination Spec", - "type": "object", - "required": ["host"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the MeiliSearch instance.", - "type": "string", - "order": 0 - }, - "api_key": { - "title": "API Key", - "airbyte_secret": true, - "description": "MeiliSearch API Key. See the docs for more information on how to obtain this key.", - "type": "string", - "order": 1 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c", - "name": "MongoDB", - "dockerRepository": "airbyte/destination-mongodb-strict-encrypt", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb", - "icon": "mongodb.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDB Destination Spec", - "type": "object", - "required": ["database", "auth_type"], - "additionalProperties": true, - "properties": { - "instance_type": { - "description": "MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "title": "MongoDb Instance Type", - "type": "object", - "order": 0, - "oneOf": [ - { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], - "properties": { - "instance": { - "type": "string", - "enum": ["standalone"], - "default": "standalone" - }, - "host": { - "title": "Host", - "type": "string", - "description": "The Host of a Mongo database to be replicated.", - "order": 0 - }, - "port": { - "title": "Port", - "type": "integer", - "description": "The Port of a Mongo database to be replicated.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { - "type": "string", - "enum": ["replica"], - "default": "replica" - }, - "server_addresses": { - "title": "Server addresses", - "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member seperated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "A replica set name.", - "order": 1 - } - } - }, - { - "title": "MongoDB Atlas", - "additionalProperties": false, - "required": ["instance", "cluster_url"], - "properties": { - "instance": { - "type": "string", - "enum": ["atlas"], - "default": "atlas" - }, - "cluster_url": { - "title": "Cluster URL", - "type": "string", - "description": "URL of a cluster to connect to.", - "order": 0 - } - } - } - ] - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "auth_type": { - "title": "Authorization type", - "type": "object", - "description": "Authorization type.", - "oneOf": [ - { - "title": "None", - "additionalProperties": false, - "description": "None.", - "required": ["authorization"], - "type": "object", - "properties": { - "authorization": { - "type": "string", - "const": "none" - } - } - }, - { - "title": "Login/Password", - "additionalProperties": false, - "description": "Login/Password.", - "required": ["authorization", "username", "password"], - "type": "object", - "properties": { - "authorization": { - "type": "string", - "const": "login/password" - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 2 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "ca81ee7c-3163-4246-af40-094cc31e5e42", - "name": "MySQL", - "dockerRepository": "airbyte/destination-mysql-strict-encrypt", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql", - "icon": "mysql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySQL Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", - "name": "Oracle", - "dockerRepository": "airbyte/destination-oracle-strict-encrypt", - "dockerImageTag": "0.1.19", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", - "icon": "oracle.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Oracle Destination Spec", - "type": "object", - "required": ["host", "port", "username", "sid"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "The hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "The port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1521, - "examples": ["1521"], - "order": 1 - }, - "sid": { - "title": "SID", - "description": "The System Identifier uniquely distinguishes the instance from any other instance on the same computer.", - "type": "string", - "order": 2 - }, - "username": { - "title": "User", - "description": "The username to access the database. This user must have CREATE USER privileges in the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. The usual value for this field is \"airbyte\". In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.", - "type": "string", - "examples": ["airbyte"], - "default": "airbyte", - "order": 6 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", - "name": "Postgres", - "dockerRepository": "airbyte/destination-postgres-strict-encrypt", - "dockerImageTag": "0.3.22", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", - "icon": "postgresql.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Destination Spec", - "type": "object", - "required": ["host", "port", "username", "database", "schema"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "order": 3 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs.", - "type": "object", - "order": 7, - "oneOf": [ - { - "title": "prefer", - "additionalProperties": false, - "description": "Prefer SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "prefer", - "enum": ["prefer"], - "default": "prefer", - "order": 0 - } - } - }, - { - "title": "require", - "additionalProperties": false, - "description": "Require SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "require", - "enum": ["require"], - "default": "require", - "order": 0 - } - } - }, - { - "title": "verify-ca", - "additionalProperties": false, - "description": "Verify-ca SSL mode.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-ca", - "enum": ["verify-ca"], - "default": "verify-ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "verify-full", - "additionalProperties": false, - "description": "Verify-full SSL mode.", - "required": [ - "mode", - "ca_certificate", - "client_certificate", - "client_key" - ], - "properties": { - "mode": { - "type": "string", - "const": "verify-full", - "enum": ["verify-full"], - "default": "verify-full", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2 - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 8 - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "2340cbba-358e-11ec-8d3d-0242ac130203", - "name": "Pulsar", - "dockerRepository": "airbyte/destination-pulsar", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pulsar", - "icon": "pulsar.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/pulsar", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pulsar Destination Spec", - "type": "object", - "required": [ - "brokers", - "use_tls", - "topic_type", - "topic_tenant", - "topic_namespace", - "topic_pattern", - "compression_type", - "send_timeout_ms", - "max_pending_messages", - "max_pending_messages_across_partitions", - "batching_enabled", - "batching_max_messages", - "batching_max_publish_delay", - "block_if_queue_full" - ], - "additionalProperties": true, - "properties": { - "brokers": { - "title": "Pulsar brokers", - "description": "A list of host/port pairs to use for establishing the initial connection to the Pulsar cluster.", - "type": "string", - "examples": ["broker1:6650,broker2:6650"] - }, - "use_tls": { - "title": "Use TLS", - "description": "Whether to use TLS encryption on the connection.", - "type": "boolean", - "default": false - }, - "topic_type": { - "title": "Topic type", - "description": "It identifies type of topic. Pulsar supports two kind of topics: persistent and non-persistent. In persistent topic, all messages are durably persisted on disk (that means on multiple disks unless the broker is standalone), whereas non-persistent topic does not persist message into storage disk.", - "type": "string", - "default": "persistent", - "enum": ["persistent", "non-persistent"] - }, - "topic_tenant": { - "title": "Topic tenant", - "description": "The topic tenant within the instance. Tenants are essential to multi-tenancy in Pulsar, and spread across clusters.", - "type": "string", - "default": "public", - "examples": ["public"] - }, - "topic_namespace": { - "title": "Topic namespace", - "description": "The administrative unit of the topic, which acts as a grouping mechanism for related topics. Most topic configuration is performed at the namespace level. Each tenant has one or multiple namespaces.", - "type": "string", - "default": "default", - "examples": ["default"] - }, - "topic_pattern": { - "title": "Topic pattern", - "description": "Topic pattern in which the records will be sent. You can use patterns like '{namespace}' and/or '{stream}' to send the message to a specific topic based on these values. Notice that the topic name will be transformed to a standard naming convention.", - "type": "string", - "examples": ["sample.topic", "{namespace}.{stream}.sample"] - }, - "topic_test": { - "title": "Test topic", - "description": "Topic to test if Airbyte can produce messages.", - "type": "string", - "examples": ["test.topic"] - }, - "producer_name": { - "title": "Producer name", - "description": "Name for the producer. If not filled, the system will generate a globally unique name which can be accessed with.", - "type": "string", - "examples": ["airbyte-producer"] - }, - "producer_sync": { - "title": "Sync producer", - "description": "Wait synchronously until the record has been sent to Pulsar.", - "type": "boolean", - "default": false - }, - "compression_type": { - "title": "Compression type", - "description": "Compression type for the producer.", - "type": "string", - "default": "NONE", - "enum": ["NONE", "LZ4", "ZLIB", "ZSTD", "SNAPPY"] - }, - "send_timeout_ms": { - "title": "Message send timeout", - "description": "If a message is not acknowledged by a server before the send-timeout expires, an error occurs (in ms).", - "type": "integer", - "default": 30000 - }, - "max_pending_messages": { - "title": "Max pending messages", - "description": "The maximum size of a queue holding pending messages.", - "type": "integer", - "default": 1000 - }, - "max_pending_messages_across_partitions": { - "title": "Max pending messages across partitions", - "description": "The maximum number of pending messages across partitions.", - "type": "integer", - "default": 50000 - }, - "batching_enabled": { - "title": "Enable batching", - "description": "Control whether automatic batching of messages is enabled for the producer.", - "type": "boolean", - "default": true - }, - "batching_max_messages": { - "title": "Batching max messages", - "description": "Maximum number of messages permitted in a batch.", - "type": "integer", - "default": 1000 - }, - "batching_max_publish_delay": { - "title": "Batching max publish delay", - "description": " Time period in milliseconds within which the messages sent will be batched.", - "type": "integer", - "default": 1 - }, - "block_if_queue_full": { - "title": "Block if queue is full", - "description": "If the send operation should block when the outgoing message queue is full.", - "type": "boolean", - "default": false - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "e06ad785-ad6f-4647-b2e8-3027a5c59454", - "name": "RabbitMQ", - "dockerRepository": "airbyte/destination-rabbitmq", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rabbitmq", - "icon": "pulsar.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rabbitmq", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Rabbitmq", - "type": "object", - "required": ["host", "routing_key"], - "additionalProperties": false, - "properties": { - "ssl": { - "type": "boolean", - "description": "SSL enabled.", - "default": true - }, - "host": { - "type": "string", - "description": "The RabbitMQ host name." - }, - "port": { - "type": "integer", - "description": "The RabbitMQ port." - }, - "virtual_host": { - "type": "string", - "description": "The RabbitMQ virtual host name." - }, - "username": { - "type": "string", - "description": "The username to connect." - }, - "password": { - "type": "string", - "description": "The password to connect." - }, - "exchange": { - "type": "string", - "description": "The exchange name." - }, - "routing_key": { - "type": "string", - "description": "The routing key." - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", - "name": "Redis", - "dockerRepository": "airbyte/destination-redis", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redis", - "icon": "redis.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redis", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redis Destination Spec", - "type": "object", - "required": ["host", "port", "username", "password", "cache_type"], - "additionalProperties": false, - "properties": { - "host": { - "title": "Host", - "description": "Redis host to connect to.", - "type": "string", - "examples": ["localhost,127.0.0.1"], - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of Redis.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 6379, - "order": 2 - }, - "username": { - "title": "Username", - "description": "Username associated with Redis.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with Redis.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "cache_type": { - "title": "Cache type", - "type": "string", - "default": "hash", - "description": "Redis cache type to store data in.", - "enum": ["hash"], - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", - "name": "Redshift", - "dockerRepository": "airbyte/destination-redshift", - "dockerImageTag": "0.3.47", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "icon": "redshift.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redshift Destination Spec", - "type": "object", - "required": [ - "host", - "port", - "database", - "username", - "password", - "schema" - ], - "additionalProperties": true, - "properties": { - "host": { - "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com)", - "type": "string", - "title": "Host" - }, - "port": { - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5439, - "examples": ["5439"], - "title": "Port" - }, - "username": { - "description": "Username to use to access the database.", - "type": "string", - "title": "Username" - }, - "password": { - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password" - }, - "database": { - "description": "Name of the database.", - "type": "string", - "title": "Database" - }, - "schema": { - "description": "The default schema tables are written to if the source does not specify a namespace. Unless specifically configured, the usual value for this field is \"public\".", - "type": "string", - "examples": ["public"], - "default": "public", - "title": "Default Schema" - }, - "uploading_method": { - "title": "Uploading Method", - "type": "object", - "description": "The method how the data will be uploaded to the database.", - "oneOf": [ - { - "title": "Standard", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "S3 Staging", - "required": [ - "method", - "s3_bucket_name", - "s3_bucket_region", - "access_key_id", - "secret_access_key" - ], - "properties": { - "method": { - "type": "string", - "const": "S3 Staging" - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the staging S3 bucket to use if utilising a COPY strategy. COPY is recommended for production workloads for better speed and scalability. See AWS docs for more details.", - "examples": ["airbyte.staging"] - }, - "s3_bucket_path": { - "title": "S3 Bucket Path (Optional)", - "type": "string", - "description": "The directory under the S3 bucket where data will be written. If not provided, then defaults to the root directory. See path's name recommendations for more details.", - "examples": ["data_sync/test"] - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 staging bucket to use if utilising a COPY strategy. See AWS docs for details.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1" - ] - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 8 - }, - "access_key_id": { - "type": "string", - "description": "This ID grants access to the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Key Id", - "airbyte_secret": true - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the above access key id. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Access Key", - "airbyte_secret": true - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables (Optional)", - "type": "boolean", - "description": "Whether to delete the staging files from S3 after completing the sync. See docs for details.", - "default": true - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "How to encrypt the staging data", - "default": { - "encryption_type": "none" - }, - "oneOf": [ - { - "title": "No encryption", - "description": "Staging data will be stored in plaintext.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "none", - "enum": ["none"], - "default": "none" - } - } - }, - { - "title": "AES-CBC envelope encryption", - "description": "Staging data will be encrypted using AES-CBC envelope encryption.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "aes_cbc_envelope", - "enum": ["aes_cbc_envelope"], - "default": "aes_cbc_envelope" - }, - "key_encrypting_key": { - "type": "string", - "title": "Key", - "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", - "airbyte_secret": true - } - } - } - ] - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "beta", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "2c9d93a7-9a17-4789-9de9-f46f0097eb70", - "name": "Rockset", - "dockerRepository": "airbyte/destination-rockset", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rockset", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/rockset", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Rockset Destination Spec", - "type": "object", - "required": ["api_key", "workspace"], - "additionalProperties": false, - "properties": { - "api_key": { - "title": "Api Key", - "description": "Rockset api key", - "type": "string", - "order": 0, - "airbyte_secret": true - }, - "workspace": { - "title": "Workspace", - "description": "The Rockset workspace in which collections will be created + written to.", - "type": "string", - "examples": ["commons", "my_workspace"], - "default": "commons", - "airbyte_secret": false, - "order": 1 - }, - "api_server": { - "title": "Api Server", - "description": "Rockset api URL", - "type": "string", - "airbyte_secret": false, - "default": "https://api.rs2.usw2.rockset.com", - "pattern": "^https:\\/\\/.*.rockset.com$", - "order": 2 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append", "overwrite"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", - "name": "S3", - "dockerRepository": "airbyte/destination-s3", - "dockerImageTag": "0.3.14", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", - "icon": "s3.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", - "protocol_version": "0.2.2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "S3 Destination Spec", - "type": "object", - "required": [ - "s3_bucket_name", - "s3_bucket_path", - "s3_bucket_region", - "format" - ], - "properties": { - "access_key_id": { - "type": "string", - "description": "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", - "title": "S3 Key ID *", - "airbyte_secret": true, - "examples": ["A012345678910EXAMPLE"], - "order": 0 - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the access key ID. Read more here", - "title": "S3 Access Key *", - "airbyte_secret": true, - "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], - "order": 1 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "The name of the S3 bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "s3_bucket_path": { - "title": "S3 Bucket Path", - "description": "Directory under the S3 bucket where data will be written. Read more here", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "The region of the S3 bucket. See here for all region codes.", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 4 - }, - "format": { - "title": "Output Format *", - "type": "object", - "description": "Format of the data output. See here for more details", - "oneOf": [ - { - "title": "Avro: Apache Avro", - "required": ["format_type", "compression_codec"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["Avro"], - "default": "Avro", - "order": 0 - }, - "compression_codec": { - "title": "Compression Codec *", - "description": "The compression algorithm used to compress data. Default to no compression.", - "type": "object", - "oneOf": [ - { - "title": "No Compression", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["no compression"], - "default": "no compression" - } - } - }, - { - "title": "Deflate", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["Deflate"], - "default": "Deflate" - }, - "compression_level": { - "title": "Deflate Level", - "description": "0: no compression & fastest, 9: best compression & slowest.", - "type": "integer", - "default": 0, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "bzip2", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["bzip2"], - "default": "bzip2" - } - } - }, - { - "title": "xz", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["xz"], - "default": "xz" - }, - "compression_level": { - "title": "Compression Level", - "description": "See here for details.", - "type": "integer", - "default": 6, - "minimum": 0, - "maximum": 9 - } - } - }, - { - "title": "zstandard", - "required": ["codec", "compression_level"], - "properties": { - "codec": { - "type": "string", - "enum": ["zstandard"], - "default": "zstandard" - }, - "compression_level": { - "title": "Compression Level", - "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", - "type": "integer", - "default": 3, - "minimum": -5, - "maximum": 22 - }, - "include_checksum": { - "title": "Include Checksum", - "description": "If true, include a checksum with each data block.", - "type": "boolean", - "default": false - } - } - }, - { - "title": "snappy", - "required": ["codec"], - "properties": { - "codec": { - "type": "string", - "enum": ["snappy"], - "default": "snappy" - } - } - } - ], - "order": 1 - } - } - }, - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type", "flattening"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["CSV"], - "default": "CSV" - }, - "flattening": { - "type": "string", - "title": "Normalization (Flattening)", - "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "JSON Lines: Newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["JSONL"], - "default": "JSONL" - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "Parquet: Columnar Storage", - "required": ["format_type"], - "properties": { - "format_type": { - "title": "Format Type *", - "type": "string", - "enum": ["Parquet"], - "default": "Parquet" - }, - "compression_codec": { - "title": "Compression Codec (Optional)", - "description": "The compression algorithm used to compress data pages.", - "type": "string", - "enum": [ - "UNCOMPRESSED", - "SNAPPY", - "GZIP", - "LZO", - "BROTLI", - "LZ4", - "ZSTD" - ], - "default": "UNCOMPRESSED" - }, - "block_size_mb": { - "title": "Block Size (Row Group Size) (MB) (Optional)", - "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", - "type": "integer", - "default": 128, - "examples": [128] - }, - "max_padding_size_mb": { - "title": "Max Padding Size (MB) (Optional)", - "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", - "type": "integer", - "default": 8, - "examples": [8] - }, - "page_size_kb": { - "title": "Page Size (KB) (Optional)", - "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_page_size_kb": { - "title": "Dictionary Page Size (KB) (Optional)", - "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", - "type": "integer", - "default": 1024, - "examples": [1024] - }, - "dictionary_encoding": { - "title": "Dictionary Encoding (Optional)", - "description": "Default: true.", - "type": "boolean", - "default": true - } - } - } - ], - "order": 5 - }, - "s3_endpoint": { - "title": "Endpoint (Optional)", - "type": "string", - "default": "", - "description": "Your S3 endpoint url. Read more here", - "examples": ["http://localhost:9000"], - "order": 6 - }, - "s3_path_format": { - "title": "S3 Path Format (Optional)", - "description": "Format string on how data will be organized inside the S3 bucket directory. Read more here", - "type": "string", - "examples": [ - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_" - ], - "order": 7 - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 8 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - }, - { - "destinationDefinitionId": "e9810f61-4bab-46d2-bb22-edfc902e0644", - "name": "SFTP-JSON", - "dockerRepository": "airbyte/destination-sftp-json", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/sftp-json", - "icon": "sftp.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/sftp-json", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination SFTP JSON", - "type": "object", - "required": ["host", "username", "password", "destination_path"], - "additionalProperties": false, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the SFTP server.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the SFTP server.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": [22], - "order": 1 - }, - "username": { - "title": "User", - "description": "Username to use to access the SFTP server.", - "type": "string", - "order": 2 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "destination_path": { - "title": "Destination path", - "type": "string", - "description": "Path to the directory where json files will be written.", - "examples": ["/json_data"], - "order": 4 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "3dc6f384-cd6b-4be3-ad16-a41450899bf0", - "name": "Scylla", - "dockerRepository": "airbyte/destination-scylla", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/scylla", - "icon": "scylla.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/scylla", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Scylla Destination Spec", - "type": "object", - "required": ["keyspace", "username", "password", "address", "port"], - "additionalProperties": true, - "properties": { - "keyspace": { - "title": "Keyspace", - "description": "Default Scylla keyspace to create data in.", - "type": "string", - "order": 0 - }, - "username": { - "title": "Username", - "description": "Username to use to access Scylla.", - "type": "string", - "order": 1 - }, - "password": { - "title": "Password", - "description": "Password associated with Scylla.", - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "address": { - "title": "Address", - "description": "Address to connect to.", - "type": "string", - "order": 3 - }, - "port": { - "title": "Port", - "description": "Port of Scylla.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9042, - "order": 4 - }, - "replication": { - "title": "Replication factor", - "type": "integer", - "description": "Indicates to how many nodes the data should be replicated to.", - "default": 1, - "order": 5 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", - "name": "Snowflake", - "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.4.34", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", - "icon": "snowflake.svg", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snowflake Destination Spec", - "type": "object", - "required": [ - "host", - "role", - "warehouse", - "database", - "schema", - "username" - ], - "additionalProperties": true, - "properties": { - "host": { - "description": "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)", - "examples": [ - "accountname.us-east-2.aws.snowflakecomputing.com", - "accountname.snowflakecomputing.com" - ], - "type": "string", - "title": "Host", - "order": 0 - }, - "role": { - "description": "Enter the role that you want to use to access Snowflake", - "examples": ["AIRBYTE_ROLE"], - "type": "string", - "title": "Role", - "order": 1 - }, - "warehouse": { - "description": "Enter the name of the warehouse that you want to sync data into", - "examples": ["AIRBYTE_WAREHOUSE"], - "type": "string", - "title": "Warehouse", - "order": 2 - }, - "database": { - "description": "Enter the name of the database you want to sync data into", - "examples": ["AIRBYTE_DATABASE"], - "type": "string", - "title": "Database", - "order": 3 - }, - "schema": { - "description": "Enter the name of the default schema", - "examples": ["AIRBYTE_SCHEMA"], - "type": "string", - "title": "Default Schema", - "order": 4 - }, - "username": { - "description": "Enter the name of the user you want to use to access the database", - "examples": ["AIRBYTE_USER"], - "type": "string", - "title": "Username", - "order": 5 - }, - "credentials": { - "title": "Authorization Method", - "description": "", - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "order": 0, - "required": ["access_token", "refresh_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "OAuth2.0", - "enum": ["OAuth2.0"], - "default": "OAuth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "Enter your application's Client ID", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "Enter your application's Client secret", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Enter you application's Access Token", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Enter your application's Refresh Token", - "airbyte_secret": true - } - } - }, - { - "title": "Key Pair Authentication", - "type": "object", - "order": 1, - "required": ["private_key"], - "properties": { - "auth_type": { - "type": "string", - "const": "Key Pair Authentication", - "enum": ["Key Pair Authentication"], - "default": "Key Pair Authentication", - "order": 0 - }, - "private_key": { - "type": "string", - "title": "Private Key", - "description": "RSA Private key to use for Snowflake connection. See the docs for more information on how to obtain this key.", - "multiline": true, - "airbyte_secret": true - }, - "private_key_password": { - "type": "string", - "title": "Passphrase (Optional)", - "description": "Passphrase for private key", - "airbyte_secret": true - } - } - }, - { - "title": "Username and Password", - "type": "object", - "required": ["password"], - "order": 2, - "properties": { - "password": { - "description": "Enter the password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password", - "order": 1 - } - } - } - ], - "order": 6 - }, - "jdbc_url_params": { - "description": "Enter the additional properties to pass to the JDBC URL string when connecting to the database (formatted as key=value pairs separated by the symbol &). Example: key1=value1&key2=value2&key3=value3", - "title": "JDBC URL Params", - "type": "string", - "order": 7 - }, - "loading_method": { - "type": "object", - "title": "Data Staging Method", - "description": "Select a data staging method", - "order": 8, - "oneOf": [ - { - "title": "Select another option", - "description": "Select another option", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Standard"], - "default": "Standard" - } - } - }, - { - "title": "[Recommended] Internal Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Internal Staging"], - "default": "Internal Staging" - } - } - }, - { - "title": "AWS S3 Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "s3_bucket_name", - "access_key_id", - "secret_access_key" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["S3 Staging"], - "default": "S3 Staging", - "order": 0 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "Enter your S3 bucket name", - "examples": ["airbyte.staging"], - "order": 1 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "Enter the region where your S3 bucket resides", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-south-1", - "eu-north-1", - "sa-east-1", - "me-south-1" - ], - "order": 2 - }, - "access_key_id": { - "type": "string", - "description": "Enter your AWS access key ID. Airbyte requires Read and Write permissions on your S3 bucket ", - "title": "AWS access key ID", - "airbyte_secret": true, - "order": 3 - }, - "secret_access_key": { - "type": "string", - "description": "Enter your AWS secret access key", - "title": "AWS secret access key", - "airbyte_secret": true, - "order": 4 - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables", - "type": "boolean", - "description": "Toggle to delete staging files from the S3 bucket after a successful sync", - "default": true, - "order": 5 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "Choose a data encryption method for the staging data", - "default": { - "encryption_type": "none" - }, - "order": 6, - "oneOf": [ - { - "title": "No encryption", - "description": "Staging data will be stored in plaintext.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "none", - "enum": ["none"], - "default": "none" - } - } - }, - { - "title": "AES-CBC envelope encryption", - "description": "Staging data will be encrypted using AES-CBC envelope encryption.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "aes_cbc_envelope", - "enum": ["aes_cbc_envelope"], - "default": "aes_cbc_envelope" - }, - "key_encrypting_key": { - "type": "string", - "title": "Key", - "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", - "airbyte_secret": true - } - } - } - ] - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern (Optional)", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 7 - } - } - }, - { - "title": "Google Cloud Storage Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "project_id", - "bucket_name", - "credentials_json" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["GCS Staging"], - "default": "GCS Staging", - "order": 0 - }, - "project_id": { - "title": "Google Cloud project ID", - "type": "string", - "description": "Enter the Google Cloud project ID", - "examples": ["my-project"], - "order": 1 - }, - "bucket_name": { - "title": "Cloud Storage bucket name", - "type": "string", - "description": "Enter the Cloud Storage bucket name", - "examples": ["airbyte-staging"], - "order": 2 - }, - "credentials_json": { - "title": "Google Application Credentials", - "type": "string", - "description": "Enter your Google Cloud service account key in the JSON format with read/write access to your Cloud Storage staging bucket", - "airbyte_secret": true, - "multiline": true, - "order": 3 - } - } - }, - { - "title": "Azure Blob Storage Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "azure_blob_storage_account_name", - "azure_blob_storage_container_name", - "azure_blob_storage_sas_token" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Azure Blob Staging"], - "default": "Azure Blob Staging", - "order": 0 - }, - "azure_blob_storage_endpoint_domain_name": { - "title": "Azure Blob Storage Endpoint", - "type": "string", - "default": "blob.core.windows.net", - "description": "Enter the Azure Blob Storage endpoint domain name", - "examples": ["blob.core.windows.net"], - "order": 1 - }, - "azure_blob_storage_account_name": { - "title": "Azure Blob Storage account name", - "type": "string", - "description": "Enter your Azure Blob Storage account name", - "examples": ["airbyte5storage"], - "order": 2 - }, - "azure_blob_storage_container_name": { - "title": "Azure Blob Storage Container Name", - "type": "string", - "description": "Enter your Azure Blob Storage container name", - "examples": ["airbytetestcontainername"], - "order": 3 - }, - "azure_blob_storage_sas_token": { - "title": "SAS Token", - "type": "string", - "airbyte_secret": true, - "description": "Enter the Shared access signature (SAS) token to grant Snowflake limited access to objects in your Azure Blob Storage account", - "examples": [ - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D" - ], - "order": 4 - } - } - } - ] - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "OAuth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "host": { - "type": "string", - "path_in_connector_config": ["host"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available", - "resourceRequirements": { - "jobSpecific": [ - { - "jobType": "sync", - "resourceRequirements": { - "memory_request": "1Gi", - "memory_limit": "1Gi" - } - } - ] - } - } - ], - "sources": [ - { - "sourceDefinitionId": "6ff047c0-f5d5-4ce5-8c81-204a830fa7e1", - "name": "AWS CloudTrail", - "dockerRepository": "airbyte/source-aws-cloudtrail", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/aws-cloudtrail", - "icon": "awscloudtrail.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/aws-cloudtrail", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Aws CloudTrail Spec", - "type": "object", - "required": [ - "aws_key_id", - "aws_secret_key", - "aws_region_name", - "start_date" - ], - "additionalProperties": true, - "properties": { - "aws_key_id": { - "type": "string", - "title": "Key ID", - "description": "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "aws_secret_key": { - "type": "string", - "title": "Secret Key", - "description": "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "aws_region_name": { - "type": "string", - "title": "Region Name", - "description": "The default AWS Region to use, for example, us-west-1 or us-west-2. When specifying a Region inline during client initialization, this property is named region_name." - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date you would like to replicate data. Data in AWS CloudTrail is available for last 90 days only. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "default": "1970-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212", - "name": "Airtable", - "dockerRepository": "airbyte/source-airtable", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/airtable", - "icon": "airtable.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/airtable", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Airtable Source Spec", - "type": "object", - "required": ["api_key", "base_id", "tables"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "The API Key for the Airtable account. See the Support Guide for more information on how to obtain this key.", - "title": "API Key", - "airbyte_secret": true, - "examples": ["key1234567890"] - }, - "base_id": { - "type": "string", - "description": "The Base ID to integrate the data from. You can find the Base ID following the link Airtable API, log in to your account, select the base you need and find Base ID in the docs.", - "title": "Base ID", - "examples": ["app1234567890"] - }, - "tables": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The list of Tables to integrate.", - "title": "Tables", - "examples": ["table 1", "table 2"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", - "name": "Amazon Ads", - "dockerRepository": "airbyte/source-amazon-ads", - "dockerImageTag": "0.1.18", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-ads", - "icon": "amazonads.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-ads", - "connectionSpecification": { - "title": "Amazon Ads Spec", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "client_id": { - "title": "Client ID", - "description": "The client ID of your Amazon Ads developer application. See the docs for more information.", - "order": 1, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The client secret of your Amazon Ads developer application. See the docs for more information.", - "airbyte_secret": true, - "order": 2, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Amazon Ads refresh token. See the docs for more information on how to obtain this token.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "region": { - "title": "Region *", - "description": "Region to pull data from (EU/NA/FE). See docs for more details.", - "enum": ["NA", "EU", "FE"], - "type": "string", - "default": "NA", - "order": 4 - }, - "report_wait_timeout": { - "title": "Report Wait Timeout *", - "description": "Timeout duration in minutes for Reports. Default is 30 minutes.", - "default": 30, - "examples": [30, 120], - "order": 5, - "type": "integer" - }, - "report_generation_max_retries": { - "title": "Report Generation Maximum Retries *", - "description": "Maximum retries Airbyte will attempt for fetching report data. Default is 5.", - "default": 5, - "examples": [5, 10, 15], - "order": 6, - "type": "integer" - }, - "start_date": { - "title": "Start Date (Optional)", - "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", - "examples": ["2022-10-10", "2022-10-22"], - "order": 7, - "type": "string" - }, - "profiles": { - "title": "Profile IDs (Optional)", - "description": "Profile IDs you want to fetch data for. See docs for more details.", - "order": 8, - "type": "array", - "items": { - "type": "integer" - } - } - }, - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "983fd355-6bf3-4709-91b5-37afa391eeb6", - "name": "Amazon SQS", - "dockerRepository": "airbyte/source-amazon-sqs", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-sqs", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-sqs", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Amazon SQS Source Spec", - "type": "object", - "required": ["queue_url", "region", "delete_messages"], - "additionalProperties": false, - "properties": { - "queue_url": { - "title": "Queue URL", - "description": "URL of the SQS Queue", - "type": "string", - "examples": [ - "https://sqs.eu-west-1.amazonaws.com/1234567890/my-example-queue" - ], - "order": 0 - }, - "region": { - "title": "AWS Region", - "description": "AWS Region of the SQS Queue", - "type": "string", - "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "me-south-1", - "us-gov-east-1", - "us-gov-west-1" - ], - "order": 1 - }, - "delete_messages": { - "title": "Delete Messages After Read", - "description": "If Enabled, messages will be deleted from the SQS Queue after being read. If Disabled, messages are left in the queue and can be read more than once. WARNING: Enabling this option can result in data loss in cases of failure, use with caution, see documentation for more detail. ", - "type": "boolean", - "default": false, - "order": 2 - }, - "max_batch_size": { - "title": "Max Batch Size", - "description": "Max amount of messages to get in one batch (10 max)", - "type": "integer", - "examples": ["5"], - "order": 3 - }, - "max_wait_time": { - "title": "Max Wait Time", - "description": "Max amount of time in seconds to wait for messages in a single poll (20 max)", - "type": "integer", - "examples": ["5"], - "order": 4 - }, - "attributes_to_return": { - "title": "Message Attributes To Return", - "description": "Comma separated list of Mesage Attribute names to return", - "type": "string", - "examples": ["attr1,attr2"], - "order": 5 - }, - "visibility_timeout": { - "title": "Message Visibility Timeout", - "description": "Modify the Visibility Timeout of the individual message from the Queue's default (seconds).", - "type": "integer", - "examples": ["15"], - "order": 6 - }, - "access_key": { - "title": "AWS IAM Access Key ID", - "description": "The Access Key ID of the AWS IAM Role to use for pulling messages", - "type": "string", - "examples": ["xxxxxHRNxxx3TBxxxxxx"], - "airbyte_secret": true, - "order": 7 - }, - "secret_key": { - "title": "AWS IAM Secret Key", - "description": "The Secret Key of the AWS IAM Role to use for pulling messages", - "type": "string", - "examples": ["hu+qE5exxxxT6o/ZrKsxxxxxxBhxxXLexxxxxVKz"], - "airbyte_secret": true, - "order": 8 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460", - "name": "Amazon Seller Partner", - "dockerRepository": "airbyte/source-amazon-seller-partner", - "dockerImageTag": "0.2.25", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "icon": "amazonsellerpartner.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/amazon-seller-partner", - "connectionSpecification": { - "title": "Amazon Seller Partner Spec", - "type": "object", - "properties": { - "app_id": { - "title": "App Id *", - "description": "Your Amazon App ID", - "airbyte_secret": true, - "order": 0, - "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 1, - "type": "string" - }, - "lwa_app_id": { - "title": "LWA Client Id", - "description": "Your Login with Amazon Client ID.", - "order": 2, - "type": "string" - }, - "lwa_client_secret": { - "title": "LWA Client Secret", - "description": "Your Login with Amazon Client Secret.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "The Refresh Token obtained via OAuth flow authorization.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 5, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 6, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 7, - "type": "string" - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - }, - "replication_end_date": { - "title": "End Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - }, - "period_in_days": { - "title": "Period In Days", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", - "default": 30, - "examples": ["30", "365"], - "type": "integer" - }, - "report_options": { - "title": "Report Options", - "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", - "examples": [ - "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "type": "string" - }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "type": "integer" - }, - "aws_environment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "region": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "type": "string" - } - }, - "required": [ - "lwa_app_id", - "lwa_client_secret", - "refresh_token", - "aws_access_key", - "aws_secret_key", - "role_arn", - "replication_start_date", - "aws_environment", - "region" - ], - "additionalProperties": true, - "definitions": { - "AWSEnvironment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "AWSRegion": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "type": "string" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "app_id": { - "type": "string", - "path_in_connector_config": ["app_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string" - }, - "lwa_client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string", - "path_in_connector_config": ["lwa_app_id"] - }, - "lwa_client_secret": { - "type": "string", - "path_in_connector_config": ["lwa_client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "fa9f58c6-2d03-4237-aaa4-07d75e0c1396", - "name": "Amplitude", - "dockerRepository": "airbyte/source-amplitude", - "dockerImageTag": "0.1.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude", - "icon": "amplitude.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Amplitude Spec", - "type": "object", - "required": ["api_key", "secret_key", "start_date"], - "additionalProperties": true, - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "description": "Amplitude API Key. See the setup guide for more information on how to obtain this key.", - "airbyte_secret": true - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Amplitude Secret Key. See the setup guide for more information on how to obtain this key.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Replication Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-01-25T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "47f17145-fe20-4ef5-a548-e29b048adf84", - "name": "Apify Dataset", - "dockerRepository": "airbyte/source-apify-dataset", - "dockerImageTag": "0.1.11", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/apify-dataset", - "icon": "apify.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/apify-dataset", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Apify Dataset Spec", - "type": "object", - "required": ["datasetId"], - "additionalProperties": false, - "properties": { - "datasetId": { - "type": "string", - "title": "Dataset ID", - "description": "ID of the dataset you would like to load to Airbyte." - }, - "clean": { - "type": "boolean", - "title": "Clean", - "description": "If set to true, only clean items will be downloaded from the dataset. See description of what clean means in Apify API docs. If not sure, set clean to false." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "798ae795-5189-42b6-b64e-3cb91db93338", - "name": "Azure Table Storage", - "dockerRepository": "airbyte/source-azure-table", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/azure-table", - "icon": "azureblobstorage.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Azure Data Table Spec", - "type": "object", - "required": ["storage_account_name", "storage_access_key"], - "additionalProperties": false, - "properties": { - "storage_account_name": { - "title": "Account Name", - "type": "string", - "description": "The name of your storage account.", - "order": 0, - "airbyte_secret": false - }, - "storage_access_key": { - "title": "Access Key", - "type": "string", - "description": "Azure Table Storage Access Key. See the docs for more information on how to obtain this key.", - "order": 1, - "airbyte_secret": true - }, - "storage_endpoint_suffix": { - "title": "Endpoint Suffix", - "type": "string", - "description": "Azure Table Storage service account URL suffix. See the docs for more information on how to obtain endpoint suffix", - "order": 2, - "default": "core.windows.net", - "examples": ["core.windows.net", "core.chinacloudapi.cn"], - "airbyte_secret": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "59c5501b-9f95-411e-9269-7143c939adbd", - "name": "BigCommerce", - "dockerRepository": "airbyte/source-bigcommerce", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigcommerce", - "icon": "bigcommerce.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigcommerce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigCommerce Source CDK Specifications", - "type": "object", - "required": ["start_date", "store_hash", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date you would like to replicate data. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "store_hash": { - "type": "string", - "title": "Store Hash", - "description": "The hash code of the store. For https://api.bigcommerce.com/stores/HASH_CODE/v3/, The store's hash code is 'HASH_CODE'." - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c", - "name": "BigQuery", - "dockerRepository": "airbyte/source-bigquery", - "dockerImageTag": "0.2.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigquery", - "icon": "bigquery.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigquery", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Source Spec", - "type": "object", - "required": ["project_id", "credentials_json"], - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset.", - "title": "Project ID" - }, - "dataset_id": { - "type": "string", - "description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery.", - "title": "Default Dataset ID" - }, - "credentials_json": { - "type": "string", - "description": "The contents of your Service Account Key JSON file. See the docs for more information on how to obtain this key.", - "title": "Credentials JSON", - "airbyte_secret": true - } - } - }, - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": [], - "supported_sync_modes": ["overwrite", "append", "append_dedup"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "47f25999-dd5e-4636-8c39-e7cea2453331", - "name": "Bing Ads", - "dockerRepository": "airbyte/source-bing-ads", - "dockerImageTag": "0.1.10", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bing-ads", - "icon": "bingads.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/bing-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Bing Ads Spec", - "type": "object", - "required": [ - "developer_token", - "client_id", - "refresh_token", - "reports_start_date" - ], - "additionalProperties": true, - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0" - }, - "tenant_id": { - "type": "string", - "title": "Tenant ID", - "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", - "airbyte_secret": true, - "default": "common", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Microsoft Advertising developer application.", - "default": "", - "airbyte_secret": true, - "order": 2 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 3 - }, - "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user. See more info in the docs.", - "airbyte_secret": true, - "order": 4 - }, - "reports_start_date": { - "type": "string", - "title": "Reports replication start date", - "format": "date", - "default": "2020-01-01", - "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "tenant_id": { - "type": "string", - "path_in_connector_config": ["tenant_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "63cea06f-1c75-458d-88fe-ad48c7cb27fd", - "name": "Braintree", - "dockerRepository": "airbyte/source-braintree", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/braintree", - "icon": "braintree.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/braintree", - "connectionSpecification": { - "title": "Braintree Spec", - "type": "object", - "properties": { - "merchant_id": { - "title": "Merchant ID", - "description": "The unique identifier for your entire gateway account. See the docs for more information on how to obtain this ID.", - "name": "Merchant ID", - "type": "string" - }, - "public_key": { - "title": "Public Key", - "description": "Braintree Public Key. See the docs for more information on how to obtain this key.", - "name": "Public Key", - "type": "string" - }, - "private_key": { - "title": "Private Key", - "description": "Braintree Private Key. See the docs for more information on how to obtain this key.", - "name": "Private Key", - "airbyte_secret": true, - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "name": "Start Date", - "examples": ["2020", "2020-12-30", "2020-11-22 20:20:05"], - "type": "string", - "format": "date-time" - }, - "environment": { - "title": "Environment", - "description": "Environment specifies where the data will come from.", - "name": "Environment", - "examples": ["sandbox", "production", "qa", "development"], - "enum": ["Development", "Sandbox", "Qa", "Production"], - "type": "string" - } - }, - "required": [ - "merchant_id", - "public_key", - "private_key", - "environment" - ] - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "686473f1-76d9-4994-9cc7-9b13da46147c", - "name": "Chargebee", - "dockerRepository": "airbyte/source-chargebee", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chargebee", - "icon": "chargebee.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://apidocs.chargebee.com/docs/api", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Chargebee Spec", - "type": "object", - "required": ["site", "site_api_key", "start_date", "product_catalog"], - "additionalProperties": true, - "properties": { - "site": { - "type": "string", - "title": "Site", - "description": "The site prefix for your Chargebee instance.", - "examples": ["airbyte-test"] - }, - "site_api_key": { - "type": "string", - "title": "API Key", - "description": "Chargebee API Key. See the docs for more information on how to obtain this key.", - "examples": ["test_3yzfanAXF66USdWC9wQcM555DQJkSYoppu"], - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-01-25T00:00:00Z"] - }, - "product_catalog": { - "title": "Product Catalog", - "type": "string", - "description": "Product Catalog version of your Chargebee site. Instructions on how to find your version you may find here under `API Version` section.", - "enum": ["1.0", "2.0"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "b6604cbd-1b12-4c08-8767-e140d0fb0877", - "name": "Chartmogul", - "dockerRepository": "airbyte/source-chartmogul", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chartmogul", - "icon": "chartmogul.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/chartmogul", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Chartmogul Spec", - "type": "object", - "required": ["api_key", "start_date", "interval"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Chartmogul API key", - "airbyte_secret": true, - "order": 0 - }, - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. When feasible, any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 1 - }, - "interval": { - "type": "string", - "description": "Some APIs such as Metrics require intervals to cluster data.", - "enum": ["day", "week", "month", "quarter"], - "default": "month", - "order": 2 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "bad83517-5e54-4a3d-9b53-63e85fbd4d7c", - "name": "ClickHouse", - "dockerRepository": "airbyte/source-clickhouse-strict-encrypt", - "dockerImageTag": "0.1.8", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse", - "icon": "clickhouse.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ClickHouse Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The host endpoint of the Clickhouse cluster.", - "title": "Host", - "type": "string" - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"] - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["default"] - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string" - }, - "password": { - "description": "The password associated with this username.", - "title": "Password", - "type": "string", - "airbyte_secret": true - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "dfffecb7-9a13-43e9-acdc-b92af7997ca9", - "name": "Close.com", - "dockerRepository": "airbyte/source-close-com", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/close-com", - "icon": "close.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/close-com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Close.com Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Close.com API key (usually starts with 'api_'; find yours here).", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "description": "The start date to sync data. Leave blank for full sync. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "default": "2021-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635", - "name": "Delighted", - "dockerRepository": "airbyte/source-delighted", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/delighted", - "icon": "delighted.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Delighted Spec", - "type": "object", - "required": ["since", "api_key"], - "additionalProperties": false, - "properties": { - "since": { - "title": "Since", - "type": "string", - "description": "The date from which you'd like to replicate the data", - "examples": ["2022-05-30 04:50:23"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$", - "order": 0 - }, - "api_key": { - "title": "Delighted API Key", - "type": "string", - "description": "A Delighted API key.", - "airbyte_secret": true, - "order": 1 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7", - "name": "Dixa", - "dockerRepository": "airbyte/source-dixa", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa", - "icon": "dixa.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Dixa Spec", - "type": "object", - "required": ["api_token", "start_date"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "description": "Dixa API token", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "description": "The connector pulls records updated from this date onwards.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "batch_size": { - "type": "integer", - "description": "Number of days to batch into one request. Max 31.", - "pattern": "^[0-9]{1,2}$", - "examples": [1, 31], - "default": 31 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "72d405a3-56d8-499f-a571-667c03406e43", - "name": "Dockerhub", - "dockerRepository": "airbyte/source-dockerhub", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dockerhub", - "icon": "dockerhub.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/dockerhub", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Dockerhub Spec", - "type": "object", - "required": ["docker_username"], - "additionalProperties": false, - "properties": { - "docker_username": { - "type": "string", - "description": "Username of DockerHub person or organization (for https://hub.docker.com/v2/repositories/USERNAME/ API call)", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["airbyte"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd", - "name": "End-to-End Testing (Mock API)", - "dockerRepository": "airbyte/source-e2e-test-cloud", - "dockerImageTag": "2.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/e2e-test", - "icon": "airbyte.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/e2e-test", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Cloud E2E Test Source Spec", - "type": "object", - "required": ["max_messages", "mock_catalog"], - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "const": "CONTINUOUS_FEED", - "default": "CONTINUOUS_FEED", - "order": 10 - }, - "max_messages": { - "title": "Max Records", - "description": "Number of records to emit per stream. Min 1. Max 100 billion.", - "type": "integer", - "default": 100, - "min": 1, - "max": 100000000000, - "order": 20 - }, - "seed": { - "title": "Random Seed", - "description": "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", - "type": "integer", - "default": 0, - "examples": [42], - "min": 0, - "max": 1000000, - "order": 30 - }, - "message_interval_ms": { - "title": "Message Interval (ms)", - "description": "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", - "type": "integer", - "min": 0, - "max": 60000, - "default": 0, - "order": 40 - }, - "mock_catalog": { - "title": "Mock Catalog", - "type": "object", - "order": 50, - "oneOf": [ - { - "title": "Single Schema", - "description": "A catalog with one or multiple streams that share the same schema.", - "required": ["type", "stream_name", "stream_schema"], - "properties": { - "type": { - "type": "string", - "const": "SINGLE_STREAM", - "default": "SINGLE_STREAM" - }, - "stream_name": { - "title": "Stream Name", - "description": "Name of the data stream.", - "type": "string", - "default": "data_stream" - }, - "stream_schema": { - "title": "Stream Schema", - "description": "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", - "type": "string", - "default": "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" - }, - "stream_duplication": { - "title": "Duplicate the stream N times", - "description": "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", - "type": "integer", - "default": 1, - "min": 1, - "max": 10000 - } - } - }, - { - "title": "Multi Schema", - "description": "A catalog with multiple data streams, each with a different schema.", - "required": ["type", "stream_schemas"], - "properties": { - "type": { - "type": "string", - "const": "MULTI_STREAM", - "default": "MULTI_STREAM" - }, - "stream_schemas": { - "title": "Streams and Schemas", - "description": "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", - "type": "string", - "default": "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", - "name": "Exchange Rates Api", - "dockerRepository": "airbyte/source-exchange-rates", - "dockerImageTag": "0.2.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "icon": "exchangeratesapi.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "exchangeratesapi.io Source Spec", - "type": "object", - "required": ["start_date", "access_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start getting data from that date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "access_key": { - "type": "string", - "description": "Your API Access Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "base": { - "type": "string", - "description": "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default base currency is EUR", - "examples": ["EUR", "USD"] - }, - "ignore_weekends": { - "type": "boolean", - "description": "Ignore weekends? (Exchanges don't run on weekends)", - "default": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", - "name": "Facebook Marketing", - "dockerRepository": "airbyte/source-facebook-marketing", - "dockerImageTag": "0.2.60", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "icon": "facebook.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", - "connectionSpecification": { - "title": "Source Facebook Marketing", - "type": "object", - "properties": { - "account_id": { - "title": "Account ID", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API.", - "order": 0, - "examples": ["111111111111111"], - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "order": 1, - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.", - "order": 2, - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-26T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the access token generated. See the docs for more information", - "order": 3, - "airbyte_secret": true, - "type": "string" - }, - "include_deleted": { - "title": "Include Deleted", - "description": "Include data from deleted Campaigns, Ads, and AdSets", - "default": false, - "order": 4, - "type": "boolean" - }, - "fetch_thumbnail_images": { - "title": "Fetch Thumbnail Images", - "description": "In each Ad Creative, fetch the thumbnail_url and store the result in thumbnail_data_url", - "default": false, - "order": 5, - "type": "boolean" - }, - "custom_insights": { - "title": "Custom Insights", - "description": "A list which contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)", - "order": 6, - "type": "array", - "items": { - "title": "InsightConfig", - "description": "Config for custom insights", - "type": "object", - "properties": { - "name": { - "title": "Name", - "description": "The name value of insight", - "type": "string" - }, - "fields": { - "title": "Fields", - "description": "A list of chosen fields for fields parameter", - "default": [], - "type": "array", - "items": { - "title": "ValidEnums", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "account_currency", - "account_id", - "account_name", - "action_values", - "actions", - "ad_bid_value", - "ad_click_actions", - "ad_id", - "ad_impression_actions", - "ad_name", - "adset_bid_value", - "adset_end", - "adset_id", - "adset_name", - "adset_start", - "age_targeting", - "attribution_setting", - "auction_bid", - "auction_competitiveness", - "auction_max_competitor_bid", - "buying_type", - "campaign_id", - "campaign_name", - "canvas_avg_view_percent", - "canvas_avg_view_time", - "catalog_segment_actions", - "catalog_segment_value", - "catalog_segment_value_mobile_purchase_roas", - "catalog_segment_value_omni_purchase_roas", - "catalog_segment_value_website_purchase_roas", - "clicks", - "conversion_rate_ranking", - "conversion_values", - "conversions", - "converted_product_quantity", - "converted_product_value", - "cost_per_15_sec_video_view", - "cost_per_2_sec_continuous_video_view", - "cost_per_action_type", - "cost_per_ad_click", - "cost_per_conversion", - "cost_per_dda_countby_convs", - "cost_per_estimated_ad_recallers", - "cost_per_inline_link_click", - "cost_per_inline_post_engagement", - "cost_per_one_thousand_ad_impression", - "cost_per_outbound_click", - "cost_per_thruplay", - "cost_per_unique_action_type", - "cost_per_unique_click", - "cost_per_unique_conversion", - "cost_per_unique_inline_link_click", - "cost_per_unique_outbound_click", - "cpc", - "cpm", - "cpp", - "created_time", - "ctr", - "date_start", - "date_stop", - "dda_countby_convs", - "dda_results", - "engagement_rate_ranking", - "estimated_ad_recall_rate", - "estimated_ad_recall_rate_lower_bound", - "estimated_ad_recall_rate_upper_bound", - "estimated_ad_recallers", - "estimated_ad_recallers_lower_bound", - "estimated_ad_recallers_upper_bound", - "frequency", - "full_view_impressions", - "full_view_reach", - "gender_targeting", - "impressions", - "inline_link_click_ctr", - "inline_link_clicks", - "inline_post_engagement", - "instant_experience_clicks_to_open", - "instant_experience_clicks_to_start", - "instant_experience_outbound_clicks", - "interactive_component_tap", - "labels", - "location", - "mobile_app_purchase_roas", - "objective", - "optimization_goal", - "outbound_clicks", - "outbound_clicks_ctr", - "place_page_name", - "purchase_roas", - "qualifying_question_qualify_answer_rate", - "quality_ranking", - "quality_score_ectr", - "quality_score_ecvr", - "quality_score_organic", - "reach", - "social_spend", - "spend", - "total_postbacks", - "unique_actions", - "unique_clicks", - "unique_conversions", - "unique_ctr", - "unique_inline_link_click_ctr", - "unique_inline_link_clicks", - "unique_link_clicks_ctr", - "unique_outbound_clicks", - "unique_outbound_clicks_ctr", - "unique_video_continuous_2_sec_watched_actions", - "unique_video_view_15_sec", - "updated_time", - "video_15_sec_watched_actions", - "video_30_sec_watched_actions", - "video_avg_time_watched_actions", - "video_continuous_2_sec_watched_actions", - "video_p100_watched_actions", - "video_p25_watched_actions", - "video_p50_watched_actions", - "video_p75_watched_actions", - "video_p95_watched_actions", - "video_play_actions", - "video_play_curve_actions", - "video_play_retention_0_to_15s_actions", - "video_play_retention_20_to_60s_actions", - "video_play_retention_graph_actions", - "video_thruplay_watched_actions", - "video_time_watched_actions", - "website_ctr", - "website_purchase_roas", - "wish_bid" - ] - } - }, - "breakdowns": { - "title": "Breakdowns", - "description": "A list of chosen breakdowns for breakdowns", - "default": [], - "type": "array", - "items": { - "title": "ValidBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "ad_format_asset", - "age", - "app_id", - "body_asset", - "call_to_action_asset", - "country", - "description_asset", - "device_platform", - "dma", - "frequency_value", - "gender", - "hourly_stats_aggregated_by_advertiser_time_zone", - "hourly_stats_aggregated_by_audience_time_zone", - "image_asset", - "impression_device", - "link_url_asset", - "place_page_id", - "platform_position", - "product_id", - "publisher_platform", - "region", - "skan_conversion_id", - "title_asset", - "video_asset" - ] - } - }, - "action_breakdowns": { - "title": "Action Breakdowns", - "description": "A list of chosen action_breakdowns for action_breakdowns", - "default": [], - "type": "array", - "items": { - "title": "ValidActionBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", - "enum": [ - "action_canvas_component_name", - "action_carousel_card_id", - "action_carousel_card_name", - "action_destination", - "action_device", - "action_reaction", - "action_target_id", - "action_type", - "action_video_sound", - "action_video_type" - ] - } - }, - "time_increment": { - "title": "Time Increment", - "description": "Time window in days by which to aggregate statistics. The sync will be chunked into N day intervals, where N is the number of days you specified. For example, if you set this value to 7, then all statistics will be reported as 7-day aggregates by starting from the start_date. If the start and end dates are October 1st and October 30th, then the connector will output 5 records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days only).", - "default": 1, - "exclusiveMaximum": 90, - "exclusiveMinimum": 0, - "type": "integer" - }, - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for this stream, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for this stream, in the format YYYY-MM-DDT00:00:00Z. All data generated between the start date and this date will be replicated. Not setting this option will result in always syncing the latest data.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-26T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "insights_lookback_window": { - "title": "Custom Insights Lookback Window", - "description": "The attribution window", - "default": 28, - "maximum": 28, - "mininum": 1, - "exclusiveMinimum": 0, - "type": "integer" - } - }, - "required": ["name"] - } - }, - "page_size": { - "title": "Page Size of Requests", - "description": "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", - "default": 100, - "order": 7, - "exclusiveMinimum": 0, - "type": "integer" - }, - "insights_lookback_window": { - "title": "Insights Lookback Window", - "description": "The attribution window", - "default": 28, - "order": 8, - "maximum": 28, - "mininum": 1, - "exclusiveMinimum": 0, - "type": "integer" - }, - "max_batch_size": { - "title": "Maximum size of Batched Requests", - "description": "Maximum batch size used when sending batch requests to Facebook API. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", - "default": 50, - "order": 9, - "exclusiveMinimum": 0, - "type": "integer" - } - }, - "required": ["account_id", "start_date", "access_token"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "dfd88b22-b603-4c3d-aad7-3701784586b1", - "name": "Faker", - "dockerRepository": "airbyte/source-faker", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/faker", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/faker", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Faker Source Spec", - "type": "object", - "required": ["count"], - "additionalProperties": false, - "properties": { - "count": { - "title": "Count", - "description": "How many users should be generated in total. This setting does not apply to the purchases or products stream.", - "type": "integer", - "minimum": 1, - "default": 1000, - "order": 0 - }, - "seed": { - "title": "Seed", - "description": "Manually control the faker random seed to return the same values on subsequent runs (leave -1 for random)", - "type": "integer", - "default": -1, - "order": 1 - }, - "records_per_sync": { - "title": "Records Per Sync", - "description": "How many fake records will be returned for each sync, for each stream? By default, it will take 2 syncs to create the requested 1000 records.", - "type": "integer", - "minimum": 1, - "default": 500, - "order": 2 - }, - "records_per_slice": { - "title": "Records Per Stream Slice", - "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", - "type": "integer", - "minimum": 1, - "default": 100, - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", - "name": "File", - "dockerRepository": "airbyte/source-file-secure", - "dockerImageTag": "0.2.20", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/file", - "icon": "file.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/file", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "File Source Spec", - "type": "object", - "additionalProperties": true, - "required": ["dataset_name", "format", "url", "provider"], - "properties": { - "dataset_name": { - "type": "string", - "title": "Dataset Name", - "description": "The Name of the final table to replicate this file into (should include letters, numbers dash and underscores only)." - }, - "format": { - "type": "string", - "enum": [ - "csv", - "json", - "jsonl", - "excel", - "feather", - "parquet", - "yaml" - ], - "default": "csv", - "title": "File Format", - "description": "The Format of the file which should be replicated (Warning: some formats may be experimental, please refer to the docs)." - }, - "reader_options": { - "type": "string", - "title": "Reader Options", - "description": "This should be a string in JSON format. It depends on the chosen file format to provide additional options and tune its behavior.", - "examples": ["{}", "{\"sep\": \" \"}"] - }, - "url": { - "type": "string", - "title": "URL", - "description": "The URL path to access the file which should be replicated." - }, - "provider": { - "type": "object", - "title": "Storage Provider", - "description": "The storage Provider or Location of the file(s) which should be replicated.", - "default": "Public Web", - "oneOf": [ - { - "title": "HTTPS: Public Web", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "const": "HTTPS" - }, - "user_agent": { - "type": "boolean", - "title": "User-Agent", - "default": false, - "description": "Add User-Agent to request" - } - } - }, - { - "title": "GCS: Google Cloud Storage", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "GCS" - }, - "service_account_json": { - "type": "string", - "title": "Service Account JSON", - "description": "In order to access private Buckets stored on Google Cloud, this connector would need a service account json credentials with the proper permissions as described here. Please generate the credentials.json file and copy/paste its content to this field (expecting JSON formats). If accessing publicly available data, this field is not necessary." - } - } - }, - { - "title": "S3: Amazon Web Services", - "required": ["storage"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "S3" - }, - "aws_access_key_id": { - "type": "string", - "title": "AWS Access Key ID", - "description": "In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary." - }, - "aws_secret_access_key": { - "type": "string", - "title": "AWS Secret Access Key", - "description": "In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - } - } - }, - { - "title": "AzBlob: Azure Blob Storage", - "required": ["storage", "storage_account"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "AzBlob" - }, - "storage_account": { - "type": "string", - "title": "Storage Account", - "description": "The globally unique name of the storage account that the desired blob sits within. See here for more details." - }, - "sas_token": { - "type": "string", - "title": "SAS Token", - "description": "To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a SAS (Shared Access Signature) token. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - }, - "shared_key": { - "type": "string", - "title": "Shared Key", - "description": "To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a storage account shared key (aka account key or access key). If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true - } - } - }, - { - "title": "SSH: Secure Shell", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SSH" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - }, - { - "title": "SCP: Secure copy protocol", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SCP" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - }, - { - "title": "SFTP: Secure File Transfer Protocol", - "required": ["storage", "user", "host"], - "properties": { - "storage": { - "type": "string", - "title": "Storage", - "const": "SFTP" - }, - "user": { - "type": "string", - "title": "User", - "description": "" - }, - "password": { - "type": "string", - "title": "Password", - "description": "", - "airbyte_secret": true - }, - "host": { - "type": "string", - "title": "Host", - "description": "" - }, - "port": { - "type": "string", - "title": "Port", - "default": "22", - "description": "" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "6f2ac653-8623-43c4-8950-19218c7caf3d", - "name": "Firebolt", - "dockerRepository": "airbyte/source-firebolt", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/firebolt", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/firebolt", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Firebolt Spec", - "type": "object", - "required": ["username", "password", "database"], - "additionalProperties": false, - "properties": { - "username": { - "type": "string", - "title": "Username", - "description": "Firebolt email address you use to login.", - "examples": ["username@email.com"] - }, - "password": { - "type": "string", - "title": "Password", - "description": "Firebolt password." - }, - "account": { - "type": "string", - "title": "Account", - "description": "Firebolt account to login." - }, - "host": { - "type": "string", - "title": "Host", - "description": "The host name of your Firebolt database.", - "examples": ["api.app.firebolt.io"] - }, - "database": { - "type": "string", - "title": "Database", - "description": "The database to connect to." - }, - "engine": { - "type": "string", - "title": "Engine", - "description": "Engine name or url to connect to." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", - "name": "Freshdesk", - "dockerRepository": "airbyte/source-freshdesk", - "dockerImageTag": "0.3.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshdesk", - "icon": "freshdesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshdesk", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshdesk Spec", - "type": "object", - "required": ["domain", "api_key"], - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "description": "Freshdesk domain", - "title": "Domain", - "examples": ["myaccount.freshdesk.com"], - "pattern": "^[a-zA-Z0-9._-]*\\.freshdesk\\.com$" - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Freshdesk API Key. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "requests_per_minute": { - "title": "Requests per minute", - "type": "integer", - "description": "The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "UTC date and time. Any data created after this date will be replicated. If this parameter is not set, all data will be replicated.", - "format": "date-time", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2020-12-01T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", - "name": "GitHub", - "dockerRepository": "airbyte/source-github", - "dockerImageTag": "0.2.46", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", - "icon": "github.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/github", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "GitHub Source Spec", - "type": "object", - "required": ["start_date", "repository"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication *", - "description": "Choose how to authenticate to GitHub", - "type": "object", - "order": 0, - "oneOf": [ - { - "type": "object", - "title": "OAuth", - "required": ["access_token"], - "properties": { - "option_title": { - "type": "string", - "const": "OAuth Credentials", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "OAuth access token", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Personal Access Token", - "required": ["personal_access_token"], - "properties": { - "option_title": { - "type": "string", - "const": "PAT Credentials", - "order": 0 - }, - "personal_access_token": { - "type": "string", - "title": "Personal Access Tokens", - "description": "Log into GitHub and then generate a personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with \",\"", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Start date", - "description": "The date from which you'd like to replicate data from GitHub in the format YYYY-MM-DDT00:00:00Z. For the streams which support this configuration, only data generated on or after the start date will be replicated. This field doesn't apply to all streams, see the docs for more info", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 1 - }, - "repository": { - "type": "string", - "examples": [ - "airbytehq/airbyte airbytehq/another-repo", - "airbytehq/*", - "airbytehq/airbyte" - ], - "title": "GitHub Repositories", - "description": "Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.", - "order": 2 - }, - "branch": { - "type": "string", - "title": "Branch (Optional)", - "examples": [ - "airbytehq/airbyte/master airbytehq/airbyte/my-branch" - ], - "description": "Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", - "order": 3 - }, - "page_size_for_large_streams": { - "type": "integer", - "title": "Page size for large streams (Optional)", - "minimum": 1, - "maximum": 100, - "default": 10, - "description": "The Github connector contains several streams with a large amount of data. The page size of such streams depends on the size of your repository. We recommended that you specify values between 10 and 30.", - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "option_title"], - "predicate_value": "OAuth Credentials", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80", - "name": "Gitlab", - "dockerRepository": "airbyte/source-gitlab", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab", - "icon": "gitlab.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Gitlab Singer Spec", - "type": "object", - "required": ["api_url", "private_token", "start_date"], - "additionalProperties": false, - "properties": { - "api_url": { - "type": "string", - "examples": ["gitlab.com"], - "description": "Please enter your basic URL from Gitlab instance", - "pattern": "^gitlab[a-zA-Z0-9._-]*\\.com$" - }, - "private_token": { - "type": "string", - "description": "Log into your Gitlab account and then generate a personal Access Token.", - "airbyte_secret": true - }, - "groups": { - "type": "string", - "examples": ["airbyte.io"], - "description": "Space-delimited list of groups. e.g. airbyte.io" - }, - "projects": { - "type": "string", - "examples": ["airbyte.io/documentation"], - "description": "Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab" - }, - "start_date": { - "type": "string", - "description": "The date from which you'd like to replicate data for Gitlab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50", - "name": "Google Ads", - "dockerRepository": "airbyte/source-google-ads", - "dockerImageTag": "0.1.44", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-ads", - "icon": "google-adwords.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Ads Spec", - "type": "object", - "required": ["credentials", "start_date", "customer_id"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "object", - "description": "", - "title": "Google Credentials", - "order": 0, - "required": [ - "developer_token", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "developer_token": { - "type": "string", - "title": "Developer Token", - "order": 0, - "description": "Developer token granted by Google to use their APIs. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "client_id": { - "type": "string", - "title": "Client ID", - "order": 1, - "description": "The Client ID of your Google Ads developer application. More instruction on how to find this value in our docs" - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "order": 2, - "description": "The Client Secret of your Google Ads developer application. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "order": 3, - "description": "The token for obtaining a new access token. More instruction on how to find this value in our docs", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token (Optional)", - "order": 4, - "description": "Access Token for making authenticated requests. More instruction on how to find this value in our docs", - "airbyte_secret": true - } - } - }, - "customer_id": { - "title": "Customer ID(s)", - "type": "string", - "description": "Comma separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for a manager account.", - "pattern": "^[0-9]{10}(,[0-9]{10})*$", - "examples": ["6783948572,5839201945"], - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2017-01-25"], - "order": 2 - }, - "end_date": { - "type": "string", - "title": "End Date (Optional)", - "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2017-01-30"], - "order": 6 - }, - "custom_queries": { - "type": "array", - "title": "Custom GAQL Queries (Optional)", - "description": "", - "order": 3, - "items": { - "type": "object", - "properties": { - "query": { - "type": "string", - "title": "Custom Query", - "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression because it is used by incremental streams. See Google's query builder for more information.", - "examples": [ - "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type FROM campaign WHERE campaign.status = 'PAUSED'" - ] - }, - "table_name": { - "type": "string", - "title": "Destination Table Name", - "description": "The table name in your destination database for choosen query." - } - } - } - }, - "login_customer_id": { - "type": "string", - "title": "Login Customer ID for Managed Accounts (Optional)", - "description": "If your access to the customer account is through a manager account, this field is required and must be set to the customer ID of the manager account (10-digit number without dashes). More information about this field you can see here", - "pattern": "^([0-9]{10})?$", - "examples": ["7349206847"], - "order": 4 - }, - "conversion_window_days": { - "title": "Conversion Window (Optional)", - "type": "integer", - "description": "A conversion window is the period of time after an ad interaction (such as an ad click or video view) during which a conversion, such as a purchase, is recorded in Google Ads. For more information, see Google's documentation.", - "minimum": 0, - "maximum": 1095, - "default": 14, - "examples": [14], - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [ - ["client_id"], - ["client_secret"], - ["developer_token"] - ], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", - "name": "Google Analytics", - "dockerRepository": "airbyte/source-google-analytics-v4", - "dockerImageTag": "0.1.25", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics", - "icon": "google-analytics.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Analytics V4 Spec", - "type": "object", - "required": ["view_id", "start_date"], - "additionalProperties": true, - "properties": { - "credentials": { - "order": 0, - "type": "object", - "title": "Credentials", - "description": "Credentials for the service", - "oneOf": [ - { - "title": "Authenticate via Google (Oauth)", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Analytics developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Analytics developer application.", - "airbyte_secret": true, - "order": 2 - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining a new access token.", - "airbyte_secret": true, - "order": 3 - }, - "access_token": { - "title": "Access Token (Optional)", - "type": "string", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "type": "object", - "title": "Service Account Key Authentication", - "required": ["credentials_json"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service", - "order": 0 - }, - "credentials_json": { - "title": "Service Account JSON Key", - "type": "string", - "description": "The JSON key of the service account to use for authorization", - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ], - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "order": 1, - "type": "string", - "title": "Replication Start Date", - "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2020-06-01"] - }, - "view_id": { - "order": 2, - "type": "string", - "title": "View ID", - "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the Google Analytics Account Explorer." - }, - "custom_reports": { - "order": 3, - "type": "string", - "title": "Custom Reports (Optional)", - "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." - }, - "window_in_days": { - "type": "integer", - "title": "Data request time increment in days (Optional)", - "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. ", - "examples": [30, 60, 90, 120, 200, 364], - "default": 1, - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246", - "name": "Google Directory", - "dockerRepository": "airbyte/source-google-directory", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory", - "icon": "googledirectory.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Directory Spec", - "type": "object", - "required": ["credentials_json", "email"], - "additionalProperties": false, - "properties": { - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "eb4c9e00-db83-4d63-a386-39cfa91012a8", - "name": "Google Search Console", - "dockerRepository": "airbyte/source-google-search-console", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-search-console", - "icon": "googlesearchconsole.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-search-console", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Search Console Spec", - "type": "object", - "required": ["site_urls", "start_date", "authorization"], - "properties": { - "site_urls": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Website URL Property", - "description": "The URLs of the website property attached to your GSC account. Read more here.", - "examples": ["https://example1.com", "https://example2.com"], - "order": 0 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1 - }, - "end_date": { - "type": "string", - "title": "End Date", - "description": "UTC date in the format 2017-01-25. Any data after this date will not be replicated. Must be greater or equal to the start date field.", - "examples": ["2021-12-12"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2 - }, - "authorization": { - "type": "object", - "title": "Authentication Type", - "description": "", - "order": 3, - "oneOf": [ - { - "title": "OAuth", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The client ID of your Google Search Console developer application. Read more here.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The client secret of your Google Search Console developer application. Read more here.", - "airbyte_secret": true - }, - "access_token": { - "title": "Access Token", - "type": "string", - "description": "Access token for making authenticated requests. Read more here.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining a new access token. Read more here.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Service Account Key Authentication", - "required": ["auth_type", "service_account_info", "email"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service", - "order": 0 - }, - "service_account_info": { - "title": "Service Account JSON Key", - "type": "string", - "description": "The JSON key of the service account to use for authorization. Read more here.", - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - }, - "email": { - "title": "Admin Email", - "type": "string", - "description": "The email of the user which has permissions to access the Google Workspace Admin APIs." - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["authorization", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7", - "name": "Google Sheets", - "dockerRepository": "airbyte/source-google-sheets", - "dockerImageTag": "0.2.17", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "icon": "google-sheets.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": true, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "Enter the link to the Google spreadsheet you want to sync", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit" - ] - }, - "row_batch_size": { - "type": "integer", - "title": "Row Batch Size", - "description": "Number of rows fetched when making a Google Sheet API call. Defaults to 200.", - "default": 200 - }, - "credentials": { - "type": "object", - "title": "Authentication", - "description": "Credentials for connecting to the Google Sheets API", - "oneOf": [ - { - "title": "Authenticate via Google (OAuth)", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "Enter your Google application's Client ID", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "Enter your Google application's Client Secret", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Enter your Google application's refresh token", - "airbyte_secret": true - } - } - }, - { - "title": "Service Account Key Authentication", - "type": "object", - "required": ["auth_type", "service_account_info"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service" - }, - "service_account_info": { - "type": "string", - "title": "Service Account Information.", - "description": "Enter your Google Cloud service account key in JSON format", - "airbyte_secret": true, - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734", - "name": "Google Workspace Admin Reports", - "dockerRepository": "airbyte/source-google-workspace-admin-reports", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports", - "icon": "googleworkpace.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Directory Spec", - "type": "object", - "required": ["credentials_json", "email"], - "additionalProperties": false, - "properties": { - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." - }, - "lookback": { - "type": "integer", - "minimum": 0, - "maximum": 180, - "description": "Sets the range of time shown in the report. Reports API allows from up to 180 days ago. " - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", - "name": "Greenhouse", - "dockerRepository": "airbyte/source-greenhouse", - "dockerImageTag": "0.2.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", - "icon": "greenhouse.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Greenhouse Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Greenhouse API Key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", - "name": "HubSpot", - "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.82", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "icon": "hubspot.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "HubSpot Source Spec", - "type": "object", - "required": ["start_date", "credentials"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - }, - "credentials": { - "title": "Authentication", - "description": "Choose how to authenticate to HubSpot.", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "credentials_title" - ], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials", - "const": "OAuth Credentials", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your HubSpot developer application. See the Hubspot docs if you need help finding this ID.", - "type": "string", - "examples": ["123456789000"] - }, - "client_secret": { - "title": "Client Secret", - "description": "The client secret for your HubSpot developer application. See the Hubspot docs if you need help finding this secret.", - "type": "string", - "examples": ["secret"], - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Refresh token to renew an expired access token. See the Hubspot docs if you need help finding this token.", - "type": "string", - "examples": ["refresh_token"], - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API key", - "required": ["api_key", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials set", - "const": "API Key Credentials", - "order": 0 - }, - "api_key": { - "title": "API key", - "description": "HubSpot API Key. See the Hubspot docs if you need help finding this key.", - "type": "string", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Private APP", - "required": ["access_token", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials", - "description": "Name of the credentials set", - "const": "Private App Credentials", - "order": 0 - }, - "access_token": { - "title": "Access token", - "description": "HubSpot Access token. See the Hubspot docs if you need help finding this token.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "6acf6b55-4f1e-4fca-944e-1a3caef8aba8", - "name": "Instagram", - "dockerRepository": "airbyte/source-instagram", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/instagram", - "icon": "instagram.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/instagram", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/instagram", - "connectionSpecification": { - "title": "Source Instagram", - "type": "object", - "properties": { - "start_date": { - "title": "Start Date", - "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the access token generated. See the docs for more information", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["start_date", "access_token"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", - "name": "Intercom", - "dockerRepository": "airbyte/source-intercom", - "dockerImageTag": "0.1.25", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", - "icon": "intercom.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Intercom Spec", - "type": "object", - "required": ["start_date", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-11-16T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "access_token": { - "title": "Access token", - "type": "string", - "description": "Access token for making authenticated requests. See the Intercom docs for more information.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "2e875208-0c0b-4ee4-9e92-1cb3156ea799", - "name": "Iterable", - "dockerRepository": "airbyte/source-iterable", - "dockerImageTag": "0.1.15", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/iterable", - "icon": "iterable.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/iterable", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Iterable Spec", - "type": "object", - "required": ["start_date", "api_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-04-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Iterable API Key. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "95e8cffd-b8c4-4039-968e-d32fb4a69bde", - "name": "Klaviyo", - "dockerRepository": "airbyte/source-klaviyo", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "icon": "klaviyo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/klaviyo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Klaviyo Spec", - "type": "object", - "properties": { - "api_key": { - "title": "Api Key", - "description": "Klaviyo API Key. See our docs if you need help finding this key.", - "airbyte_secret": true, - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string" - } - }, - "required": ["api_key", "start_date"] - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3", - "name": "Kustomer", - "dockerRepository": "airbyte/source-kustomer-singer", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Kustomer Singer Spec", - "type": "object", - "required": ["api_token", "start_date"], - "additionalProperties": true, - "properties": { - "api_token": { - "title": "API Token", - "type": "string", - "description": "Kustomer API Token. See the docs on how to obtain this", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate the data", - "examples": ["2019-01-01T00:00:00Z"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "789f8e7a-2d28-11ec-8d3d-0242ac130003", - "name": "Lemlist", - "dockerRepository": "airbyte/source-lemlist", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/lemlist", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Lemlist Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "API key to access your lemlist account.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", - "name": "LinkedIn Ads", - "dockerRepository": "airbyte/source-linkedin-ads", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads", - "icon": "linkedin.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Linkedin Ads Spec", - "type": "object", - "required": ["start_date"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication *", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "oAuth2.0" - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The client ID of the LinkedIn Ads developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client secret", - "description": "The client secret the LinkedIn Ads developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh token", - "description": "The key to refresh the expired access token.", - "airbyte_secret": true - } - } - }, - { - "title": "Access token", - "type": "object", - "required": ["access_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - "title": "Access token", - "description": "The token value generated using the authentication code. See the docs to obtain yours.", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "description": "UTC date in the format 2020-09-17. Any data before this date will not be replicated.", - "examples": ["2021-05-17"] - }, - "account_ids": { - "title": "Account IDs (Optional)", - "type": "array", - "description": "Specify the account IDs separated by a space, to pull the data from. Leave empty, if you want to pull the data from all associated accounts. See the LinkedIn Ads docs for more info.", - "items": { - "type": "integer" - }, - "default": [] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e", - "name": "Linnworks", - "dockerRepository": "airbyte/source-linnworks", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks", - "icon": "linnworks.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Linnworks Spec", - "type": "object", - "required": [ - "application_id", - "application_secret", - "token", - "start_date" - ], - "additionalProperties": false, - "properties": { - "application_id": { - "title": "Application ID.", - "description": "Linnworks Application ID", - "type": "string" - }, - "application_secret": { - "title": "Application Secret", - "description": "Linnworks Application Secret", - "type": "string", - "airbyte_secret": true - }, - "token": { - "title": "API Token", - "type": "string" - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "type": "string", - "format": "date-time" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", - "name": "Looker", - "dockerRepository": "airbyte/source-looker", - "dockerImageTag": "0.2.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", - "icon": "looker.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Looker Spec", - "type": "object", - "required": ["domain", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "domain": { - "type": "string", - "title": "Domain", - "examples": [ - "domainname.looker.com", - "looker.clientname.com", - "123.123.124.123:8000" - ], - "description": "Domain for your Looker account, e.g. airbyte.cloud.looker.com,looker.[clientname].com,IP address" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID is first part of an API3 key that is specific to each Looker user. See the docs for more information on how to generate this key." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret is second part of an API3 key." - }, - "run_look_ids": { - "title": "Look IDs to Run", - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]*$" - }, - "description": "The IDs of any Looks to run (optional)" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002", - "name": "Mailchimp", - "dockerRepository": "airbyte/source-mailchimp", - "dockerImageTag": "0.2.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mailchimp", - "icon": "mailchimp.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mailchimp", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Mailchimp Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "object", - "title": "Authentication *", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "required": ["auth_type", "access_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "title": "Access Token", - "type": "string", - "description": "An access token generated using the above client ID and secret.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Key", - "required": ["auth_type", "apikey"], - "properties": { - "auth_type": { - "type": "string", - "const": "apikey", - "order": 1 - }, - "apikey": { - "type": "string", - "title": "API Key", - "description": "Mailchimp API Key. See the docs for information on how to generate this key.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", - "name": "Marketo", - "dockerRepository": "airbyte/source-marketo", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo", - "icon": "marketo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Marketo Spec", - "type": "object", - "required": [ - "domain_url", - "client_id", - "client_secret", - "start_date" - ], - "additionalProperties": true, - "properties": { - "domain_url": { - "title": "Domain URL", - "type": "string", - "order": 3, - "description": "Your Marketo Base URL. See the docs for info on how to obtain this.", - "examples": ["https://000-AAA-000.mktorest.com"], - "airbyte_secret": true - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Marketo developer application. See the docs for info on how to obtain this.", - "order": 0, - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Marketo developer application. See the docs for info on how to obtain this.", - "order": 1, - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "order": 2, - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-09-25T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "c7cb421b-942e-4468-99ee-e369bcabaec5", - "name": "Metabase", - "dockerRepository": "airbyte/source-metabase", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/metabase", - "icon": "metabase.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/metabase", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Metabase Source Spec", - "type": "object", - "required": ["instance_api_url"], - "additionalProperties": true, - "properties": { - "instance_api_url": { - "type": "string", - "title": "Metabase Instance API URL", - "description": "URL to your metabase instance API", - "examples": ["http://localhost:3000/api/"], - "order": 0 - }, - "username": { - "type": "string", - "order": 1 - }, - "password": { - "type": "string", - "airbyte_secret": true, - "order": 2 - }, - "session_token": { - "type": "string", - "description": "To generate your session token, you need to run the following command: ``` curl -X POST \\\n -H \"Content-Type: application/json\" \\\n -d '{\"username\": \"person@metabase.com\", \"password\": \"fakepassword\"}' \\\n http://localhost:3000/api/session\n``` Then copy the value of the `id` field returned by a successful call to that API.\nNote that by default, sessions are good for 14 days and needs to be regenerated.", - "airbyte_secret": true, - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", - "name": "Microsoft SQL Server (MSSQL)", - "dockerRepository": "airbyte/source-mssql-strict-encrypt", - "dockerImageTag": "0.4.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", - "icon": "mssql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MSSQL Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The hostname of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "examples": ["1433"], - "order": 1 - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["master"], - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 5 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used when communicating with the database.", - "order": 6, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate", - "enum": ["encrypted_trust_server_certificate"], - "default": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": [ - "ssl_method", - "trustStoreName", - "trustStorePassword" - ], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 - } - } - } - ] - }, - "replication": { - "type": "object", - "title": "Replication Method", - "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "default": "STANDARD", - "order": 8, - "oneOf": [ - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["replication_type"], - "properties": { - "replication_type": { - "type": "string", - "const": "STANDARD", - "enum": ["STANDARD"], - "default": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "required": ["replication_type"], - "properties": { - "replication_type": { - "type": "string", - "const": "CDC", - "enum": ["CDC"], - "default": "CDC", - "order": 0 - }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227", - "name": "Microsoft teams", - "dockerRepository": "airbyte/source-microsoft-teams", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/microsoft-teams", - "icon": "microsoft-teams.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/microsoft-teams", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Microsoft Teams Spec", - "type": "object", - "required": ["period"], - "additionalProperties": true, - "properties": { - "period": { - "type": "string", - "title": "Period", - "description": "Specifies the length of time over which the Team Device Report stream is aggregated. The supported values are: D7, D30, D90, and D180.", - "examples": ["D7"] - }, - "credentials": { - "title": "Authentication mechanism", - "description": "Choose how to authenticate to Microsoft", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via Microsoft (OAuth 2.0)", - "required": [ - "tenant_id", - "client_id", - "client_secret", - "refresh_token" - ], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "enum": ["Client"], - "default": "Client", - "order": 0 - }, - "tenant_id": { - "title": "Directory (tenant) ID", - "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Microsoft Teams developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Microsoft Teams developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A Refresh Token to renew the expired Access Token.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Authenticate via Microsoft", - "required": ["tenant_id", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "enum": ["Token"], - "default": "Token", - "order": 0 - }, - "tenant_id": { - "title": "Directory (tenant) ID", - "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Microsoft Teams developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Microsoft Teams developer application.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "tenant_id": { - "type": "string", - "path_in_connector_config": ["credentials", "tenant_id"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", - "name": "Mixpanel", - "dockerRepository": "airbyte/source-mixpanel", - "dockerImageTag": "0.1.20", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", - "icon": "mixpanel.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Mixpanel Spec", - "type": "object", - "required": ["api_secret"], - "properties": { - "api_secret": { - "order": 0, - "title": "Project Secret", - "type": "string", - "description": "Mixpanel project secret. See the docs for more information on how to obtain this.", - "airbyte_secret": true - }, - "attribution_window": { - "order": 1, - "title": "Attribution Window", - "type": "integer", - "description": " A period of time for attributing results to ads and the lookback period after those actions occur during which ad results are counted. Default attribution window is 5 days.", - "default": 5 - }, - "project_timezone": { - "order": 2, - "title": "Project Timezone", - "type": "string", - "description": "Time zone in which integer date times are stored. The project timezone may be found in the project settings in the Mixpanel console.", - "default": "US/Pacific", - "examples": ["US/Pacific", "UTC"] - }, - "select_properties_by_default": { - "order": 3, - "title": "Select Properties By Default", - "type": "boolean", - "description": "Setting this config parameter to TRUE ensures that new properties on events and engage records are captured. Otherwise new properties will be ignored.", - "default": true - }, - "start_date": { - "order": 4, - "title": "Start Date", - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If this option is not set, the connector will replicate data from up to one year ago by default.", - "examples": ["2021-11-16"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" - }, - "end_date": { - "order": 5, - "title": "End Date", - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated. Left empty to always sync to most recent date", - "examples": ["2021-11-16"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" - }, - "region": { - "order": 6, - "title": "Region", - "description": "The region of mixpanel domain instance either US or EU.", - "type": "string", - "enum": ["US", "EU"], - "default": "US" - }, - "date_window_size": { - "order": 7, - "title": "Date slicing window", - "description": "Defines window size in days, that used to slice through data. You can reduce it, if amount of data in each window is too big for your environment.", - "type": "integer", - "default": 30 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "80a54ea2-9959-4040-aac1-eee42423ec9b", - "name": "Monday", - "dockerRepository": "airbyte/source-monday", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/monday", - "icon": "monday.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/monday", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Monday Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_type", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "subdomain": { - "type": "string", - "title": "Subdomain/Slug (Optional)", - "description": "Slug/subdomain of the account, or the first part of the URL that comes before .monday.com", - "default": "", - "order": 0 - }, - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 1 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "api_token": { - "type": "string", - "title": "Personal API Token", - "description": "API Token for making authenticated requests.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["credentials", "subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e", - "name": "MongoDb", - "dockerRepository": "airbyte/source-mongodb-strict-encrypt", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "icon": "mongodb.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDb Source Spec", - "type": "object", - "required": ["database"], - "additionalProperties": true, - "properties": { - "instance_type": { - "type": "object", - "title": "MongoDb Instance Type", - "description": "The MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "order": 0, - "oneOf": [ - { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], - "properties": { - "instance": { - "type": "string", - "enum": ["standalone"], - "default": "standalone" - }, - "host": { - "title": "Host", - "type": "string", - "description": "The host name of the Mongo database.", - "order": 0 - }, - "port": { - "title": "Port", - "type": "integer", - "description": "The port of the Mongo database.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { - "type": "string", - "enum": ["replica"], - "default": "replica" - }, - "server_addresses": { - "title": "Server Addresses", - "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member separated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "A replica set in MongoDB is a group of mongod processes that maintain the same data set.", - "order": 1 - } - } - }, - { - "title": "MongoDB Atlas", - "additionalProperties": false, - "required": ["instance", "cluster_url"], - "properties": { - "instance": { - "type": "string", - "enum": ["atlas"], - "default": "atlas" - }, - "cluster_url": { - "title": "Cluster URL", - "type": "string", - "description": "The URL of a cluster to connect to.", - "order": 0 - } - } - } - ] - }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 1 - }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 2 - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 3 - }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "722ba4bf-06ec-45a4-8dd5-72e4a5cf3903", - "name": "My Hours", - "dockerRepository": "airbyte/source-my-hours", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/my-hours", - "icon": "my-hours.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/my-hours", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "My Hours Spec", - "type": "object", - "required": ["email", "password", "start_date"], - "additionalProperties": false, - "properties": { - "email": { - "title": "Email", - "type": "string", - "description": "Your My Hours username", - "example": "john@doe.com" - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated to the username", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "description": "Start date for collecting time logs", - "examples": ["%Y-%m-%d", "2016-01-01"], - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "logs_batch_size": { - "title": "Time logs batch size", - "description": "Pagination size used for retrieving logs in days", - "examples": [30], - "type": "integer", - "minimum": 1, - "maximum": 365, - "default": 30 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", - "name": "MySQL", - "dockerRepository": "airbyte/source-mysql-strict-encrypt", - "dockerImageTag": "0.6.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", - "icon": "mysql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySql Source Spec", - "type": "object", - "required": [ - "host", - "port", - "database", - "username", - "replication_method" - ], - "properties": { - "host": { - "description": "The host name of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port to connect to.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "description": "The database name.", - "title": "Database", - "type": "string", - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 - }, - "replication_method": { - "type": "string", - "title": "Replication Method", - "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "order": 7, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "1d4fdb25-64fc-4569-92da-fcdca79a8372", - "name": "Okta", - "dockerRepository": "airbyte/source-okta", - "dockerImageTag": "0.1.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/okta", - "icon": "okta.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/okta", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Okta Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "title": "Okta domain", - "description": "The Okta domain. See the docs for instructions on how to find it.", - "airbyte_secret": false - }, - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SSZ. Any data before this date will not be replicated.", - "examples": ["2022-07-22T00:00:00Z"], - "title": "Start Date" - }, - "credentials": { - "title": "Authorization Method *", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to obtain new Access Token, when it's expired.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "api_token": { - "type": "string", - "title": "Personal API Token", - "description": "An Okta token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["domain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", - "name": "Oracle DB", - "dockerRepository": "airbyte/source-oracle-strict-encrypt", - "dockerImageTag": "0.3.17", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle", - "icon": "oracle.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Oracle Source Spec", - "type": "object", - "required": ["host", "port", "username", "encryption"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of the database.\nOracle Corporations recommends the following port numbers:\n1521 - Default listening port for client connections to the listener. \n2484 - Recommended and officially registered listening port for client connections to the listener using TCP/IP with SSL", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 1521, - "order": 2 - }, - "connection_data": { - "title": "Connect by", - "type": "object", - "description": "Connect data that will be used for DB connection", - "order": 3, - "oneOf": [ - { - "title": "Service name", - "description": "Use service name", - "required": ["service_name"], - "properties": { - "connection_type": { - "type": "string", - "const": "service_name", - "default": "service_name", - "order": 0 - }, - "service_name": { - "title": "Service name", - "type": "string", - "order": 1 - } - } - }, - { - "title": "System ID (SID)", - "description": "Use SID (Oracle System Identifier)", - "required": ["sid"], - "properties": { - "connection_type": { - "type": "string", - "const": "sid", - "default": "sid", - "order": 0 - }, - "sid": { - "title": "System ID (SID)", - "type": "string", - "order": 1 - } - } - } - ] - }, - "username": { - "title": "User", - "description": "The username which is used to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true, - "order": 6 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 7 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "The encryption method with is used when communicating with the database.", - "order": 8, - "oneOf": [ - { - "title": "Native Network Encryption (NNE)", - "description": "The native network encryption gives you the ability to encrypt database connections, without the configuration overhead of TCP/IP and SSL/TLS and without the need to open and listen on different ports.", - "required": ["encryption_method"], - "properties": { - "encryption_method": { - "type": "string", - "const": "client_nne", - "enum": ["client_nne"], - "default": "client_nne" - }, - "encryption_algorithm": { - "type": "string", - "description": "This parameter defines what encryption algorithm is used.", - "title": "Encryption Algorithm", - "default": "AES256", - "enum": ["AES256", "RC4_56", "3DES168"] - } - } - }, - { - "title": "TLS Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": ["encryption_method", "ssl_certificate"], - "properties": { - "encryption_method": { - "type": "string", - "const": "encrypted_verify_certificate", - "enum": ["encrypted_verify_certificate"], - "default": "encrypted_verify_certificate" - }, - "ssl_certificate": { - "title": "SSL PEM File", - "description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations.", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "7f0455fb-4518-4ec0-b7a3-d808bf8081cc", - "name": "Orb", - "dockerRepository": "airbyte/source-orb", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/orb", - "icon": "orb.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.withorb.com/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Orb Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "title": "Orb API Key", - "description": "Orb API Key, issued from the Orb admin console.", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2022-03-01T00:00:00Z. Any data with created_at before this data will not be synced.", - "examples": ["2022-03-01T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window (in days)", - "default": 0, - "minimum": 0, - "description": "When set to N, the connector will always refresh resources created within the past N days. By default, updated objects that are not newly created are not incrementally synced.", - "order": 3 - }, - "string_event_properties_keys": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Event properties keys (string values)", - "description": "Property key names to extract from all events, in order to enrich ledger entries corresponding to an event deduction.", - "order": 4 - }, - "numeric_event_properties_keys": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Event properties keys (numeric values)", - "description": "Property key names to extract from all events, in order to enrich ledger entries corresponding to an event deduction.", - "order": 5 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a", - "name": "Orbit", - "dockerRepository": "airbyte/source-orbit", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/orbit", - "icon": "orbit.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/orbit", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Orbit Source Spec", - "type": "object", - "required": ["api_token", "workspace"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "airbyte_secret": true, - "title": "API Token", - "description": "Authorizes you to work with Orbit workspaces associated with the token.", - "order": 0 - }, - "workspace": { - "type": "string", - "title": "Workspace", - "description": "The unique name of the workspace that your API token is associated with.", - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "Date in the format 2022-06-26. Only load members whose last activities are after this date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", - "name": "Paypal Transaction", - "dockerRepository": "airbyte/source-paypal-transaction", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transaction", - "icon": "paypal.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transactions", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Paypal Transaction Search", - "type": "object", - "required": ["start_date", "is_sandbox"], - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Paypal developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client secret", - "description": "The Client Secret of your Paypal developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh token (Optional)", - "description": "The key to refresh the expired access token.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time.", - "examples": ["2021-06-11T23:59:59-00:00"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" - }, - "is_sandbox": { - "title": "Sandbox", - "description": "Determines whether to use the sandbox or production environment.", - "type": "boolean", - "default": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "3052c77e-8b91-47e2-97a0-a29a22794b4b", - "name": "PersistIq", - "dockerRepository": "airbyte/source-persistiq", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", - "icon": "persistiq.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Persistiq Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "PersistIq API Key. See the docs for more information on where to find that key.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", - "name": "PokeAPI", - "dockerRepository": "airbyte/source-pokeapi", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", - "icon": "pokeapi.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pokeapi Spec", - "type": "object", - "required": ["pokemon_name"], - "additionalProperties": false, - "properties": { - "pokemon_name": { - "type": "string", - "title": "Pokemon Name", - "description": "Pokemon requested from the API.", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["ditto", "luxray", "snorlax"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f", - "name": "PostHog", - "dockerRepository": "airbyte/source-posthog", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/posthog", - "icon": "posthog.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/posthog", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PostHog Spec", - "type": "object", - "required": ["api_key", "start_date"], - "properties": { - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate the data. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "api_key": { - "type": "string", - "airbyte_secret": true, - "title": "API Key", - "description": "API Key. See the docs for information on how to generate this key." - }, - "base_url": { - "type": "string", - "default": "https://app.posthog.com", - "title": "Base URL", - "description": "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com).", - "examples": ["https://posthog.example.com"] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", - "name": "Postgres", - "dockerRepository": "airbyte/source-postgres-strict-encrypt", - "dockerImageTag": "1.0.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", - "icon": "postgresql.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1 - }, - "database": { - "title": "Database Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["public"], - "order": 3 - }, - "username": { - "title": "Username", - "description": "Username to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 6 - }, - "ssl_mode": { - "title": "SSL Modes", - "description": "SSL connection modes. \n
    • disable - Disables encryption of communication between Airbyte and source database
      • \n
    • allow - Enables encryption only when required by the source database
      • \n
    • prefer - allows unencrypted connection only if the source database does not support encryption
      • \n
    • require - Always require encryption. If the source database server does not support encryption, connection will fail
      • \n
    • verify-ca - Always require encryption and verifies that the source database server has a valid SSL certificate
      • \n
    • verify-full - This is the most secure mode. Always require encryption and verifies the identity of the source database server
    \n Read more in the docs.", - "type": "object", - "order": 7, - "oneOf": [ - { - "title": "allow", - "additionalProperties": false, - "description": "Allow SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "allow", - "enum": ["allow"], - "default": "allow", - "order": 0 - } - } - }, - { - "title": "prefer", - "additionalProperties": false, - "description": "Prefer SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "prefer", - "enum": ["prefer"], - "default": "prefer", - "order": 0 - } - } - }, - { - "title": "require", - "additionalProperties": false, - "description": "Require SSL mode.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "require", - "enum": ["require"], - "default": "require", - "order": 0 - } - } - }, - { - "title": "verify-ca", - "additionalProperties": false, - "description": "Verify-ca SSL mode.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-ca", - "enum": ["verify-ca"], - "default": "verify-ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_key_password": { - "type": "string", - "title": "Client Key Password (Optional)", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "verify-full", - "additionalProperties": false, - "description": "Verify-full SSL mode.", - "required": [ - "mode", - "ca_certificate", - "client_certificate", - "client_key" - ], - "properties": { - "mode": { - "type": "string", - "const": "verify-full", - "enum": ["verify-full"], - "default": "verify-full", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA Certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client Certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2 - }, - "client_key": { - "type": "string", - "title": "Client Key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3 - }, - "client_key_password": { - "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Replication Method", - "description": "Replication method for extracting data from the database.", - "order": 8, - "oneOf": [ - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard", - "enum": ["Standard"], - "default": "Standard", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", - "required": ["method", "replication_slot", "publication"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "enum": ["CDC"], - "default": "CDC", - "order": 0 - }, - "plugin": { - "type": "string", - "title": "Plugin", - "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", - "enum": ["pgoutput", "wal2json"], - "default": "pgoutput", - "order": 1 - }, - "replication_slot": { - "type": "string", - "title": "Replication Slot", - "description": "A plugin logical replication slot. Read about replication slots.", - "order": 2 - }, - "publication": { - "type": "string", - "title": "Publication", - "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 3 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "order": 4, - "min": 120, - "max": 1200 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "d60a46d4-709f-4092-a6b7-2457f7d455f5", - "name": "Prestashop", - "dockerRepository": "airbyte/source-prestashop", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/prestashop", - "icon": "prestashop.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PrestaShop Spec", - "type": "object", - "required": ["url", "access_key"], - "additionalProperties": false, - "properties": { - "url": { - "type": "string", - "description": "Shop URL without trailing slash (domain name or IP address)" - }, - "access_key": { - "type": "string", - "description": "Your PrestaShop access key. See the docs for info on how to obtain this.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b08e4776-d1de-4e80-ab5c-1e51dad934a2", - "name": "Qualaroo", - "dockerRepository": "airbyte/source-qualaroo", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/qualaroo", - "icon": "qualaroo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/qualaroo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Qualaroo Spec", - "type": "object", - "required": ["token", "key", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": "string", - "title": "API token", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "key": { - "type": "string", - "title": "API key", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-03-01T00:00:00.000Z"] - }, - "survey_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]{1,8}$" - }, - "title": "Qualaroo survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all surveys to which you have access will be replicated." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["token"], ["key"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", - "name": "Recharge", - "dockerRepository": "airbyte/source-recharge", - "dockerImageTag": "0.1.7", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/recharge", - "icon": "recharge.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/recharge", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Recharge Spec", - "type": "object", - "required": ["start_date", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Recharge API, in the format YYYY-MM-DDT00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-05-14T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The value of the Access Token generated. See the docs for more information.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01", - "name": "Recurly", - "dockerRepository": "airbyte/source-recurly", - "dockerImageTag": "0.4.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/recurly", - "icon": "recurly.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/recurly", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Recurly Source Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": true, - "description": "Recurly API Key. See the docs for more information on how to generate this key.", - "order": 1 - }, - "begin_time": { - "type": "string", - "description": "ISO8601 timestamp from which the replication from Recurly API will start from.", - "examples": ["2021-12-01T00:00:00"], - "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$", - "order": 2 - }, - "end_time": { - "type": "string", - "description": "ISO8601 timestamp to which the replication from Recurly API will stop. Records after that date won't be imported.", - "examples": ["2021-12-01T00:00:00"], - "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$", - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", - "name": "Redshift", - "dockerRepository": "airbyte/source-redshift", - "dockerImageTag": "0.3.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/redshift", - "icon": "redshift.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Redshift Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "password"], - "properties": { - "host": { - "title": "Host", - "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com).", - "type": "string", - "order": 1 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5439, - "examples": ["5439"], - "order": 2 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "examples": ["master"], - "order": 3 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Specify one or more explicitly or keep empty to process all schemas. Schema names are case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "examples": ["public"], - "order": 4 - }, - "username": { - "title": "Username", - "description": "Username to use to access the database.", - "type": "string", - "order": 5 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "db04ecd1-42e7-4115-9cec-95812905c626", - "name": "Retently", - "dockerRepository": "airbyte/source-retently", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/retently", - "icon": "retently.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Retently Api Spec", - "type": "object", - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication Mechanism", - "description": "Choose how to authenticate to Retently", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via Retently (OAuth)", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "enum": ["Client"], - "default": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Retently developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Retently developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Retently Refresh Token which can be used to fetch new Bearer Tokens when the current one expires.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Authenticate with API Token", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "enum": ["Token"], - "default": "Token", - "order": 0 - }, - "api_key": { - "title": "API Token", - "description": "Retently API Token. See the docs for more information on how to obtain this key.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2", - "name": "S3", - "dockerRepository": "airbyte/source-s3", - "dockerImageTag": "0.1.18", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/s3", - "icon": "s3.svg", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/s3", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/s3", - "connectionSpecification": { - "title": "S3 Source Spec", - "type": "object", - "properties": { - "dataset": { - "title": "Output Stream Name", - "description": "The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", - "pattern": "^([A-Za-z0-9-_]+)$", - "order": 0, - "type": "string" - }, - "path_pattern": { - "title": "Pattern of files to replicate", - "description": "A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", - "examples": [ - "**", - "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" - ], - "order": 10, - "type": "string" - }, - "format": { - "title": "File Format", - "description": "The format of the files you'd like to replicate", - "default": "csv", - "order": 20, - "type": "object", - "oneOf": [ - { - "title": "CSV", - "description": "This connector utilises PyArrow (Apache Arrow) for CSV parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "csv", - "type": "string" - }, - "delimiter": { - "title": "Delimiter", - "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", - "default": ",", - "minLength": 1, - "order": 0, - "type": "string" - }, - "infer_datatypes": { - "title": "Infer Datatypes", - "description": "Configures whether a schema for the source should be inferred from the current data or not. If set to false and a custom schema is set, then the manually enforced schema is used. If a schema is not manually set, and this is set to false, then all fields will be read as strings", - "default": true, - "order": 1, - "type": "boolean" - }, - "quote_char": { - "title": "Quote Character", - "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", - "default": "\"", - "order": 2, - "type": "string" - }, - "escape_char": { - "title": "Escape Character", - "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", - "order": 3, - "type": "string" - }, - "encoding": { - "title": "Encoding", - "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", - "default": "utf8", - "order": 4, - "type": "string" - }, - "double_quote": { - "title": "Double Quote", - "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", - "default": true, - "order": 5, - "type": "boolean" - }, - "newlines_in_values": { - "title": "Allow newlines in values", - "description": "Whether newline characters are allowed in CSV values. Turning this on may affect performance. Leave blank to default to False.", - "default": false, - "order": 6, - "type": "boolean" - }, - "additional_reader_options": { - "title": "Additional Reader Options", - "description": "Optionally add a valid JSON string here to provide additional options to the csv reader. Mappings must correspond to options detailed here. 'column_types' is used internally to handle schema so overriding that would likely cause problems.", - "default": "{}", - "examples": [ - "{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"]}" - ], - "order": 7, - "type": "string" - }, - "advanced_options": { - "title": "Advanced Options", - "description": "Optionally add a valid JSON string here to provide additional Pyarrow ReadOptions. Specify 'column_names' here if your CSV doesn't have header, or if you want to use custom column names. 'block_size' and 'encoding' are already used above, specify them again here will override the values above.", - "default": "{}", - "examples": [ - "{\"column_names\": [\"column1\", \"column2\"]}" - ], - "order": 8, - "type": "string" - }, - "block_size": { - "title": "Block Size", - "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", - "default": 10000, - "order": 9, - "type": "integer" - } - } - }, - { - "title": "Parquet", - "description": "This connector utilises PyArrow (Apache Arrow) for Parquet parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "parquet", - "type": "string" - }, - "columns": { - "title": "Selected Columns", - "description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here as a comma-delimited list. Leave it empty to sync all columns.", - "order": 0, - "type": "array", - "items": { - "type": "string" - } - }, - "batch_size": { - "title": "Record batch size", - "description": "Maximum number of records per batch read from the input files. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide.", - "default": 65536, - "order": 1, - "type": "integer" - }, - "buffer_size": { - "title": "Buffer Size", - "description": "Perform read buffering when deserializing individual column chunks. By default every group column will be loaded fully to memory. This option can help avoid out-of-memory errors if your data is particularly wide.", - "default": 2, - "type": "integer" - } - } - }, - { - "title": "Avro", - "description": "This connector utilises fastavro for Avro parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "avro", - "type": "string" - } - } - }, - { - "title": "Jsonl", - "description": "This connector uses PyArrow for JSON Lines (jsonl) file parsing.", - "type": "object", - "properties": { - "filetype": { - "title": "Filetype", - "const": "jsonl", - "type": "string" - }, - "newlines_in_values": { - "title": "Allow newlines in values", - "description": "Whether newline characters are allowed in JSON values. Turning this on may affect performance. Leave blank to default to False.", - "default": false, - "order": 0, - "type": "boolean" - }, - "unexpected_field_behavior": { - "title": "Unexpected field behavior", - "description": "How JSON fields outside of explicit_schema (if given) are treated. Check PyArrow documentation for details", - "default": "infer", - "examples": ["ignore", "infer", "error"], - "order": 1, - "allOf": [ - { - "title": "UnexpectedFieldBehaviorEnum", - "description": "An enumeration.", - "enum": ["ignore", "infer", "error"], - "type": "string" - } - ] - }, - "block_size": { - "title": "Block Size", - "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", - "default": 10000, - "order": 2, - "type": "integer" - } - } - } - ] - }, - "schema": { - "title": "Manually enforced data schema (Optional)", - "description": "Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", - "default": "{}", - "examples": [ - "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" - ], - "order": 30, - "type": "string" - }, - "provider": { - "title": "S3: Amazon Web Services", - "type": "object", - "properties": { - "bucket": { - "title": "Bucket", - "description": "Name of the S3 bucket where the file(s) exist.", - "order": 0, - "type": "string" - }, - "aws_access_key_id": { - "title": "AWS Access Key ID", - "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true, - "order": 1, - "type": "string" - }, - "aws_secret_access_key": { - "title": "AWS Secret Access Key", - "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", - "airbyte_secret": true, - "order": 2, - "type": "string" - }, - "path_prefix": { - "title": "Path Prefix", - "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", - "default": "", - "order": 3, - "type": "string" - }, - "endpoint": { - "title": "Endpoint", - "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", - "default": "", - "order": 4, - "type": "string" - }, - "use_ssl": { - "title": "Use TLS", - "description": "Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, non-AWS server", - "order": 5, - "type": "boolean" - }, - "verify_ssl_cert": { - "title": "Verify TLS Certificates", - "description": "Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server", - "order": 6, - "type": "boolean" - } - }, - "required": ["bucket"], - "order": 11, - "description": "Use this to load files from S3 or S3-compatible services" - } - }, - "required": ["dataset", "path_pattern", "provider"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "a827c52e-791c-4135-a245-e233c5255199", - "name": "SFTP", - "dockerRepository": "airbyte/source-sftp", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/sftp", - "sourceType": "file", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/source/sftp", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "SFTP Source Spec", - "type": "object", - "required": ["user", "host", "port"], - "additionalProperties": true, - "properties": { - "user": { - "title": "User Name", - "description": "The server user", - "type": "string", - "order": 0 - }, - "host": { - "title": "Host Address", - "description": "The server host address", - "type": "string", - "examples": ["www.host.com", "192.0.2.1"], - "order": 1 - }, - "port": { - "title": "Port", - "description": "The server port", - "type": "integer", - "default": 22, - "examples": ["22"], - "order": 2 - }, - "credentials": { - "type": "object", - "title": "Authentication *", - "description": "The server authentication method", - "order": 3, - "oneOf": [ - { - "title": "Password Authentication", - "required": ["auth_method", "auth_user_password"], - "properties": { - "auth_method": { - "description": "Connect through password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "auth_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 1 - } - } - }, - { - "title": "SSH Key Authentication", - "required": ["auth_method", "auth_ssh_key"], - "properties": { - "auth_method": { - "description": "Connect through ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "auth_ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 1 - } - } - } - ] - }, - "file_types": { - "title": "File types", - "description": "Coma separated file types. Currently only 'csv' and 'json' types are supported.", - "type": "string", - "default": "csv,json", - "order": 4, - "examples": ["csv,json", "csv"] - }, - "folder_path": { - "title": "Folder Path (Optional)", - "description": "The directory to search files for sync", - "type": "string", - "default": "", - "examples": ["/logs/2022"], - "order": 5 - }, - "file_pattern": { - "title": "File Pattern (Optional)", - "description": "The regular expression to specify files for sync in a chosen Folder Path", - "type": "string", - "default": "", - "examples": [ - "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`" - ], - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "b117307c-14b6-41aa-9422-947e34922962", - "name": "Salesforce", - "dockerRepository": "airbyte/source-salesforce", - "dockerImageTag": "1.0.13", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "icon": "salesforce.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "title": "Sandbox", - "description": "Toggle if you're using a Salesforce Sandbox", - "type": "boolean", - "default": false, - "order": 1 - }, - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "description": "Enter your Salesforce developer application's Client ID", - "type": "string", - "order": 2 - }, - "client_secret": { - "title": "Client Secret", - "description": "Enter your Salesforce developer application's Client secret", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Enter your application's Salesforce Refresh Token used for Airbyte to access your Salesforce account.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "start_date": { - "title": "Start Date", - "description": "Enter the date in the YYYY-MM-DD format. Airbyte will replicate the data added on and after this date. If this field is blank, Airbyte will replicate all data.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2021-07-25", "2021-07-25T00:00:00Z"], - "order": 5 - }, - "streams_criteria": { - "type": "array", - "order": 6, - "items": { - "type": "object", - "required": ["criteria", "value"], - "properties": { - "criteria": { - "type": "string", - "title": "Search criteria", - "enum": [ - "starts with", - "ends with", - "contains", - "exacts", - "starts not with", - "ends not with", - "not contains", - "not exacts" - ], - "order": 1, - "default": "contains" - }, - "value": { - "type": "string", - "title": "Search value", - "order": 2 - } - } - }, - "title": "Filter Salesforce Objects (Optional)", - "description": "Filter streams relevant to you" - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "is_sandbox": { - "type": "boolean", - "path_in_connector_config": ["is_sandbox"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77", - "name": "Salesforce (Singer)", - "dockerRepository": "airbyte/source-salesforce-singer", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "_comment": "todo (cgardens) - allow default fields. is_sandbox and api_type should not be required and fall back on default. depends on change in api.", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "start_date", - "api_type" - ], - "additionalProperties": false, - "properties": { - "client_id": { - "description": "The Consumer Key that can be found when viewing your app in Salesforce", - "type": "string" - }, - "client_secret": { - "description": "The Consumer Secret that can be found when viewing your app in Salesforce", - "type": "string", - "airbyte_secret": true - }, - "refresh_token": { - "description": "Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow this guide to retrieve it.", - "type": "string", - "airbyte_secret": true - }, - "start_date": { - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"] - }, - "is_sandbox": { - "description": "Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. We provide more info on this field in the docs.", - "type": "boolean", - "default": false - }, - "api_type": { - "description": "Unless you know that you are transferring a very small amount of data, prefer using the BULK API. This will help avoid using up all of your API call quota with Salesforce. Valid values are BULK or REST.", - "type": "string", - "enum": ["BULK", "REST"], - "default": "BULK" - }, - "quota_percent_per_run": { - "description": "determines the maximum allowed API quota percentage the connector is allowed to consume per sync job", - "type": ["number", null], - "default": null - }, - "quota_percent_total": { - "description": "Determines the maximum allowed API quota percentage the connector is allowed to consume at any time", - "type": ["number", "null"], - "default": null - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", - "name": "Sendgrid", - "dockerRepository": "airbyte/source-sendgrid", - "dockerImageTag": "0.2.8", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sendgrid", - "icon": "sendgrid.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sendgrid", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Sendgrid Spec", - "type": "object", - "required": ["apikey"], - "additionalProperties": false, - "properties": { - "apikey": { - "title": "Sendgrid API key", - "type": "string", - "description": "API Key, use admin to generate this key.", - "order": 0 - }, - "start_time": { - "title": "Start time", - "type": "integer", - "description": "Start time in timestamp integer format. Any data before this timestamp will not be replicated.", - "examples": [1558359837], - "order": 1 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "cdaf146a-9b75-49fd-9dd2-9d64a0bb4781", - "name": "Sentry", - "dockerRepository": "airbyte/source-sentry", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sentry", - "icon": "sentry.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/sentry", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Sentry Spec", - "type": "object", - "required": ["auth_token", "organization", "project"], - "additionalProperties": false, - "properties": { - "auth_token": { - "type": "string", - "title": "Authentication Tokens", - "description": "Log into Sentry and then create authentication tokens.For self-hosted, you can find or create authentication tokens by visiting \"{instance_url_prefix}/settings/account/api/auth-tokens/\"", - "airbyte_secret": true - }, - "hostname": { - "type": "string", - "title": "Host Name", - "description": "Host name of Sentry API server.For self-hosted, specify your host name here. Otherwise, leave it empty.", - "default": "sentry.io" - }, - "organization": { - "type": "string", - "title": "Organization", - "description": "The slug of the organization the groups belong to." - }, - "project": { - "type": "string", - "title": "Project", - "description": "The name (slug) of the Project you want to sync." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", - "name": "Shopify", - "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.37", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", - "icon": "shopify.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shopify Source CDK Specifications", - "type": "object", - "required": ["shop", "start_date"], - "additionalProperties": true, - "properties": { - "shop": { - "type": "string", - "title": "Shopify Store", - "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME'.", - "order": 1 - }, - "credentials": { - "title": "Shopify Authorization Method", - "description": "The authorization method to use to retrieve data from Shopify", - "type": "object", - "order": 2, - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "description": "OAuth2.0", - "required": ["auth_method"], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of the Shopify developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of the Shopify developer application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "title": "API Password", - "description": "API Password Auth", - "type": "object", - "required": ["auth_method", "api_password"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_password", - "order": 0 - }, - "api_password": { - "type": "string", - "title": "API Password", - "description": "The API Password for your private application in the `Shopify` store.", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Replication Start Date", - "description": "The date you would like to replicate data from. Format: YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "shop": { - "type": "string", - "path_in_connector_config": ["shop"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "2fed2292-5586-480c-af92-9944e39fe12d", - "name": "Short.io", - "dockerRepository": "airbyte/source-shortio", - "dockerImageTag": "0.1.3", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/shortio", - "icon": "short.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://developers.short.io/reference", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shortio Spec", - "type": "object", - "required": ["domain_id", "secret_key", "start_date"], - "properties": { - "domain_id": { - "type": "string", - "desciprtion": "Short.io Domain ID", - "title": "Domain ID", - "airbyte_secret": false - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Short.io Secret Key", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "airbyte_secret": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "374ebc65-6636-4ea0-925c-7d35999a8ffc", - "name": "Smartsheets", - "dockerRepository": "airbyte/source-smartsheets", - "dockerImageTag": "0.1.12", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/smartsheets", - "icon": "smartsheet.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/smartsheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Smartsheets Source Spec", - "type": "object", - "required": ["access_token", "spreadsheet_id"], - "additionalProperties": true, - "properties": { - "access_token": { - "title": "Access Token", - "description": "The access token to use for accessing your data from Smartsheets. This access token must be generated by a user with at least read access to the data you'd like to replicate. Generate an access token in the Smartsheets main menu by clicking Account > Apps & Integrations > API Access. See the setup guide for information on how to obtain this token.", - "type": "string", - "order": 0, - "airbyte_secret": true - }, - "spreadsheet_id": { - "title": "Sheet ID", - "description": "The spreadsheet ID. Find it by opening the spreadsheet then navigating to File > Properties", - "type": "string", - "order": 1 - }, - "start_datetime": { - "title": "Start Datetime (Optional)", - "type": "string", - "examples": ["2000-01-01T13:00:00", "2000-01-01T13:00:00-07:00"], - "description": "Only rows modified after this date/time will be replicated. This should be an ISO 8601 string, for instance: `2000-01-01T13:00:00`", - "format": "date-time", - "default": "2020-01-01T00:00:00+00:00", - "order": 2, - "airbyte_hidden": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": [], - "predicate_value": "", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": {} - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b", - "name": "Snapchat Marketing", - "dockerRepository": "airbyte/source-snapchat-marketing", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snapchat-marketing", - "icon": "snapchat.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snapchat-marketing", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snapchat Marketing Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Snapchat developer application.", - "airbyte_secret": true, - "order": 0 - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Snapchat developer application.", - "airbyte_secret": true, - "order": 1 - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 2 - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "Date in the format 2022-01-01. Any data before this date will not be replicated.", - "examples": ["2022-01-01"], - "default": "2022-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 3 - }, - "end_date": { - "type": "string", - "title": "End Date (Optional)", - "description": "Date in the format 2017-01-25. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2022-01-30"], - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", - "name": "Snowflake", - "dockerRepository": "airbyte/source-snowflake", - "dockerImageTag": "0.1.19", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", - "icon": "snowflake.svg", - "sourceType": "database", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Snowflake Source Spec", - "type": "object", - "required": ["host", "role", "warehouse", "database"], - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "order": 0, - "required": ["client_id", "client_secret", "auth_type"], - "properties": { - "auth_type": { - "type": "string", - "const": "OAuth", - "default": "OAuth", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Snowflake developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Snowflake developer application.", - "airbyte_secret": true, - "order": 2 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token for making authenticated requests.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "Username and Password", - "type": "object", - "required": ["username", "password", "auth_type"], - "order": 1, - "properties": { - "auth_type": { - "type": "string", - "const": "username/password", - "default": "username/password", - "order": 0 - }, - "username": { - "description": "The username you created to allow Airbyte to access the database.", - "examples": ["AIRBYTE_USER"], - "type": "string", - "title": "Username", - "order": 1 - }, - "password": { - "description": "The password associated with the username.", - "type": "string", - "airbyte_secret": true, - "title": "Password", - "order": 2 - } - } - } - ], - "order": 0 - }, - "host": { - "description": "The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).", - "examples": ["accountname.us-east-2.aws.snowflakecomputing.com"], - "type": "string", - "title": "Account Name", - "order": 1 - }, - "role": { - "description": "The role you created for Airbyte to access Snowflake.", - "examples": ["AIRBYTE_ROLE"], - "type": "string", - "title": "Role", - "order": 2 - }, - "warehouse": { - "description": "The warehouse you created for Airbyte to access data.", - "examples": ["AIRBYTE_WAREHOUSE"], - "type": "string", - "title": "Warehouse", - "order": 3 - }, - "database": { - "description": "The database you created for Airbyte to access data.", - "examples": ["AIRBYTE_DATABASE"], - "type": "string", - "title": "Database", - "order": 4 - }, - "schema": { - "description": "The source Snowflake schema tables. Leave empty to access tables from multiple schemas.", - "examples": ["AIRBYTE_SCHEMA"], - "type": "string", - "title": "Schema", - "order": 5 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 6 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "OAuth", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "host": { - "type": "string", - "path_in_connector_config": ["host"] - }, - "role": { - "type": "string", - "path_in_connector_config": ["role"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", - "name": "Square", - "dockerRepository": "airbyte/source-square", - "dockerImageTag": "0.1.4", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", - "icon": "square.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/square", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Square Source CDK Specifications", - "type": "object", - "required": ["is_sandbox"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "type": "boolean", - "description": "Determines whether to use the sandbox or production environment.", - "title": "Sandbox", - "examples": [true, false], - "default": false - }, - "start_date": { - "type": "string", - "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated. If not set, all data will be replicated.", - "title": "Start Date", - "examples": ["2021-01-01"], - "default": "2021-01-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "include_deleted_objects": { - "type": "boolean", - "description": "In some streams there is an option to include deleted objects (Items, Categories, Discounts, Taxes)", - "title": "Include Deleted Objects", - "examples": [true, false], - "default": false - }, - "credentials": { - "type": "object", - "title": "Credential Type", - "oneOf": [ - { - "title": "Oauth authentication", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Oauth", - "enum": ["Oauth"], - "default": "Oauth", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Square-issued ID of your application", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Square-issued application secret for your application", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A refresh token generated using the above client ID and secret", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Key", - "required": ["auth_type", "api_key"], - "properties": { - "auth_type": { - "type": "string", - "const": "Apikey", - "enum": ["Apikey"], - "default": "Apikey", - "order": 1 - }, - "api_key": { - "title": "API key token", - "type": "string", - "description": "The API key for a Square application", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Oauth", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", - "name": "Stripe", - "dockerRepository": "airbyte/source-stripe", - "dockerImageTag": "0.1.37", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "icon": "stripe.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "protocol_version": "0.2.1", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["client_secret", "account_id", "start_date"], - "properties": { - "account_id": { - "type": "string", - "title": "Account ID", - "description": "Your Stripe account ID (starts with 'acct_', find yours here).", - "order": 0 - }, - "client_secret": { - "type": "string", - "title": "Secret Key", - "description": "Stripe API key (usually starts with 'sk_live_'; find yours here).", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Replication start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Only data generated after this date will be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window in days (Optional)", - "default": 0, - "minimum": 0, - "description": "When set, the connector will always re-export data from the past N days, where N is the value set here. This is useful if your data is frequently updated after creation. More info here", - "order": 3 - }, - "slice_range": { - "type": "integer", - "title": "Data request time increment in days (Optional)", - "default": 365, - "minimum": 1, - "examples": [1, 3, 10, 30, 180, 360], - "description": "The time increment used by the connector when requesting data from the Stripe API. The bigger the value is, the less requests will be made and faster the sync will be. On the other hand, the more seldom the state is persisted.", - "order": 4 - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5", - "name": "SurveyMonkey", - "dockerRepository": "airbyte/source-surveymonkey", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/surveymonkey", - "icon": "surveymonkey.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/surveymonkey", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "SurveyMonkey Spec", - "type": "object", - "required": ["access_token", "start_date"], - "additionalProperties": true, - "properties": { - "access_token": { - "title": "Access Token", - "order": 0, - "type": "string", - "airbyte_secret": true, - "description": "Access Token for making authenticated requests. See the docs for information on how to generate this key." - }, - "start_date": { - "title": "Start Date", - "order": 1, - "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "survey_ids": { - "type": "array", - "order": 2, - "items": { - "type": "string", - "pattern": "^[0-9]{8,9}$" - }, - "title": "Survey Monkey survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all boards to which you have access will be replicated." - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "d1aa448b-7c54-498e-ad95-263cbebcd2db", - "name": "Tempo", - "dockerRepository": "airbyte/source-tempo", - "dockerImageTag": "0.2.5", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tempo", - "icon": "tempo.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Tempo Spec", - "type": "object", - "required": ["api_token"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "title": "API token", - "description": "Tempo API Token. Go to Tempo>Settings, scroll down to Data Access and select API integration.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35", - "name": "TikTok Marketing", - "dockerRepository": "airbyte/source-tiktok-marketing", - "dockerImageTag": "0.1.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "icon": "tiktok.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/tiktok-marketing", - "connectionSpecification": { - "title": "TikTok Marketing Source Spec", - "type": "object", - "properties": { - "credentials": { - "title": "Authentication Method", - "description": "Authentication method", - "default": {}, - "order": 0, - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The Developer Application App ID.", - "airbyte_secret": true, - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The Developer Application Secret.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["app_id", "secret", "access_token"] - }, - { - "title": "Sandbox Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "sandbox_access_token", - "order": 0, - "type": "string" - }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The long-term authorized access token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["advertiser_id", "access_token"] - } - ] - }, - "start_date": { - "title": "Replication Start Date *", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", - "default": "2016-09-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1, - "type": "string" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2, - "type": "string" - }, - "report_granularity": { - "title": "Report Aggregation Granularity", - "description": "The granularity used for aggregating performance data in reports. See the docs.", - "enum": ["LIFETIME", "DAY", "HOUR"], - "order": 3, - "airbyte_hidden": true, - "type": "string" - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [ - "overwrite", - "append", - "append_dedup" - ], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "title": "CompleteOauthOutputSpecification", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "path_in_connector_config": ["credentials", "access_token"], - "type": "string" - } - }, - "required": ["access_token"] - }, - "complete_oauth_server_input_specification": { - "title": "CompleteOauthServerInputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "type": "string" - }, - "secret": { - "title": "Secret", - "type": "string" - } - }, - "required": ["app_id", "secret"] - }, - "complete_oauth_server_output_specification": { - "title": "CompleteOauthServerOutputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "path_in_connector_config": ["credentials", "app_id"], - "type": "string" - }, - "secret": { - "title": "Secret", - "path_in_connector_config": ["credentials", "secret"], - "type": "string" - } - }, - "required": ["app_id", "secret"] - } - } - }, - "additionalProperties": true - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "b9dc6155-672e-42ea-b10d-9f1f1fb95ab1", - "name": "Twilio", - "dockerRepository": "airbyte/source-twilio", - "dockerImageTag": "0.1.6", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/twilio", - "icon": "twilio.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/twilio", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Twilio Spec", - "type": "object", - "required": ["account_sid", "auth_token", "start_date"], - "additionalProperties": false, - "properties": { - "account_sid": { - "title": "Account ID", - "description": "Twilio account SID", - "airbyte_secret": true, - "type": "string", - "order": 1 - }, - "auth_token": { - "title": "Auth Token", - "description": "Twilio Auth Token.", - "airbyte_secret": true, - "type": "string", - "order": 2 - }, - "start_date": { - "title": "Replication Start Date", - "description": "UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2020-10-01T00:00:00Z"], - "type": "string", - "order": 3 - }, - "lookback_window": { - "title": "Lookback window", - "description": "How far into the past to look for records. (in minutes)", - "examples": [60], - "default": 0, - "type": "integer", - "order": 4 - } - } - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "e7eff203-90bf-43e5-a240-19ea3056c474", - "name": "Typeform", - "dockerRepository": "airbyte/source-typeform", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/typeform", - "icon": "typeform.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/typeform", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Typeform Spec", - "type": "object", - "required": ["token", "start_date"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "description": "The date you would like to replicate data. Format: YYYY-MM-DDTHH:mm:ss[Z].", - "examples": ["2020-01-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "token": { - "type": "string", - "description": "The API Token for a Typeform account.", - "airbyte_secret": true - }, - "form_ids": { - "title": "Form IDs to replicate", - "description": "When this parameter is set, the connector will replicate data only from the input forms. Otherwise, all forms in your Typeform account will be replicated. You can find form IDs in your form URLs. For example, in the URL \"https://mysite.typeform.com/to/u6nXL7\" the form_id is u6nXL7. You can find form URLs on Share panel", - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "c4cfaeda-c757-489a-8aba-859fb08b6970", - "name": "US Census", - "dockerRepository": "airbyte/source-us-census", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/us-census", - "icon": "uscensus.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/us-census", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "https://api.census.gov/ Source Spec", - "type": "object", - "required": ["api_key", "query_path"], - "additionalProperties": false, - "properties": { - "query_params": { - "type": "string", - "description": "The query parameters portion of the GET request, without the api key", - "pattern": "^\\w+=[\\w,:*]+(&(?!key)\\w+=[\\w,:*]+)*$", - "examples": [ - "get=NAME,NAICS2017_LABEL,LFO_LABEL,EMPSZES_LABEL,ESTAB,PAYANN,PAYQTR1,EMP&for=us:*&NAICS2017=72&LFO=001&EMPSZES=001", - "get=MOVEDIN,GEOID1,GEOID2,MOVEDOUT,FULL1_NAME,FULL2_NAME,MOVEDNET&for=county:*" - ] - }, - "query_path": { - "type": "string", - "description": "The path portion of the GET request", - "pattern": "^data(\\/[\\w\\d]+)+$", - "examples": [ - "data/2019/cbp", - "data/2018/acs", - "data/timeseries/healthins/sahie" - ] - }, - "api_key": { - "type": "string", - "description": "Your API Key. Get your key here.", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "ef580275-d9a9-48bb-af5e-db0f5855be04", - "name": "Webflow", - "dockerRepository": "airbyte/source-webflow", - "dockerImageTag": "0.1.2", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/webflow", - "icon": "webflow.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/webflow", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Webflow Spec", - "type": "object", - "required": ["api_key", "site_id"], - "additionalProperties": false, - "properties": { - "site_id": { - "title": "Site id", - "type": "string", - "description": "The id of the Webflow site you are requesting data from. See https://developers.webflow.com/#sites", - "example": "a relatively long hex sequence", - "order": 0 - }, - "api_key": { - "title": "API token", - "type": "string", - "description": "The API token for authenticating to Webflow. See https://university.webflow.com/lesson/intro-to-the-webflow-api", - "example": "a very long hex sequence", - "order": 1, - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "afa734e4-3571-11ec-991a-1e0031268139", - "name": "YouTube Analytics", - "dockerRepository": "airbyte/source-youtube-analytics", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/youtube-analytics", - "icon": "youtube.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/youtube-analytics", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "YouTube Analytics Spec", - "type": "object", - "required": ["credentials"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authenticate via OAuth 2.0", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your developer application", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The client secret of your developer application", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A refresh token generated using the above client ID and secret", - "airbyte_secret": true - } - } - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } - }, - "public": true, - "custom": false, - "releaseStage": "beta" - }, - { - "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4", - "name": "Zendesk Chat", - "dockerRepository": "airbyte/source-zendesk-chat", - "dockerImageTag": "0.1.9", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Chat Spec", - "type": "object", - "required": ["start_date"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Chat API, in the format YYYY-MM-DDT00:00:00Z.", - "examples": ["2021-02-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "subdomain": { - "type": "string", - "title": "Subdomain (Optional)", - "description": "Required if you access Zendesk Chat from a Zendesk Support subdomain.", - "default": "" - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": ["credentials"], - "properties": { - "credentials": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to obtain new Access Token, when it's expired.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Access Token", - "required": ["credentials", "access_token"], - "properties": { - "credentials": { - "type": "string", - "const": "access_token", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Access Token to make authenticated requests.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "credentials"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "325e0640-e7b3-4e24-b823-3361008f603f", - "name": "Zendesk Sunshine", - "dockerRepository": "airbyte/source-zendesk-sunshine", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-sunshine", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk_sunshine", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Sunshine Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "subdomain": { - "title": "Subdomain", - "type": "string", - "description": "The subdomain for your Zendesk Account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_method", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "enum": ["oauth2.0"], - "default": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Long-term access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_method", "api_token", "email"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_token", - "enum": ["api_token"], - "default": "api_token", - "order": 1 - }, - "api_token": { - "type": "string", - "title": "API Token", - "description": "API Token. See the docs for information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "title": "Email", - "description": "The user email for your Zendesk account" - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "79c1aa37-dae3-42ae-b333-d1c105477715", - "name": "Zendesk Support", - "dockerRepository": "airbyte/source-zendesk-support", - "dockerImageTag": "0.2.14", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-support", - "icon": "zendesk.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-support", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Zendesk Support Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Support API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2020-10-15T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "subdomain": { - "type": "string", - "title": "Subdomain", - "description": "This is your Zendesk subdomain that can be found in your account URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN is the value of your subdomain." - }, - "credentials": { - "title": "Authentication *", - "type": "object", - "description": "Zendesk service provides two authentication methods. Choose between: `OAuth2.0` or `API token`.", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "required": ["access_token"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The value of the API token generated. See the docs for more information.", - "airbyte_secret": true - } - } - }, - { - "title": "API Token", - "type": "object", - "required": ["email", "api_token"], - "additionalProperties": true, - "properties": { - "credentials": { - "type": "string", - "const": "api_token", - "order": 0 - }, - "email": { - "title": "Email", - "type": "string", - "description": "The user email for your Zendesk account." - }, - "api_token": { - "title": "API Token", - "type": "string", - "description": "The value of the API token generated. See the docs for more information.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "credentials"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - }, - "public": true, - "custom": false, - "releaseStage": "generally_available" - }, - { - "sourceDefinitionId": "f1e4c7f6-db5c-4035-981f-d35ab4998794", - "name": "Zenloop", - "dockerRepository": "airbyte/source-zenloop", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zenloop", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zenloop", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zenloop Spec", - "type": "object", - "required": ["api_token"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "description": "Zenloop API Token. You can get the API token in settings page here ", - "airbyte_secret": true - }, - "date_from": { - "type": "string", - "description": "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24. Leave empty if only data from current data should be synced", - "examples": ["2021-10-24T03:30:30Z"] - }, - "survey_id": { - "type": "string", - "description": "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys", - "airbyte_secret": true - }, - "survey_group_id": { - "type": "string", - "description": "Zenloop Survey Group ID. Can be found by pulling All Survey Groups via SurveyGroups stream. Leave empty to pull answers from all survey groups", - "airbyte_secret": true - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - }, - { - "sourceDefinitionId": "3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5", - "name": "Zuora", - "dockerRepository": "airbyte/source-zuora", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zuora", - "icon": "zuora.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zuora", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zuora Connector Configuration", - "type": "object", - "required": ["start_date", "client_id", "client_secret"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start Date in format: YYYY-MM-DD", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "window_in_days": { - "type": "integer", - "description": "The amount of days for each data-chunk begining from start_date. Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364, as for a Year).", - "examples": [30, 60, 90, 120, 200, 364], - "default": 90 - }, - "client_id": { - "type": "string", - "description": "Client ID", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "description": "Client Secret", - "airbyte_secret": true - }, - "is_sandbox": { - "type": "boolean", - "description": "Defines whether use the SANDBOX or PRODUCTION environment.", - "default": false - } - } - }, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": [] - }, - "public": true, - "custom": false, - "releaseStage": "alpha" - } - ] -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index cc3b7e90aaf5..946b011adba3 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -1,56 +1,43 @@ - -dependencies { - - implementation project(':airbyte-cdk:java:airbyte-cdk:core') - implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations')) - - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - - testImplementation 'org.mockito:mockito-core:4.6.1' - - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' - - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} - implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - implementation 'org.apache.commons:commons-csv:1.4' - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - implementation libs.bundles.junit - testImplementation libs.junit.jupiter.system.stubs - - -} - java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs.remove("-Werror") + options.compilerArgs += "-Xlint:-try,-deprecation,-this-escape" } compileTestJava { options.compilerArgs += "-Xlint:-try" } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-deprecation" + } +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + + // Re-export dependencies for gcs-destinations. + api 'com.amazonaws:aws-java-sdk-s3:1.12.647' + api ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + api 'com.github.alexmojaki:s3-stream-upload:2.2.4' + api 'org.apache.avro:avro:1.11.3' + api 'org.apache.commons:commons-csv:1.10.0' + api 'org.apache.commons:commons-text:1.11.0' + api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } + api 'org.apache.hadoop:hadoop-common:3.3.6' + api 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6' + api 'org.apache.parquet:parquet-avro:1.13.1' + runtimeOnly 'com.hadoop.gplcompression:hadoop-lzo:0.4.20' + + testImplementation 'org.mockito:mockito-inline:5.2.0' + + testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations')) } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java deleted file mode 100644 index bdf669194a91..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; - -import com.amazonaws.services.s3.AmazonS3; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; -import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter; -import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; -import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.io.IOException; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import org.apache.commons.csv.CSVFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class S3StreamCopier implements StreamCopier { - - private static final Logger LOGGER = LoggerFactory.getLogger(S3StreamCopier.class); - - private static final int DEFAULT_UPLOAD_THREADS = 10; // The S3 cli uses 10 threads by default. - private static final int DEFAULT_QUEUE_CAPACITY = DEFAULT_UPLOAD_THREADS; - - protected final AmazonS3 s3Client; - protected final S3DestinationConfig s3Config; - protected final String tmpTableName; - protected final String schemaName; - protected final String streamName; - protected final JdbcDatabase db; - protected final ConfiguredAirbyteStream configuredAirbyteStream; - protected final String stagingFolder; - protected final Map stagingWritersByFile = new HashMap<>(); - private final DestinationSyncMode destSyncMode; - private final StandardNameTransformer nameTransformer; - private final SqlOperations sqlOperations; - private final Timestamp uploadTime; - protected final Set activeStagingWriterFileNames = new HashSet<>(); - protected final Set stagingFileNames = new LinkedHashSet<>(); - private final boolean purgeStagingData; - - // The number of batches of records that will be inserted into each file. - private final int maxPartsPerFile; - // The number of batches inserted into the current file. - private int partsAddedToCurrentFile; - private String currentFile; - - public S3StreamCopier(final String stagingFolder, - final String schema, - final AmazonS3 client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final ConfiguredAirbyteStream configuredAirbyteStream, - final Timestamp uploadTime, - final int maxPartsPerFile) { - this.destSyncMode = configuredAirbyteStream.getDestinationSyncMode(); - this.schemaName = schema; - this.streamName = configuredAirbyteStream.getStream().getName(); - this.stagingFolder = stagingFolder; - this.db = db; - this.nameTransformer = nameTransformer; - this.sqlOperations = sqlOperations; - this.configuredAirbyteStream = configuredAirbyteStream; - this.uploadTime = uploadTime; - this.tmpTableName = nameTransformer.getTmpTableName(this.streamName); - this.s3Client = client; - this.s3Config = config.s3Config(); - this.purgeStagingData = config.purgeStagingData(); - - this.maxPartsPerFile = maxPartsPerFile; - this.partsAddedToCurrentFile = 0; - } - - @Override - public String prepareStagingFile() { - if (partsAddedToCurrentFile == 0) { - - try { - // The Flattening value is actually ignored, because we pass an explicit CsvSheetGenerator. So just - // pass in null. - final S3FormatConfig csvFormatConfig = new S3CsvFormatConfig(null, CompressionType.NO_COMPRESSION); - final S3DestinationConfig writerS3Config = S3DestinationConfig.create(s3Config).withFormatConfig(csvFormatConfig).get(); - final S3CsvWriter writer = new S3CsvWriter.Builder( - writerS3Config, - s3Client, - configuredAirbyteStream, - uploadTime) - .uploadThreads(DEFAULT_UPLOAD_THREADS) - .queueCapacity(DEFAULT_QUEUE_CAPACITY) - .csvSettings(CSVFormat.DEFAULT) - .withHeader(false) - .csvSheetGenerator(new StagingDatabaseCsvSheetGenerator()) - .build(); - currentFile = writer.getOutputPath(); - stagingWritersByFile.put(currentFile, writer); - activeStagingWriterFileNames.add(currentFile); - stagingFileNames.add(currentFile); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - partsAddedToCurrentFile = (partsAddedToCurrentFile + 1) % maxPartsPerFile; - return currentFile; - } - - @Override - public void write(final UUID id, final AirbyteRecordMessage recordMessage, final String filename) throws Exception { - if (stagingWritersByFile.containsKey(filename)) { - stagingWritersByFile.get(filename).write(id, recordMessage); - } - } - - @Override - public void closeNonCurrentStagingFileWriters() throws Exception { - final Set removedKeys = new HashSet<>(); - for (final String key : activeStagingWriterFileNames) { - if (!key.equals(currentFile)) { - stagingWritersByFile.get(key).close(false); - stagingWritersByFile.remove(key); - removedKeys.add(key); - } - } - activeStagingWriterFileNames.removeAll(removedKeys); - } - - @Override - public void closeStagingUploader(final boolean hasFailed) throws Exception { - for (final DestinationFileWriter writer : stagingWritersByFile.values()) { - writer.close(hasFailed); - } - } - - @Override - public void createDestinationSchema() throws Exception { - LOGGER.info("Creating schema in destination if it doesn't exist: {}", schemaName); - sqlOperations.createSchemaIfNotExists(db, schemaName); - } - - @Override - public void createTemporaryTable() throws Exception { - LOGGER.info("Preparing tmp table in destination for stream: {}, schema: {}, tmp table name: {}.", streamName, schemaName, tmpTableName); - sqlOperations.createTableIfNotExists(db, schemaName, tmpTableName); - } - - @Override - public void copyStagingFileToTemporaryTable() throws Exception { - LOGGER.info("Starting copy to tmp table: {} in destination for stream: {}, schema: {}, .", tmpTableName, streamName, schemaName); - for (final String fileName : stagingFileNames) { - copyS3CsvFileIntoTable(db, getFullS3Path(s3Config.getBucketName(), fileName), schemaName, tmpTableName, s3Config); - } - LOGGER.info("Copy to tmp table {} in destination for stream {} complete.", tmpTableName, streamName); - } - - @Override - public String createDestinationTable() throws Exception { - final var destTableName = nameTransformer.getRawTableName(streamName); - LOGGER.info("Preparing table {} in destination.", destTableName); - sqlOperations.createTableIfNotExists(db, schemaName, destTableName); - LOGGER.info("Table {} in destination prepared.", tmpTableName); - - return destTableName; - } - - @Override - public String generateMergeStatement(final String destTableName) { - LOGGER.info("Preparing to merge tmp table {} to dest table: {}, schema: {}, in destination.", tmpTableName, destTableName, schemaName); - final var queries = new StringBuilder(); - if (destSyncMode.equals(DestinationSyncMode.OVERWRITE)) { - queries.append(sqlOperations.truncateTableQuery(db, schemaName, destTableName)); - LOGGER.info("Destination OVERWRITE mode detected. Dest table: {}, schema: {}, truncated.", destTableName, schemaName); - } - queries.append(sqlOperations.insertTableQuery(db, schemaName, tmpTableName, destTableName)); - return queries.toString(); - } - - @Override - public void removeFileAndDropTmpTable() throws Exception { - if (purgeStagingData) { - for (final String fileName : stagingFileNames) { - s3Client.deleteObject(s3Config.getBucketName(), fileName); - LOGGER.info("S3 staging file {} cleaned.", fileName); - } - } - - LOGGER.info("Begin cleaning {} tmp table in destination.", tmpTableName); - sqlOperations.dropTableIfExists(db, schemaName, tmpTableName); - LOGGER.info("{} tmp table in destination cleaned.", tmpTableName); - } - - @Override - public String getCurrentFile() { - return currentFile; - } - - protected static String getFullS3Path(final String s3BucketName, final String s3StagingFile) { - return String.join("/", "s3:/", s3BucketName, s3StagingFile); - } - - @VisibleForTesting - public String getTmpTableName() { - return tmpTableName; - } - - @VisibleForTesting - public Map getStagingWritersByFile() { - return stagingWritersByFile; - } - - @VisibleForTesting - public Set getStagingFiles() { - return stagingFileNames; - } - - public abstract void copyS3CsvFileIntoTable(JdbcDatabase database, - String s3FileLocation, - String schema, - String tableName, - S3DestinationConfig s3Config) - throws SQLException; - -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java deleted file mode 100644 index b9b94c72c329..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; - -public abstract class S3StreamCopierFactory implements StreamCopierFactory { - - /** - * Used by the copy consumer. - */ - @Override - public StreamCopier create(final String configuredSchema, - final S3CopyConfig config, - final String stagingFolder, - final ConfiguredAirbyteStream configuredStream, - final StandardNameTransformer nameTransformer, - final JdbcDatabase db, - final SqlOperations sqlOperations) { - try { - final AirbyteStream stream = configuredStream.getStream(); - final String schema = StreamCopierFactory.getSchema(stream.getNamespace(), configuredSchema, nameTransformer); - final AmazonS3 s3Client = config.s3Config().getS3Client(); - - return create(stagingFolder, schema, s3Client, db, config, nameTransformer, sqlOperations, configuredStream); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - /** - * For specific copier suppliers to implement. - */ - protected abstract StreamCopier create(String stagingFolder, - String schema, - AmazonS3 s3Client, - JdbcDatabase db, - S3CopyConfig config, - StandardNameTransformer nameTransformer, - SqlOperations sqlOperations, - ConfiguredAirbyteStream configuredStream) - throws Exception; - -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java index dfb0d0a50822..9df281e9e19b 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java @@ -31,7 +31,7 @@ protected BlobStorageOperations() { * * @return the name of the file that was uploaded. */ - public abstract String uploadRecordsToBucket(SerializableBuffer recordsData, String namespace, String streamName, String objectPath) + public abstract String uploadRecordsToBucket(SerializableBuffer recordsData, String namespace, String objectPath) throws Exception; /** diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java index b7de68235d33..38068dbf38c1 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java @@ -128,7 +128,6 @@ private FlushBufferFunction flushBufferFunction(final BlobStorageOperations stor writeConfig.addStoredFile(storageOperations.uploadRecordsToBucket( writer, writeConfig.getNamespace(), - writeConfig.getStreamName(), writeConfig.getFullOutputPath())); } catch (final Exception e) { LOGGER.error("Failed to flush and upload buffer to storage:", e); @@ -139,7 +138,7 @@ private FlushBufferFunction flushBufferFunction(final BlobStorageOperations stor private OnCloseFunction onCloseFunction(final BlobStorageOperations storageOperations, final List writeConfigs) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { if (hasFailed) { LOGGER.info("Cleaning up destination started for {} streams", writeConfigs.size()); for (final WriteConfig writeConfig : writeConfigs) { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java index 042b664e5b2a..9db0d0d4994a 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; @@ -65,6 +68,7 @@ public class S3StorageOperations extends BlobStorageOperations { private static final String FORMAT_VARIABLE_EPOCH = "${EPOCH}"; private static final String FORMAT_VARIABLE_UUID = "${UUID}"; private static final String GZ_FILE_EXTENSION = "gz"; + private final ConcurrentMap partCounts = new ConcurrentHashMap<>(); private final NamingConventionTransformer nameTransformer; protected final S3DestinationConfig s3Config; @@ -116,7 +120,6 @@ protected boolean doesBucketExist(final String bucket) { @Override public String uploadRecordsToBucket(final SerializableBuffer recordsData, final String namespace, - final String streamName, final String objectPath) { final List exceptionsThrown = new ArrayList<>(); while (exceptionsThrown.size() < UPLOAD_RETRY_LIMIT) { @@ -156,7 +159,7 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData, private String loadDataIntoBucket(final String objectPath, final SerializableBuffer recordsData) throws IOException { final long partSize = DEFAULT_PART_SIZE; final String bucket = s3Config.getBucketName(); - final String partId = UUID.randomUUID().toString(); + final String partId = getPartId(objectPath); final String fileExtension = getExtension(recordsData.getFilename()); final String fullObjectKey; if (StringUtils.isNotBlank(s3Config.getFileNamePattern())) { @@ -215,6 +218,41 @@ private String loadDataIntoBucket(final String objectPath, final SerializableBuf return newFilename; } + /** + * Users want deterministic file names (e.g. the first file part is really foo-0.csv). Using UUIDs + * (previous approach) doesn't allow that. However, using pure integers could lead to a collision + * with an upload from another thread. We also want to be able to continue the same offset between + * attempts. So, we'll count up the existing files in the directory and use that as a lazy-offset, + * assuming airbyte manages the dir and has similar naming conventions. `getPartId` will be + * 0-indexed. + */ + @VisibleForTesting + synchronized String getPartId(String objectPath) { + final AtomicInteger partCount = partCounts.computeIfAbsent(objectPath, k -> new AtomicInteger(0)); + + if (partCount.get() == 0) { + ObjectListing objects; + int objectCount = 0; + + final String bucket = s3Config.getBucketName(); + objects = s3Client.listObjects(bucket, objectPath); + + if (objects != null) { + objectCount = objectCount + objects.getObjectSummaries().size(); + while (objects != null && objects.getNextMarker() != null) { + objects = s3Client.listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(objectPath).withMarker(objects.getNextMarker())); + if (objects != null) { + objectCount = objectCount + objects.getObjectSummaries().size(); + } + } + } + + partCount.set(objectCount); + } + + return Integer.toString(partCount.getAndIncrement()); + } + @VisibleForTesting static String getFilename(final String fullPath) { return fullPath.substring(fullPath.lastIndexOf("/") + 1); diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java index d3adf4ff43ce..0ad036367837 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java @@ -37,16 +37,6 @@ class AsyncFlush implements DestinationFlushFunction { private final long optimalBatchSizeBytes; private final boolean useDestinationsV2Columns; - public AsyncFlush(final Map streamDescToWriteConfig, - final StagingOperations stagingOperations, - final JdbcDatabase database, - final ConfiguredAirbyteCatalog catalog, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper, - final boolean useDestinationsV2Columns) { - this(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, 50 * 1024 * 1024, useDestinationsV2Columns); - } - public AsyncFlush(final Map streamDescToWriteConfig, final StagingOperations stagingOperations, final JdbcDatabase database, @@ -106,7 +96,7 @@ public void flush(final StreamDescriptor decs, final Stream outputRecordCollector, - final JdbcDatabase database, - final StagingOperations stagingOperations, - final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper, - final ParsedCatalog parsedCatalog, - final String defaultNamespace, - final boolean useDestinationsV2Columns) { - return createAsync(outputRecordCollector, - database, - stagingOperations, - namingResolver, - config, - catalog, - purgeStagingData, - typerDeduperValve, - typerDeduper, - parsedCatalog, - defaultNamespace, - useDestinationsV2Columns, - Optional.empty()); + private static final Instant SYNC_DATETIME = Instant.now(); + + private final Consumer outputRecordCollector; + private final JdbcDatabase database; + private final StagingOperations stagingOperations; + private final NamingConventionTransformer namingResolver; + private final JsonNode config; + private final ConfiguredAirbyteCatalog catalog; + private final boolean purgeStagingData; + private final TypeAndDedupeOperationValve typerDeduperValve; + private final TyperDeduper typerDeduper; + private final ParsedCatalog parsedCatalog; + private final String defaultNamespace; + private final boolean useDestinationsV2Columns; + + // Optional fields + private final Optional bufferMemoryLimit; + private final long optimalBatchSizeBytes; + + private StagingConsumerFactory( + final Consumer outputRecordCollector, + final JdbcDatabase database, + final StagingOperations stagingOperations, + final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace, + final boolean useDestinationsV2Columns, + final Optional bufferMemoryLimit, + final long optimalBatchSizeBytes) { + this.outputRecordCollector = outputRecordCollector; + this.database = database; + this.stagingOperations = stagingOperations; + this.namingResolver = namingResolver; + this.config = config; + this.catalog = catalog; + this.purgeStagingData = purgeStagingData; + this.typerDeduperValve = typerDeduperValve; + this.typerDeduper = typerDeduper; + this.parsedCatalog = parsedCatalog; + this.defaultNamespace = defaultNamespace; + this.useDestinationsV2Columns = useDestinationsV2Columns; + this.bufferMemoryLimit = bufferMemoryLimit; + this.optimalBatchSizeBytes = optimalBatchSizeBytes; } - public SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, - final JdbcDatabase database, - final StagingOperations stagingOperations, - final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper, - final ParsedCatalog parsedCatalog, - final String defaultNamespace, - final boolean useDestinationsV2Columns, - final Optional bufferMemoryLimit) { + public static class Builder { + + // Required (?) fields + // (TODO which of these are _actually_ required, and which have we just coincidentally always + // provided?) + private Consumer outputRecordCollector; + private JdbcDatabase database; + private StagingOperations stagingOperations; + private NamingConventionTransformer namingResolver; + private JsonNode config; + private ConfiguredAirbyteCatalog catalog; + private boolean purgeStagingData; + private TypeAndDedupeOperationValve typerDeduperValve; + private TyperDeduper typerDeduper; + private ParsedCatalog parsedCatalog; + private String defaultNamespace; + private boolean useDestinationsV2Columns; + + // Optional fields + private Optional bufferMemoryLimit = Optional.empty(); + private long optimalBatchSizeBytes = 50 * 1024 * 1024; + + private Builder() {} + + public Builder setBufferMemoryLimit(final Optional bufferMemoryLimit) { + this.bufferMemoryLimit = bufferMemoryLimit; + return this; + } + + public Builder setOptimalBatchSizeBytes(final long optimalBatchSizeBytes) { + this.optimalBatchSizeBytes = optimalBatchSizeBytes; + return this; + } + + public StagingConsumerFactory build() { + return new StagingConsumerFactory( + outputRecordCollector, + database, + stagingOperations, + namingResolver, + config, + catalog, + purgeStagingData, + typerDeduperValve, + typerDeduper, + parsedCatalog, + defaultNamespace, + useDestinationsV2Columns, + bufferMemoryLimit, + optimalBatchSizeBytes); + } + + } + + public static Builder builder( + final Consumer outputRecordCollector, + final JdbcDatabase database, + final StagingOperations stagingOperations, + final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace, + final boolean useDestinationsV2Columns) { + final Builder builder = new Builder(); + builder.outputRecordCollector = outputRecordCollector; + builder.database = database; + builder.stagingOperations = stagingOperations; + builder.namingResolver = namingResolver; + builder.config = config; + builder.catalog = catalog; + builder.purgeStagingData = purgeStagingData; + builder.typerDeduperValve = typerDeduperValve; + builder.typerDeduper = typerDeduper; + builder.parsedCatalog = parsedCatalog; + builder.defaultNamespace = defaultNamespace; + builder.useDestinationsV2Columns = useDestinationsV2Columns; + return builder; + } + + public SerializedAirbyteMessageConsumer createAsync() { final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog, useDestinationsV2Columns); final var streamDescToWriteConfig = streamDescToWriteConfig(writeConfigs); - final var flusher = - new AsyncFlush(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, useDestinationsV2Columns); + final var flusher = new AsyncFlush( + streamDescToWriteConfig, + stagingOperations, + database, + catalog, + typerDeduperValve, + typerDeduper, + optimalBatchSizeBytes, + useDestinationsV2Columns); return new AsyncStreamConsumer( outputRecordCollector, GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), // todo (cgardens) - wrapping the old close function to avoid more code churn. - (hasFailed) -> { + (hasFailed, streamSyncSummaries) -> { try { - GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper).accept(false); - } catch (Exception e) { + GeneralStagingFunctions.onCloseFunction( + database, + stagingOperations, + writeConfigs, + purgeStagingData, + typerDeduper).accept(false, streamSyncSummaries); + } catch (final Exception e) { throw new RuntimeException(e); } }, @@ -121,7 +214,7 @@ public SerializedAirbyteMessageConsumer createAsync(final Consumer bufferMemoryLimit) { + private static long getMemoryLimit(final Optional bufferMemoryLimit) { return bufferMemoryLimit.orElse((long) (Runtime.getRuntime().maxMemory() * MEMORY_LIMIT_RATIO)); } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java deleted file mode 100644 index 770643e875e4..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockConstruction; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.google.common.collect.Lists; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.base.DestinationConfig; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator; -import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter; -import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import org.apache.commons.csv.CSVFormat; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.MockedConstruction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class S3StreamCopierTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(S3StreamCopierTest.class); - - private static final S3DestinationConfig S3_CONFIG = S3DestinationConfig.create( - "fake-bucket", - "fake-bucketPath", - "fake-region") - .withEndpoint("fake-endpoint") - .withAccessKeyCredential("fake-access-key-id", "fake-secret-access-key") - .get(); - private static final ConfiguredAirbyteStream CONFIGURED_STREAM = new ConfiguredAirbyteStream() - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(new AirbyteStream() - .withName("fake-stream") - .withNamespace("fake-namespace") - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))); - private static final int UPLOAD_THREADS = 10; - private static final int QUEUE_CAPACITY = 10; - // equivalent to Thu, 09 Dec 2021 19:17:54 GMT - private static final Timestamp UPLOAD_TIME = Timestamp.from(Instant.ofEpochMilli(1639077474000L)); - private static final int MAX_PARTS_PER_FILE = 42; - - private AmazonS3Client s3Client; - private JdbcDatabase db; - private SqlOperations sqlOperations; - private S3StreamCopier copier; - - private MockedConstruction csvWriterMockedConstruction; - private List csvWriterConstructorArguments; - - private List copyArguments; - - private record S3CsvWriterArguments(S3DestinationConfig config, - ConfiguredAirbyteStream stream, - Timestamp uploadTime, - int uploadThreads, - int queueCapacity, - boolean writeHeader, - CSVFormat csvSettings, - CsvSheetGenerator csvSheetGenerator) { - - } - - private record CopyArguments(JdbcDatabase database, - String s3FileLocation, - String schema, - String tableName, - S3DestinationConfig s3Config) { - - } - - @BeforeEach - public void setup() { - DestinationConfig.initialize(Jsons.emptyObject()); - - s3Client = mock(AmazonS3Client.class); - db = mock(JdbcDatabase.class); - sqlOperations = mock(SqlOperations.class); - - csvWriterConstructorArguments = new ArrayList<>(); - copyArguments = new ArrayList<>(); - - // This is basically RETURNS_SELF, except with getMultiPartOutputStreams configured correctly. - // Other non-void methods (e.g. toString()) will return null. - csvWriterMockedConstruction = mockConstruction( - S3CsvWriter.class, - (mock, context) -> { - // Normally, the S3CsvWriter would return a path that ends in a UUID, but this mock will generate an - // int ID to make our asserts easier. - doReturn(String.format("fakeOutputPath-%05d", csvWriterConstructorArguments.size())).when(mock).getOutputPath(); - - // Mockito doesn't seem to provide an easy way to actually retrieve these arguments later on, so - // manually store them on construction. - // _PowerMockito_ does, but I didn't want to set up that additional dependency. - final List arguments = context.arguments(); - csvWriterConstructorArguments.add(new S3CsvWriterArguments( - (S3DestinationConfig) arguments.get(0), - (ConfiguredAirbyteStream) arguments.get(2), - (Timestamp) arguments.get(3), - (int) arguments.get(4), - (int) arguments.get(5), - (boolean) arguments.get(6), - (CSVFormat) arguments.get(7), - (CsvSheetGenerator) arguments.get(8))); - }); - - copier = new S3StreamCopier( - // In reality, this is normally a UUID - see CopyConsumerFactory#createWriteConfigs - "fake-staging-folder", - "fake-schema", - s3Client, - db, - new S3CopyConfig(true, S3_CONFIG), - new StandardNameTransformer(), - sqlOperations, - CONFIGURED_STREAM, - UPLOAD_TIME, - MAX_PARTS_PER_FILE) { - - @Override - public void copyS3CsvFileIntoTable( - final JdbcDatabase database, - final String s3FileLocation, - final String schema, - final String tableName, - final S3DestinationConfig s3Config) { - copyArguments.add(new CopyArguments(database, s3FileLocation, schema, tableName, s3Config)); - } - - }; - } - - @AfterEach - public void teardown() { - csvWriterMockedConstruction.close(); - } - - @Test - public void createSequentialStagingFiles_when_multipleFilesRequested() { - // When we call prepareStagingFile() the first time, it should create exactly one S3CsvWriter. The - // next (MAX_PARTS_PER_FILE - 1) invocations - // should reuse that same writer. - for (var i = 0; i < MAX_PARTS_PER_FILE; i++) { - final String file = copier.prepareStagingFile(); - assertEquals("fakeOutputPath-00000", file, "preparing file number " + i); - assertEquals(1, csvWriterMockedConstruction.constructed().size()); - checkCsvWriterArgs(csvWriterConstructorArguments.get(0)); - } - - // Now that we've hit the MAX_PARTS_PER_FILE, we should start a new writer - final String secondFile = copier.prepareStagingFile(); - assertEquals("fakeOutputPath-00001", secondFile); - final List secondManagers = csvWriterMockedConstruction.constructed(); - assertEquals(2, secondManagers.size()); - checkCsvWriterArgs(csvWriterConstructorArguments.get(1)); - } - - private void checkCsvWriterArgs(final S3CsvWriterArguments args) { - final S3DestinationConfig s3Config = S3DestinationConfig.create(S3_CONFIG) - .withFormatConfig(new S3CsvFormatConfig(null, CompressionType.NO_COMPRESSION)) - .get(); - assertEquals(s3Config, args.config); - assertEquals(CONFIGURED_STREAM, args.stream); - assertEquals(UPLOAD_TIME, args.uploadTime); - assertEquals(UPLOAD_THREADS, args.uploadThreads); - assertEquals(QUEUE_CAPACITY, args.queueCapacity); - assertFalse(args.writeHeader); - assertEquals(CSVFormat.DEFAULT, args.csvSettings); - assertTrue( - args.csvSheetGenerator instanceof StagingDatabaseCsvSheetGenerator, - "Sheet generator was actually a " + args.csvSheetGenerator.getClass()); - } - - @Test - public void closesS3Upload_when_stagingUploaderClosedSuccessfully() throws Exception { - copier.prepareStagingFile(); - - copier.closeStagingUploader(false); - - final List managers = csvWriterMockedConstruction.constructed(); - final S3CsvWriter manager = managers.get(0); - verify(manager).close(false); - } - - @Test - public void closesS3Upload_when_stagingUploaderClosedFailingly() throws Exception { - copier.prepareStagingFile(); - - copier.closeStagingUploader(true); - - final List managers = csvWriterMockedConstruction.constructed(); - final S3CsvWriter manager = managers.get(0); - verify(manager).close(true); - } - - @Test - public void deletesStagingFiles() throws Exception { - copier.prepareStagingFile(); - doReturn(true).when(s3Client).doesObjectExist("fake-bucket", "fakeOutputPath-00000"); - - copier.removeFileAndDropTmpTable(); - - verify(s3Client).deleteObject("fake-bucket", "fakeOutputPath-00000"); - } - - @Test - public void doesNotDeleteStagingFiles_if_purgeStagingDataDisabled() throws Exception { - copier = new S3StreamCopier( - "fake-staging-folder", - "fake-schema", - s3Client, - db, - // Explicitly disable purgeStagingData - new S3CopyConfig(false, S3_CONFIG), - new StandardNameTransformer(), - sqlOperations, - CONFIGURED_STREAM, - UPLOAD_TIME, - MAX_PARTS_PER_FILE) { - - @Override - public void copyS3CsvFileIntoTable( - final JdbcDatabase database, - final String s3FileLocation, - final String schema, - final String tableName, - final S3DestinationConfig s3Config) { - copyArguments.add(new CopyArguments(database, s3FileLocation, schema, tableName, s3Config)); - } - - }; - - copier.prepareStagingFile(); - doReturn(true).when(s3Client).doesObjectExist("fake-bucket", "fakeOutputPath-00000"); - - copier.removeFileAndDropTmpTable(); - - verify(s3Client, never()).deleteObject("fake-bucket", "fakeOutputPath-00000"); - } - - @Test - public void copiesCorrectFilesToTable() throws Exception { - // Generate two files - for (int i = 0; i < MAX_PARTS_PER_FILE + 1; i++) { - copier.prepareStagingFile(); - } - - copier.copyStagingFileToTemporaryTable(); - - assertEquals(2, copyArguments.size(), "Number of invocations was actually " + copyArguments.size() + ". Arguments were " + copyArguments); - - // S3StreamCopier operates on these from a HashMap, so need to sort them in order to assert in a - // sane way. - final List sortedArgs = copyArguments.stream().sorted(Comparator.comparing(arg -> arg.s3FileLocation)).toList(); - for (int i = 0; i < sortedArgs.size(); i++) { - LOGGER.info("Checking arguments for index {}", i); - final CopyArguments args = sortedArgs.get(i); - assertEquals(String.format("s3://fake-bucket/fakeOutputPath-%05d", i), args.s3FileLocation); - assertEquals("fake-schema", args.schema); - assertTrue(args.tableName.endsWith("fake_stream"), "Table name was actually " + args.tableName); - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java index 3970edd43220..138e9d393ce5 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java @@ -19,7 +19,11 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; @@ -107,4 +111,57 @@ void testGetFilename() { assertEquals("filename.csv", S3StorageOperations.getFilename("/p1/p2/filename.csv")); } + @Test + void getPartId() throws InterruptedException { + + // Multithreaded utility class + class PartIdGetter implements Runnable { + + final List responses = new ArrayList<>(); + final S3StorageOperations s3StorageOperations; + + PartIdGetter(S3StorageOperations instance) { + s3StorageOperations = instance; + } + + public void run() { + responses.add(s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + } + + List getResponses() { + return responses; + } + + } + + PartIdGetter partIdGetter = new PartIdGetter(s3StorageOperations); + + // single threaded + partIdGetter.run(); // 0 + partIdGetter.run(); // 1 + partIdGetter.run(); // 2 + + // multithreaded + ExecutorService executor = Executors.newFixedThreadPool(3); + for (int i = 0; i < 7; i++) { + executor.execute(partIdGetter); + } + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.SECONDS); + + List responses = partIdGetter.getResponses(); + assertEquals(10, responses.size()); + for (int i = 0; i <= 9; i++) { + assertTrue(responses.contains(Integer.toString(i))); + } + } + + @Test + void getPartIdMultiplePaths() { + assertEquals("0", s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + assertEquals("1", s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + + assertEquals("0", s3StorageOperations.getPartId("other_path")); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java index d1fc9af1d91f..33faf55f8369 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java @@ -29,6 +29,7 @@ import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumReader; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class AvroSerializedBufferTest { @@ -57,6 +58,8 @@ public static void setup() { } @Test + @Disabled("Flaky on CI, See run https://github.com/airbytehq/airbyte/actions/runs/7126781640/job/19405426141?pr=33201 " + + "org.opentest4j.AssertionFailedError: Expected size between 964 and 985, but actual size was 991 ==> expected: but was: ") public void testSnappyAvroWriter() throws Exception { final S3AvroFormatConfig config = new S3AvroFormatConfig(Jsons.jsonNode(Map.of("compression_codec", Map.of( "codec", "snappy")))); diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java index 09c5b03c7aad..5fe69ffa9923 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java @@ -27,6 +27,7 @@ import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter.Builder; import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerWithMetadata; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; @@ -80,7 +81,7 @@ class S3CsvWriterTest { private AmazonS3 s3Client; - private MockedConstruction streamTransferManagerMockedConstruction; + private MockedConstruction streamTransferManagerMockedConstruction; private List streamTransferManagerConstructorArguments; private List outputStreams; @@ -95,7 +96,7 @@ public void setup() { // This is basically RETURNS_SELF, except with getMultiPartOutputStreams configured correctly. // Other non-void methods (e.g. toString()) will return null. streamTransferManagerMockedConstruction = mockConstruction( - StreamTransferManager.class, + StreamTransferManagerWithMetadata.class, (mock, context) -> { // Mockito doesn't seem to provide an easy way to actually retrieve these arguments later on, so // manually store them on construction. @@ -174,7 +175,7 @@ public void closesS3Upload_when_stagingUploaderClosedSuccessfully() throws Excep writer.close(false); - final List managers = streamTransferManagerMockedConstruction.constructed(); + final List managers = streamTransferManagerMockedConstruction.constructed(); final StreamTransferManager manager = managers.get(0); verify(manager).complete(); } @@ -185,7 +186,7 @@ public void closesS3Upload_when_stagingUploaderClosedFailingly() throws Exceptio writer.close(true); - final List managers = streamTransferManagerMockedConstruction.constructed(); + final List managers = streamTransferManagerMockedConstruction.constructed(); final StreamTransferManager manager = managers.get(0); verify(manager).abort(); } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java new file mode 100644 index 000000000000..3cbbc4bc433f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3; + +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +public class S3AvroParquetTestDataComparator extends AdvancedTestDataComparator { + + @Override + protected boolean compareDateValues(String airbyteMessageValue, String destinationValue) { + var destinationDate = LocalDate.ofEpochDay(Long.parseLong(destinationValue)); + var expectedDate = LocalDate.parse(airbyteMessageValue, DateTimeFormatter.ofPattern(AdvancedTestDataComparator.AIRBYTE_DATE_FORMAT)); + return expectedDate.equals(destinationDate); + } + + private Instant getInstantFromEpoch(String epochValue) { + return Instant.ofEpochMilli(Long.parseLong(epochValue) / 1000); + } + + @Override + protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { + return ZonedDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); + } + + @Override + protected boolean compareDateTimeValues(String airbyteMessageValue, String destinationValue) { + var format = DateTimeFormatter.ofPattern(AdvancedTestDataComparator.AIRBYTE_DATETIME_FORMAT); + LocalDateTime dateTime = LocalDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); + return super.compareDateTimeValues(airbyteMessageValue, format.format(dateTime)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java index a048fd69dbfb..0ba7ff5af30b 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java @@ -73,7 +73,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Override protected TestDataComparator getTestDataComparator() { - return new S3BaseAvroParquetTestDataComparator(); + return new S3AvroParquetTestDataComparator(); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java deleted file mode 100644 index 4fe9040168eb..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk.integrations.destination.s3; - -import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; - -public class S3BaseAvroParquetTestDataComparator extends AdvancedTestDataComparator { - - @Override - protected boolean compareDateValues(String airbyteMessageValue, String destinationValue) { - var destinationDate = LocalDate.ofEpochDay(Long.parseLong(destinationValue)); - var expectedDate = LocalDate.parse(airbyteMessageValue, DateTimeFormatter.ofPattern(AdvancedTestDataComparator.AIRBYTE_DATE_FORMAT)); - return expectedDate.equals(destinationDate); - } - - private Instant getInstantFromEpoch(String epochValue) { - return Instant.ofEpochMilli(Long.parseLong(epochValue) / 1000); - } - - @Override - protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { - return ZonedDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); - } - - @Override - protected boolean compareDateTimeValues(String airbyteMessageValue, String destinationValue) { - var format = DateTimeFormatter.ofPattern(AdvancedTestDataComparator.AIRBYTE_DATETIME_FORMAT); - LocalDateTime dateTime = LocalDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); - return super.compareDateTimeValues(airbyteMessageValue, format.format(dateTime)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java index 44abebe02905..d8a88f9591c6 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java @@ -78,7 +78,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Override protected TestDataComparator getTestDataComparator() { - return new S3BaseAvroParquetTestDataComparator(); + return new S3AvroParquetTestDataComparator(); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java index 11cb05dde761..7fd53f2aa513 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java @@ -64,11 +64,6 @@ protected JsonNode getBaseConfigJson() { return Jsons.deserialize(IOs.readFile(Path.of(secretFilePath))); } - @Override - protected String getImageName() { - return "airbyte/destination-s3:dev"; - } - @Override protected JsonNode getConfig() { return configJson; diff --git a/airbyte-cdk/java/airbyte-cdk/settings.gradle b/airbyte-cdk/java/airbyte-cdk/settings.gradle deleted file mode 100644 index 976a33b328ea..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/settings.gradle +++ /dev/null @@ -1,15 +0,0 @@ -rootProject.name = 'airbyte' - -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-api' -include ':airbyte-cdk:java:airbyte-cdk:config-models-oss' -include ':airbyte-cdk:java:airbyte-cdk:init-oss' -include ':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness' -include ':airbyte-cdk:java:airbyte-cdk:core' -include ':airbyte-cdk:java:airbyte-cdk:db-sources' -include ':airbyte-cdk:java:airbyte-cdk:db-destinations' -include ':airbyte-cdk:java:airbyte-cdk:s3-destinations' -include ':airbyte-cdk:java:airbyte-cdk:typing-deduping' diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index fc3ba8f061e8..9ec539430fa0 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -1,26 +1,21 @@ -plugins { - id 'java-library' +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-rawtypes,-unchecked" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-varargs" + } } dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:core') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation group: 'commons-codec', name: 'commons-codec', version: '1.16.0' - implementation libs.jooq - testFixturesImplementation libs.airbyte.protocol + implementation 'commons-codec:commons-codec:1.16.0' - testFixturesImplementation(platform('org.junit:junit-bom:5.8.2')) - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params' - testFixturesImplementation 'org.mockito:mockito-core:4.6.1' -} - -java { - compileJava { - options.compilerArgs.remove("-Werror") - } + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java index a33c3b715630..95c5841241b7 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java @@ -13,14 +13,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class BaseDestinationV1V2Migrator implements DestinationV1V2Migrator { +public abstract class BaseDestinationV1V2Migrator implements DestinationV1V2Migrator { protected static final Logger LOGGER = LoggerFactory.getLogger(BaseDestinationV1V2Migrator.class); @Override public void migrateIfNecessary( - final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final StreamConfig streamConfig) throws Exception { LOGGER.info("Assessing whether migration is necessary for stream {}", streamConfig.id().finalName()); @@ -59,8 +59,8 @@ protected boolean shouldMigrate(final StreamConfig streamConfig) throws Exceptio * @param destinationHandler the class which executes the sql statements * @param streamConfig the stream to migrate the raw table of */ - public void migrate(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + public void migrate(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final StreamConfig streamConfig) throws TableNotMigratedException { final var namespacedTableName = convertToV1RawName(streamConfig); diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java index 925d5037ea28..372f2999be64 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java @@ -23,14 +23,14 @@ public class CatalogParser { private static final Logger LOGGER = LoggerFactory.getLogger(CatalogParser.class); - private final SqlGenerator sqlGenerator; + private final SqlGenerator sqlGenerator; private final String rawNamespace; - public CatalogParser(final SqlGenerator sqlGenerator) { + public CatalogParser(final SqlGenerator sqlGenerator) { this(sqlGenerator, DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); } - public CatalogParser(final SqlGenerator sqlGenerator, final String rawNamespace) { + public CatalogParser(final SqlGenerator sqlGenerator, final String rawNamespace) { this.sqlGenerator = sqlGenerator; this.rawNamespace = rawNamespace; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java index de386113c39f..ec49be79cb57 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -5,17 +5,24 @@ package io.airbyte.integrations.base.destination.typing_deduping; import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; -import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.cdk.integrations.util.ConnectorExceptionUtil.getResultsOrLogAndThrowFirst; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.*; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; +import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; import static java.util.Collections.singleton; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.commons.concurrency.CompletableFutures; +import io.airbyte.commons.functional.Either; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.time.Instant; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -43,22 +50,22 @@ * Note that #prepareTables() initializes some internal state. The other methods will throw an * exception if that method was not called. */ -public class DefaultTyperDeduper implements TyperDeduper { +public class DefaultTyperDeduper implements TyperDeduper { private static final Logger LOGGER = LoggerFactory.getLogger(TyperDeduper.class); private static final String NO_SUFFIX = ""; private static final String TMP_OVERWRITE_TABLE_SUFFIX = "_airbyte_tmp"; - private final SqlGenerator sqlGenerator; - private final DestinationHandler destinationHandler; + private final SqlGenerator sqlGenerator; + private final DestinationHandler destinationHandler; - private final DestinationV1V2Migrator v1V2Migrator; + private final DestinationV1V2Migrator v1V2Migrator; private final V2TableMigrator v2TableMigrator; private final ParsedCatalog parsedCatalog; private Set overwriteStreamsWithTmpTable; private final Set> streamsWithSuccessfulSetup; - private final Map> minExtractedAtByStream; + private final Map initialRawTableStateByStream; // We only want to run a single instance of T+D per stream at a time. These objects are used for // synchronization per stream. // Use a read-write lock because we need the same semantics: @@ -72,64 +79,82 @@ public class DefaultTyperDeduper implements TyperDeduper private final ExecutorService executorService; - public DefaultTyperDeduper(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + public DefaultTyperDeduper(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final ParsedCatalog parsedCatalog, - final DestinationV1V2Migrator v1V2Migrator, - final V2TableMigrator v2TableMigrator, - final int defaultThreadCount) { + final DestinationV1V2Migrator v1V2Migrator, + final V2TableMigrator v2TableMigrator) { this.sqlGenerator = sqlGenerator; this.destinationHandler = destinationHandler; this.parsedCatalog = parsedCatalog; this.v1V2Migrator = v1V2Migrator; this.v2TableMigrator = v2TableMigrator; - this.minExtractedAtByStream = new ConcurrentHashMap<>(); + this.initialRawTableStateByStream = new ConcurrentHashMap<>(); this.streamsWithSuccessfulSetup = ConcurrentHashMap.newKeySet(parsedCatalog.streams().size()); this.tdLocks = new ConcurrentHashMap<>(); this.internalTdLocks = new ConcurrentHashMap<>(); - this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), + this.executorService = Executors.newFixedThreadPool(getCountOfTypeAndDedupeThreads(), new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); } - public DefaultTyperDeduper( - final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + public DefaultTyperDeduper(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final ParsedCatalog parsedCatalog, - final DestinationV1V2Migrator v1V2Migrator, - final int defaultThreadCount) { - this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2TableMigrator(), defaultThreadCount); + final DestinationV1V2Migrator v1V2Migrator) { + this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2TableMigrator()); } + private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception { + prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler); + } + + @Override public void prepareTables() throws Exception { if (overwriteStreamsWithTmpTable != null) { throw new IllegalStateException("Tables were already prepared."); } overwriteStreamsWithTmpTable = ConcurrentHashMap.newKeySet(); - LOGGER.info("Preparing final tables"); - final Set>> prepareTablesTasks = new HashSet<>(); - for (final StreamConfig stream : parsedCatalog.streams()) { - prepareTablesTasks.add(prepareTablesFuture(stream)); - } - CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); - reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); + LOGGER.info("Preparing tables"); + + // This is intentionally not done in parallel to avoid rate limits in some destinations. + prepareSchemas(parsedCatalog); + + // TODO: Either the migrations run the soft reset and create v2 tables or the actual prepare tables. + // unify the logic with current state of raw tables & final tables. This is done first before gather + // initial state to avoid recreating final tables later again. + final List> runMigrationsResult = + CompletableFutures.allOf(parsedCatalog.streams().stream().map(this::runMigrationsAsync).toList()).toCompletableFuture().join(); + getResultsOrLogAndThrowFirst("The following exceptions were thrown attempting to run migrations:\n", runMigrationsResult); + final List initialStates = destinationHandler.gatherInitialState(parsedCatalog.streams()); + final List> prepareTablesFutureResult = CompletableFutures.allOf( + initialStates.stream().map(this::prepareTablesFuture).toList()).toCompletableFuture().join(); + getResultsOrLogAndThrowFirst("The following exceptions were thrown attempting to prepare tables:\n", prepareTablesFutureResult); } - private CompletableFuture> prepareTablesFuture(final StreamConfig stream) { + private CompletionStage runMigrationsAsync(StreamConfig streamConfig) { + return CompletableFuture.runAsync(() -> { + try { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, streamConfig); + v2TableMigrator.migrateIfNecessary(streamConfig); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, this.executorService); + } + + private CompletionStage prepareTablesFuture(final DestinationInitialState initialState) { // For each stream, make sure that its corresponding final table exists. // Also, for OVERWRITE streams, decide if we're writing directly to the final table, or into an // _airbyte_tmp table. return CompletableFuture.supplyAsync(() -> { + final var stream = initialState.streamConfig(); try { - // Migrate the Raw Tables if this is the first v2 sync after a v1 sync - v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); - v2TableMigrator.migrateIfNecessary(stream); - - final Optional existingTable = destinationHandler.findExistingTable(stream.id()); - if (existingTable.isPresent()) { + if (initialState.isFinalTablePresent()) { LOGGER.info("Final Table exists for stream {}", stream.id().finalName()); // The table already exists. Decide whether we're writing to it directly, or using a tmp table. if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { - if (!destinationHandler.isFinalTableEmpty(stream.id()) || !sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + if (!initialState.isFinalTableEmpty() || initialState.isSchemaMismatch()) { // We want to overwrite an existing table. Write into a tmp table. We'll overwrite the table at the // end of the sync. overwriteStreamsWithTmpTable.add(stream.id()); @@ -141,7 +166,7 @@ private CompletableFuture> prepareTablesFuture(final StreamC stream.id().finalName()); } - } else if (!sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + } else if (initialState.isSchemaMismatch()) { // We're loading data directly into the existing table. Make sure it has the right schema. TypeAndDedupeTransaction.executeSoftReset(sqlGenerator, destinationHandler, stream); } @@ -150,8 +175,8 @@ private CompletableFuture> prepareTablesFuture(final StreamC // The table doesn't exist. Create it. Don't force. destinationHandler.execute(sqlGenerator.createTable(stream, NO_SUFFIX, false)); } - final Optional minTimestampForSync = destinationHandler.getMinTimestampForSync(stream.id()); - minExtractedAtByStream.put(stream.id(), minTimestampForSync); + + initialRawTableStateByStream.put(stream.id(), initialState.initialRawTableState()); streamsWithSuccessfulSetup.add(Pair.of(stream.id().originalNamespace(), stream.id().originalName())); @@ -164,10 +189,10 @@ private CompletableFuture> prepareTablesFuture(final StreamC // immediately acquire the lock. internalTdLocks.put(stream.id(), new ReentrantLock()); - return Optional.empty(); + return null; } catch (final Exception e) { LOGGER.error("Exception occurred while preparing tables for stream " + stream.id().originalName(), e); - return Optional.of(e); + throw new RuntimeException(e); } }, this.executorService); } @@ -183,21 +208,31 @@ public void typeAndDedupe(final String originalNamespace, final String originalN originalName)); } + @Override public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { final var streamConfig = parsedCatalog.getStream(originalNamespace, originalName); return tdLocks.get(streamConfig.id()).readLock(); } + private boolean streamSetupSucceeded(final StreamConfig streamConfig) { + final var originalNamespace = streamConfig.id().originalNamespace(); + final var originalName = streamConfig.id().originalName(); + if (!streamsWithSuccessfulSetup.contains(Pair.of(originalNamespace, originalName))) { + // For example, if T+D setup fails, but the consumer tries to run T+D on all streams during close, + // we should skip it. + LOGGER.warn("Skipping typing and deduping for {}.{} because we could not set up the tables for this stream.", originalNamespace, + originalName); + return false; + } + return true; + } + public CompletableFuture> typeAndDedupeTask(final StreamConfig streamConfig, final boolean mustRun) { return CompletableFuture.supplyAsync(() -> { final var originalNamespace = streamConfig.id().originalNamespace(); final var originalName = streamConfig.id().originalName(); try { - if (!streamsWithSuccessfulSetup.contains(Pair.of(originalNamespace, originalName))) { - // For example, if T+D setup fails, but the consumer tries to run T+D on all streams during close, - // we should skip it. - LOGGER.warn("Skipping typing and deduping for {}.{} because we could not set up the tables for this stream.", originalNamespace, - originalName); + if (!streamSetupSucceeded(streamConfig)) { return Optional.empty(); } @@ -217,8 +252,12 @@ public CompletableFuture> typeAndDedupeTask(final StreamConf final Lock externalLock = tdLocks.get(streamConfig.id()).writeLock(); externalLock.lock(); try { - TypeAndDedupeTransaction.executeTypeAndDedupe(sqlGenerator, destinationHandler, streamConfig, - minExtractedAtByStream.get(streamConfig.id()), + final InitialRawTableState initialRawTableState = initialRawTableStateByStream.get(streamConfig.id()); + TypeAndDedupeTransaction.executeTypeAndDedupe( + sqlGenerator, + destinationHandler, + streamConfig, + initialRawTableState.maxProcessedTimestamp(), getFinalTableSuffix(streamConfig.id())); } finally { LOGGER.info("Allowing other threads to proceed for {}.{}", originalNamespace, originalName); @@ -238,12 +277,36 @@ public CompletableFuture> typeAndDedupeTask(final StreamConf } @Override - public void typeAndDedupe() throws Exception { + public void typeAndDedupe(final Map streamSyncSummaries) throws Exception { LOGGER.info("Typing and deduping all tables"); final Set>> typeAndDedupeTasks = new HashSet<>(); - parsedCatalog.streams().forEach(streamConfig -> { - typeAndDedupeTasks.add(typeAndDedupeTask(streamConfig, true)); - }); + parsedCatalog.streams().stream() + .filter(streamConfig -> { + // Skip if stream setup failed. + if (!streamSetupSucceeded(streamConfig)) { + return false; + } + // Skip if we don't have any records for this stream. + final StreamSyncSummary streamSyncSummary = streamSyncSummaries.getOrDefault( + streamConfig.id().asStreamDescriptor(), + StreamSyncSummary.DEFAULT); + final boolean nonzeroRecords = streamSyncSummary.recordsWritten() + .map(r -> r > 0) + // If we didn't track record counts during the sync, assume we had nonzero records for this stream + .orElse(true); + final boolean unprocessedRecordsPreexist = initialRawTableStateByStream.get(streamConfig.id()).hasUnprocessedRecords(); + // If this sync emitted records, or the previous sync left behind some unprocessed records, + // then the raw table has some unprocessed records right now. + // Run T+D if either of those conditions are true. + final boolean shouldRunTypingDeduping = nonzeroRecords || unprocessedRecordsPreexist; + if (!shouldRunTypingDeduping) { + LOGGER.info( + "Skipping typing and deduping for stream {}.{} because it had no records during this sync and no unprocessed records from a previous sync.", + streamConfig.id().originalNamespace(), + streamConfig.id().originalName()); + } + return shouldRunTypingDeduping; + }).forEach(streamConfig -> typeAndDedupeTasks.add(typeAndDedupeTask(streamConfig, true))); CompletableFuture.allOf(typeAndDedupeTasks.toArray(CompletableFuture[]::new)).join(); reduceExceptions(typeAndDedupeTasks, "The Following Exceptions were thrown while typing and deduping tables:\n"); } @@ -254,6 +317,7 @@ public void typeAndDedupe() throws Exception { * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table * into the final table. */ + @Override public void commitFinalTables() throws Exception { LOGGER.info("Committing final tables"); final Set>> tableCommitTasks = new HashSet<>(); @@ -277,7 +341,7 @@ private CompletableFuture> commitFinalTableTask(final Stream final StreamId streamId = streamConfig.id(); final String finalSuffix = getFinalTableSuffix(streamId); if (!StringUtils.isEmpty(finalSuffix)) { - final String overwriteFinalTable = sqlGenerator.overwriteFinalTable(streamId, finalSuffix); + final Sql overwriteFinalTable = sqlGenerator.overwriteFinalTable(streamId, finalSuffix); LOGGER.info("Overwriting final table with tmp table for stream {}.{}", streamId.originalNamespace(), streamId.originalName()); try { destinationHandler.execute(overwriteFinalTable); diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java index 6608973c850c..f75f0fc9a040 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java @@ -4,23 +4,12 @@ package io.airbyte.integrations.base.destination.typing_deduping; -import java.time.Instant; -import java.util.Optional; +import java.util.List; -public interface DestinationHandler { +public interface DestinationHandler { - Optional findExistingTable(StreamId id) throws Exception; + void execute(final Sql sql) throws Exception; - boolean isFinalTableEmpty(StreamId id) throws Exception; - - /** - * Returns the highest timestamp such that all records with _airbyte_extracted equal to or earlier - * than that timestamp have non-null _airbyte_loaded_at. - *

    - * If the raw table is empty or does not exist, return an empty optional. - */ - Optional getMinTimestampForSync(StreamId id) throws Exception; - - void execute(final String sql) throws Exception; + List gatherInitialState(List streamConfigs) throws Exception; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialState.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialState.java new file mode 100644 index 000000000000..31aa25770790 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialState.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +/** + * Interface representing the initial state of a destination table. + * + */ +public interface DestinationInitialState { + + StreamConfig streamConfig(); + + boolean isFinalTablePresent(); + + InitialRawTableState initialRawTableState(); + + boolean isSchemaMismatch(); + + boolean isFinalTableEmpty(); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialStateImpl.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialStateImpl.java new file mode 100644 index 000000000000..e1fa315c703e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialStateImpl.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public record DestinationInitialStateImpl(StreamConfig streamConfig, + boolean isFinalTablePresent, + InitialRawTableState initialRawTableState, + boolean isSchemaMismatch, + boolean isFinalTableEmpty) + implements DestinationInitialState { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java index 7e28906673a6..5e1e26e804f1 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; -public interface DestinationV1V2Migrator { +public interface DestinationV1V2Migrator { /** * This is the primary entrypoint to this interface @@ -17,8 +17,8 @@ public interface DestinationV1V2Migrator { * @param streamConfig the stream to assess migration needs */ void migrateIfNecessary( - final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final StreamConfig streamConfig) throws TableNotMigratedException, UnexpectedSchemaException, Exception; diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java index 349437e4acec..3319af8297a0 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java @@ -12,16 +12,18 @@ public class FutureUtils { + private static final int DEFAULT_TD_THREAD_COUNT = 8; + /** - * Allow for configuring the number of typing and deduping threads via an enviornment variable in + * Allow for configuring the number of typing and deduping threads via an environment variable in * the destination container. * * @return the number of threads to use in the typing and deduping pool */ - public static int countOfTypingDedupingThreads(final int defaultThreads) { + public static int getCountOfTypeAndDedupeThreads() { return Optional.ofNullable(System.getenv("TD_THREADS")) .map(Integer::valueOf) - .orElse(defaultThreads); + .orElse(DEFAULT_TD_THREAD_COUNT); } /** diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/InitialRawTableState.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/InitialRawTableState.java new file mode 100644 index 000000000000..a037daebfa40 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/InitialRawTableState.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.time.Instant; +import java.util.Optional; + +public record InitialRawTableState(boolean hasUnprocessedRecords, Optional maxProcessedTimestamp) { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java index d9e49257d0a7..f7f5b275768f 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java @@ -4,11 +4,11 @@ package io.airbyte.integrations.base.destination.typing_deduping; -public class NoOpDestinationV1V2Migrator implements DestinationV1V2Migrator { +public class NoOpDestinationV1V2Migrator implements DestinationV1V2Migrator { @Override - public void migrateIfNecessary(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + public void migrateIfNecessary(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final StreamConfig streamConfig) throws TableNotMigratedException, UnexpectedSchemaException { // Do nothing diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt new file mode 100644 index 000000000000..9c26e4d605b8 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt @@ -0,0 +1,22 @@ +package io.airbyte.integrations.base.destination.typing_deduping + +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Condition +import java.util.concurrent.locks.Lock + +class NoOpRawTableTDLock: Lock { + override fun lock() {} + + override fun lockInterruptibly() {} + + override fun tryLock() = true + + override fun tryLock(time: Long, unit: TimeUnit) = tryLock() + + override fun unlock() {} + + override fun newCondition(): Condition { + // Always throw exception to avoid callers from using this path + throw UnsupportedOperationException("This lock implementation does not support retrieving a Condition") + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 4768271abaa1..f76bd2e07019 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -5,19 +5,23 @@ package io.airbyte.integrations.base.destination.typing_deduping; import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; -import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.getCountOfTypeAndDedupeThreads; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; +import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; +import kotlin.NotImplementedError; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.concurrent.BasicThreadFactory; /** @@ -26,99 +30,69 @@ * json->string migrations in the raw tables. */ @Slf4j -public class NoOpTyperDeduperWithV1V2Migrations implements TyperDeduper { +public class NoOpTyperDeduperWithV1V2Migrations implements TyperDeduper { - private final DestinationV1V2Migrator v1V2Migrator; + private final DestinationV1V2Migrator v1V2Migrator; private final V2TableMigrator v2TableMigrator; private final ExecutorService executorService; private final ParsedCatalog parsedCatalog; - private final SqlGenerator sqlGenerator; - private final DestinationHandler destinationHandler; + private final SqlGenerator sqlGenerator; + private final DestinationHandler destinationHandler; - public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final ParsedCatalog parsedCatalog, - final DestinationV1V2Migrator v1V2Migrator, - final V2TableMigrator v2TableMigrator, - final int defaultThreadCount) { + final DestinationV1V2Migrator v1V2Migrator, + final V2TableMigrator v2TableMigrator) { this.sqlGenerator = sqlGenerator; this.destinationHandler = destinationHandler; this.parsedCatalog = parsedCatalog; this.v1V2Migrator = v1V2Migrator; this.v2TableMigrator = v2TableMigrator; - this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), + this.executorService = Executors.newFixedThreadPool(getCountOfTypeAndDedupeThreads(), new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); } @Override public void prepareTables() throws Exception { - log.info("executing NoOp prepareTables with V1V2 migrations"); - final Set>> prepareTablesTasks = new HashSet<>(); - for (final StreamConfig stream : parsedCatalog.streams()) { - prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { - // Migrate the Raw Tables if this is the first v2 sync after a v1 sync - try { - log.info("Migrating V1->V2 for stream {}", stream.id()); - v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); - log.info("Migrating V2 legacy for stream {}", stream.id()); - v2TableMigrator.migrateIfNecessary(stream); - return Optional.empty(); - } catch (Exception e) { - return Optional.of(e); - } - }, executorService)); + try { + log.info("Ensuring schemas exist for prepareTables with V1V2 migrations"); + prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler); + final Set>> prepareTablesTasks = new HashSet<>(); + for (final StreamConfig stream : parsedCatalog.streams()) { + prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + try { + log.info("Migrating V1->V2 for stream {}", stream.id()); + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); + log.info("Migrating V2 legacy for stream {}", stream.id()); + v2TableMigrator.migrateIfNecessary(stream); + return Optional.empty(); + } catch (final Exception e) { + return Optional.of(e); + } + }, executorService)); + } + CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); + } catch (NotImplementedError | NotImplementedException e) { + log.warn( + "Could not prepare schemas or tables because this is not implemented for this destination, this should not be required for this destination to succeed"); } - CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); - reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); } @Override - public void typeAndDedupe(String originalNamespace, String originalName, boolean mustRun) { + public void typeAndDedupe(final String originalNamespace, final String originalName, final boolean mustRun) { log.info("Skipping TypeAndDedupe"); } @Override - public Lock getRawTableInsertLock(String originalNamespace, String originalName) { - return new Lock() { - - @Override - public void lock() { - - } - - @Override - public void lockInterruptibly() { - - } - - @Override - public boolean tryLock() { - // To mimic NoOp behavior always return true that lock is acquired - return true; - } - - @Override - public boolean tryLock(final long time, final TimeUnit unit) { - // To mimic NoOp behavior always return true that lock is acquired - return true; - } - - @Override - public void unlock() { - - } - - @Override - public Condition newCondition() { - // Always throw exception to avoid callers from using this path - throw new UnsupportedOperationException("This lock implementation does not support retrieving a Condition"); - } - - }; + public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { + return new NoOpRawTableTDLock(); } @Override - public void typeAndDedupe() { + public void typeAndDedupe(final Map streamSyncSummaries) { log.info("Skipping TypeAndDedupe final"); } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java index 211bb38f9bb3..af8529e3d2b2 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; @@ -66,7 +69,7 @@ public void commitFinalTables() { } @Override - public void typeAndDedupe() { + public void typeAndDedupe(final Map streamSyncSummaries) { } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java new file mode 100644 index 000000000000..bc0c1940566b --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Represents a list of SQL transactions, where each transaction consists of one or more SQL + * statements. Each transaction MUST NOT contain the BEGIN/COMMIT statements. Each inner list is a + * single transaction, and each String is a single statement within that transaction. + *

    + * Most callers likely only need a single transaction, but e.g. BigQuery disallows running DDL + * inside transactions, and so needs to run sequential "CREATE SCHEMA", "CREATE TABLE" as separate + * transactions. + *

    + * Callers are encouraged to use the static factory methods instead of the public constructor. + */ +public record Sql(List> transactions) { + + public Sql { + transactions.forEach(transaction -> { + if (transaction.isEmpty()) { + throw new IllegalArgumentException("Transaction must not be empty"); + } + if (transaction.stream().anyMatch(s -> s == null || s.isEmpty())) { + throw new IllegalArgumentException("Transaction must not contain empty statements"); + } + }); + } + + /** + * @param begin The SQL statement to start a transaction, typically "BEGIN" + * @param commit The SQL statement to commit a transaction, typically "COMMIT" + * @return A list of SQL strings, each of which represents a transaction. + */ + public List asSqlStrings(final String begin, final String commit) { + return transactions().stream() + .map(transaction -> { + // If there's only one statement, we don't need to wrap it in a transaction. + if (transaction.size() == 1) { + return transaction.get(0); + } + final StringBuilder builder = new StringBuilder(); + builder.append(begin); + builder.append(";\n"); + transaction.forEach(statement -> { + builder.append(statement); + // No semicolon - statements already end with a semicolon + builder.append("\n"); + }); + builder.append(commit); + builder.append(";\n"); + return builder.toString(); + }).toList(); + } + + /** + * Execute a list of SQL statements in a single transaction. + */ + public static Sql transactionally(final List statements) { + return create(List.of(statements)); + } + + public static Sql transactionally(final String... statements) { + return transactionally(Stream.of(statements).toList()); + } + + /** + * Execute each statement as its own transaction. + */ + public static Sql separately(final List statements) { + return create(statements.stream().map(Collections::singletonList).toList()); + } + + public static Sql separately(final String... statements) { + return separately(Stream.of(statements).toList()); + } + + /** + * Convenience method for indicating intent. Equivalent to calling + * {@link #transactionally(String...)} or {@link #separately(String...)} with the same string. + */ + public static Sql of(final String statement) { + return transactionally(statement); + } + + public static Sql concat(final Sql... sqls) { + return create(Stream.of(sqls).flatMap(sql -> sql.transactions.stream()).toList()); + } + + public static Sql concat(final List sqls) { + return create(sqls.stream().flatMap(sql -> sql.transactions.stream()).toList()); + } + + /** + * Utility method to create a Sql object without empty statements/transactions, and appending + * semicolons when needed. + */ + public static Sql create(final List> transactions) { + return new Sql(transactions.stream() + .map(transaction -> transaction.stream() + .filter(statement -> statement != null && !statement.isEmpty()) + .map(statement -> { + if (!statement.trim().endsWith(";")) { + return statement + ";"; + } + return statement; + }) + .toList()) + .filter(transaction -> !transaction.isEmpty()) + .toList()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java index ebc46df02849..bb12237ebbf9 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java @@ -9,7 +9,7 @@ import java.time.Instant; import java.util.Optional; -public interface SqlGenerator { +public interface SqlGenerator { StreamId buildStreamId(String namespace, String name, String rawNamespaceOverride); @@ -23,9 +23,7 @@ default ColumnId buildColumnId(final String name) { * Generate a SQL statement to create a fresh table to match the given stream. *

    * The generated SQL should throw an exception if the table already exists and {@code force} is - * false. Callers should use - * {@link #existingSchemaMatchesStreamConfig(StreamConfig, java.lang.Object)} if the table is known - * to exist, and potentially softReset + * false. * * @param suffix A suffix to add to the stream name. Useful for full refresh overwrite syncs, where * we write the entire sync to a temp table. @@ -33,16 +31,15 @@ default ColumnId buildColumnId(final String name) { * table already exists. If you're passing a non-empty prefix, you likely want to set this to * true. */ - String createTable(final StreamConfig stream, final String suffix, boolean force); + Sql createTable(final StreamConfig stream, final String suffix, boolean force); /** - * Check the final table's schema and compare it to what the stream config would generate. + * Used to create either the airbyte_internal or final schemas if they don't exist * - * @param stream the stream/stable in question - * @param existingTable the existing table mapped to the stream - * @return whether the existing table matches the expected schema + * @param schema the schema to create + * @return SQL to create the schema if it does not exist */ - boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final DialectTableDefinition existingTable); + Sql createSchema(final String schema); /** * Generate a SQL statement to copy new data from the raw table into the final table. @@ -72,7 +69,7 @@ default ColumnId buildColumnId(final String name) { * however sometimes we get badly typed data. In these cases we can use a more expensive * query which handles casting exceptions. */ - String updateTable(final StreamConfig stream, String finalSuffix, Optional minRawTimestamp, final boolean useExpensiveSaferCasting); + Sql updateTable(final StreamConfig stream, String finalSuffix, Optional minRawTimestamp, final boolean useExpensiveSaferCasting); /** * Drop the previous final table, and rename the new final table to match the old final table. @@ -80,31 +77,31 @@ default ColumnId buildColumnId(final String name) { * This method may assume that the stream is an OVERWRITE stream, and that the final suffix is * non-empty. Callers are responsible for verifying those are true. */ - String overwriteFinalTable(StreamId stream, String finalSuffix); + Sql overwriteFinalTable(StreamId stream, String finalSuffix); /** * Creates a sql query which will create a v2 raw table from the v1 raw table, then performs a soft * reset. * * @param streamId the stream to migrate - * @param namespace - * @param tableName + * @param namespace the namespace of the v1 raw table + * @param tableName name of the v2 raw table * @return a string containing the necessary sql to migrate */ - String migrateFromV1toV2(StreamId streamId, String namespace, String tableName); + Sql migrateFromV1toV2(StreamId streamId, String namespace, String tableName); /** * Typically we need to create a soft reset temporary table and clear loaded at values * * @return */ - default String prepareTablesForSoftReset(final StreamConfig stream) { - final String createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); - final String clearLoadedAt = clearLoadedAt(stream.id()); - return String.join("\n", createTempTable, clearLoadedAt); + default Sql prepareTablesForSoftReset(final StreamConfig stream) { + final Sql createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); + final Sql clearLoadedAt = clearLoadedAt(stream.id()); + return Sql.concat(createTempTable, clearLoadedAt); } - String clearLoadedAt(final StreamId streamId); + Sql clearLoadedAt(final StreamId streamId); /** * Implementation specific if there is no option to retry again with safe casted SQL or the specific diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java index 9851ee7b7e59..e65cfa72259c 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.StreamDescriptor; /** * In general, callers should not directly instantiate this class. Use @@ -56,6 +57,10 @@ public AirbyteStreamNameNamespacePair asPair() { return new AirbyteStreamNameNamespacePair(originalName, originalNamespace); } + public StreamDescriptor asStreamDescriptor() { + return new StreamDescriptor().withNamespace(originalNamespace).withName(originalName); + } + /** * Build the raw table name as namespace + (delimiter) + name. For example, given a stream with * namespace "public__ab" and name "abab_users", we will end up with raw table name diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java index 2327421b419d..d638125371cd 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java @@ -4,10 +4,13 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A slightly more complicated way to keep track of when to perform type and dedupe operations per @@ -15,6 +18,8 @@ */ public class TypeAndDedupeOperationValve extends ConcurrentHashMap { + private static final Logger LOGGER = LoggerFactory.getLogger(TypeAndDedupeOperationValve.class); + private static final long NEGATIVE_MILLIS = -1; private static final long SIX_HOURS_MILLIS = 1000 * 60 * 60 * 6; @@ -79,6 +84,10 @@ public void addStreamIfAbsent(final AirbyteStreamNameNamespacePair key) { * deduping. */ public boolean readyToTypeAndDedupe(final AirbyteStreamNameNamespacePair key) { + if (!DestinationConfig.getInstance().getBooleanValue("enable_incremental_final_table_updates")) { + LOGGER.info("Skipping Incremental Typing and Deduping"); + return false; + } if (!containsKey(key)) { return false; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java index 116fa6a7251b..f350c83e76ca 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java @@ -26,27 +26,28 @@ public class TypeAndDedupeTransaction { * @param suffix table suffix for temporary tables * @throws Exception if the safe query fails */ - public static void executeTypeAndDedupe(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, - final StreamConfig streamConfig, - final Optional minExtractedAt, - final String suffix) + public static void executeTypeAndDedupe(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig, + final Optional minExtractedAt, + final String suffix) throws Exception { try { - LOGGER.info("Attempting typing and deduping for {}.{} with suffix", streamConfig.id().originalNamespace(), streamConfig.id().originalName(), + LOGGER.info("Attempting typing and deduping for {}.{} with suffix {}", streamConfig.id().originalNamespace(), streamConfig.id().originalName(), suffix); - final String unsafeSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, false); + final Sql unsafeSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, false); destinationHandler.execute(unsafeSql); } catch (final Exception e) { if (sqlGenerator.shouldRetry(e)) { // TODO Destination specific non-retryable exceptions should be added. LOGGER.error("Encountered Exception on unsafe SQL for stream {} {} with suffix {}, attempting with error handling", streamConfig.id().originalNamespace(), streamConfig.id().originalName(), suffix, e); - final String saferSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, true); + final Sql saferSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, true); destinationHandler.execute(saferSql); } else { LOGGER.error("Encountered Exception on unsafe SQL for stream {} {} with suffix {}, Retry is skipped", streamConfig.id().originalNamespace(), streamConfig.id().originalName(), suffix, e); + throw e; } } } @@ -61,7 +62,9 @@ public static void executeTypeAndDedupe(final SqlGenerator * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table * into the final table. + * + * @param streamSyncSummaries Information about what happened during the sync. Implementations + * SHOULD use this information to skip T+D when possible (this is not a requirement for + * correctness, but does allow us to save time/money). This parameter MUST NOT be null. + * Streams MAY be omitted, which will be treated as though they were mapped to + * {@link StreamSyncSummary#DEFAULT}. */ - void typeAndDedupe() throws Exception; + void typeAndDedupe(Map streamSyncSummaries) throws Exception; void commitFinalTables() throws Exception; diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt new file mode 100644 index 000000000000..8f56b1a81acb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt @@ -0,0 +1,15 @@ +package io.airbyte.integrations.base.destination.typing_deduping + + +/** + * Extracts all the "raw" and "final" schemas identified in the [parsedCatalog] and ensures they + * exist in the Destination Database. + */ +fun prepareAllSchemas(parsedCatalog: ParsedCatalog, sqlGenerator: SqlGenerator, destinationHandler: DestinationHandler) { + val rawSchema = parsedCatalog.streams.mapNotNull { it.id.rawNamespace } + val finalSchema = parsedCatalog.streams.mapNotNull { it.id.finalNamespace } + val createAllSchemasSql = rawSchema.union(finalSchema) + .map { sqlGenerator.createSchema(it) } + .toList() + destinationHandler.execute(Sql.concat(createAllSchemasSql)) +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java index 3922f8ebe4bf..b0237657058b 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java @@ -19,7 +19,7 @@ class CatalogParserTest { - private SqlGenerator sqlGenerator; + private SqlGenerator sqlGenerator; private CatalogParser parser; @BeforeEach diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java index 260781126f30..916c0235722d 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java @@ -4,8 +4,10 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.separately; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.ignoreStubs; @@ -17,9 +19,12 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -27,42 +32,58 @@ public class DefaultTyperDeduperTest { private MockSqlGenerator sqlGenerator; - private DestinationHandler destinationHandler; + private DestinationHandler destinationHandler; - private DestinationV1V2Migrator migrator; + private List initialStates; + + private DestinationV1V2Migrator migrator; private TyperDeduper typerDeduper; @BeforeEach void setup() throws Exception { sqlGenerator = spy(new MockSqlGenerator()); destinationHandler = mock(DestinationHandler.class); - when(destinationHandler.getMinTimestampForSync(any())).thenReturn(Optional.empty()); - migrator = new NoOpDestinationV1V2Migrator<>(); + DestinationInitialState overwriteNsState = mock(DestinationInitialState.class); + DestinationInitialState appendNsState = mock(DestinationInitialState.class); + DestinationInitialState dedupeNsState = mock(DestinationInitialState.class); + StreamConfig overwriteStreamConfig = new StreamConfig( + new StreamId("overwrite_ns", "overwrite_stream", null, null, "overwrite_ns", "overwrite_stream"), + null, + DestinationSyncMode.OVERWRITE, + null, + null, + null); + StreamConfig appendStreamConfig = new StreamConfig( + new StreamId("append_ns", "append_stream", null, null, "append_ns", "append_stream"), + null, + DestinationSyncMode.APPEND, + null, + null, + null); + StreamConfig dedupeStreamConfig = new StreamConfig( + new StreamId("dedup_ns", "dedup_stream", null, null, "dedup_ns", "dedup_stream"), + null, + DestinationSyncMode.APPEND_DEDUP, + null, + null, + null); + when(overwriteNsState.streamConfig()).thenReturn(overwriteStreamConfig); + when(appendNsState.streamConfig()).thenReturn(appendStreamConfig); + when(dedupeNsState.streamConfig()).thenReturn(dedupeStreamConfig); + + initialStates = List.of(overwriteNsState, appendNsState, dedupeNsState); + when(destinationHandler.gatherInitialState(anyList())) + .thenReturn(initialStates); + initialStates.forEach(initialState -> when(initialState.initialRawTableState()).thenReturn(new InitialRawTableState(true, Optional.empty()))); + + migrator = new NoOpDestinationV1V2Migrator(); final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of( - new StreamConfig( - new StreamId("overwrite_ns", "overwrite_stream", null, null, "overwrite_ns", "overwrite_stream"), - null, - DestinationSyncMode.OVERWRITE, - null, - null, - null), - new StreamConfig( - new StreamId("append_ns", "append_stream", null, null, "append_ns", "append_stream"), - null, - DestinationSyncMode.APPEND, - null, - null, - null), - new StreamConfig( - new StreamId("dedup_ns", "dedup_stream", null, null, "dedup_ns", "dedup_stream"), - null, - DestinationSyncMode.APPEND_DEDUP, - null, - null, - null))); - - typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, 1); + overwriteStreamConfig, + appendStreamConfig, + dedupeStreamConfig)); + + typerDeduper = new DefaultTyperDeduper(sqlGenerator, destinationHandler, parsedCatalog, migrator); } /** @@ -70,21 +91,23 @@ void setup() throws Exception { */ @Test void emptyDestination() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.empty()); + initialStates.forEach(initialState -> when(initialState.isFinalTablePresent()).thenReturn(false)); + // when(destinationHandler.findExistingTable(any())).thenReturn(Optional.empty()); typerDeduper.prepareTables(); - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream"); - verify(destinationHandler).execute("CREATE TABLE append_ns.append_stream"); - verify(destinationHandler).execute("CREATE TABLE dedup_ns.dedup_stream"); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE append_ns.append_stream")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE dedup_ns.dedup_stream")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); - verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream WITHOUT SAFER CASTING")); typerDeduper.typeAndDedupe("append_ns", "append_stream", false); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); @@ -98,31 +121,34 @@ void emptyDestination() throws Exception { */ @Test void existingEmptyTable() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(false); + initialStates.forEach(initialState -> { + when(initialState.isFinalTablePresent()).thenReturn(true); + when(initialState.isFinalTableEmpty()).thenReturn(true); + when(initialState.isSchemaMismatch()).thenReturn(true); + }); typerDeduper.prepareTables(); - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - verify(destinationHandler).execute("PREPARE append_ns.append_stream FOR SOFT RESET"); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING"); - verify(destinationHandler).execute("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset"); - verify(destinationHandler).execute("PREPARE dedup_ns.dedup_stream FOR SOFT RESET"); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING"); - verify(destinationHandler).execute("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset"); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); + verify(destinationHandler).execute(Sql.of("PREPARE append_ns.append_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset")); + verify(destinationHandler).execute(Sql.of("PREPARE dedup_ns.dedup_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); - verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING")); typerDeduper.typeAndDedupe("append_ns", "append_stream", false); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); typerDeduper.commitFinalTables(); - verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); } @@ -132,11 +158,15 @@ void existingEmptyTable() throws Exception { */ @Test void existingEmptyTableMatchingSchema() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + initialStates.forEach(initialState -> { + when(initialState.isFinalTablePresent()).thenReturn(true); + when(initialState.isFinalTableEmpty()).thenReturn(true); + when(initialState.isSchemaMismatch()).thenReturn(true); + }); typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + clearInvocations(destinationHandler); verify(destinationHandler, never()).execute(any()); } @@ -146,36 +176,42 @@ void existingEmptyTableMatchingSchema() throws Exception { */ @Test void existingNonemptyTable() throws Exception { - when(destinationHandler.getMinTimestampForSync(any())).thenReturn(Optional.of(Instant.parse("2023-01-01T12:34:56Z"))); - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); + initialStates.forEach(initialState -> { + when(initialState.isFinalTablePresent()).thenReturn(true); + when(initialState.isFinalTableEmpty()).thenReturn(false); + when(initialState.isSchemaMismatch()).thenReturn(true); + when(initialState.initialRawTableState()).thenReturn(new InitialRawTableState(true, Optional.of(Instant.parse("2023-01-01T12:34:56Z")))); + }); typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); // NB: We only create a tmp table for the overwrite stream, and do _not_ soft reset the existing // overwrite stream's table. - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - verify(destinationHandler).execute("PREPARE append_ns.append_stream FOR SOFT RESET"); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING"); - verify(destinationHandler).execute("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset"); - verify(destinationHandler).execute("PREPARE dedup_ns.dedup_stream FOR SOFT RESET"); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING"); - verify(destinationHandler).execute("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset"); + + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); + verify(destinationHandler).execute(Sql.of("PREPARE append_ns.append_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset")); + verify(destinationHandler).execute(Sql.of("PREPARE dedup_ns.dedup_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); // NB: no airbyte_tmp suffix on the non-overwrite streams verify(destinationHandler) - .execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z"); + .execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); typerDeduper.typeAndDedupe("append_ns", "append_stream", false); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z"); + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z"); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); clearInvocations(destinationHandler); typerDeduper.commitFinalTables(); - verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); } @@ -185,15 +221,18 @@ void existingNonemptyTable() throws Exception { */ @Test void existingNonemptyTableMatchingSchema() throws Exception { - when(destinationHandler.getMinTimestampForSync(any())).thenReturn(Optional.of(Instant.now())); - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + initialStates.forEach(initialState -> { + when(initialState.isFinalTablePresent()).thenReturn(true); + when(initialState.isFinalTableEmpty()).thenReturn(false); + when(initialState.isSchemaMismatch()).thenReturn(false); + when(initialState.initialRawTableState()).thenReturn(new InitialRawTableState(true, Optional.of(Instant.now()))); + }); typerDeduper.prepareTables(); // NB: We only create one tmp table here. // Also, we need to alter the existing _real_ table, not the tmp table! - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); verifyNoMoreInteractions(ignoreStubs(destinationHandler)); } @@ -217,4 +256,48 @@ void failedSetup() throws Exception { verifyNoInteractions(ignoreStubs(destinationHandler)); } + /** + * Test a typical sync, where the previous sync left no unprocessed raw records. If this sync writes + * some records for a stream, we should run T+D for that stream. + */ + @Test + void noUnprocessedRecords() throws Exception { + initialStates.forEach(initialState -> when(initialState.initialRawTableState()).thenReturn(new InitialRawTableState(false, Optional.empty()))); + typerDeduper.prepareTables(); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe(Map.of( + new StreamDescriptor().withName("overwrite_stream").withNamespace("overwrite_ns"), new StreamSyncSummary(Optional.of(0L)), + new StreamDescriptor().withName("append_stream").withNamespace("append_ns"), new StreamSyncSummary(Optional.of(1L)))); + + // append_stream and dedup_stream should be T+D-ed. overwrite_stream has explicitly 0 records, but + // dedup_stream + // is missing from the map, so implicitly has nonzero records. + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); + verifyNoMoreInteractions(destinationHandler); + } + + /** + * Test a sync where the previous sync failed to run T+D for some stream. Even if this sync writes + * zero records, it should still run T+D. + */ + @Test + void unprocessedRecords() throws Exception { + initialStates.forEach(initialState -> when(initialState.initialRawTableState()) + .thenReturn(new InitialRawTableState(true, Optional.of(Instant.parse("2023-01-23T12:34:56Z"))))); + typerDeduper.prepareTables(); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe(Map.of( + new StreamDescriptor().withName("overwrite_stream").withNamespace("overwrite_ns"), new StreamSyncSummary(Optional.of(0L)), + new StreamDescriptor().withName("append_stream").withNamespace("append_ns"), new StreamSyncSummary(Optional.of(1L)))); + + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java index e035c595963b..0e4c80321055 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java @@ -6,6 +6,7 @@ import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; import static io.airbyte.cdk.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; +import static org.mockito.ArgumentMatchers.any; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.Optional; @@ -73,14 +74,14 @@ public void testMismatchedSchemaThrowsException() throws Exception { public void testMigrate() throws Exception { final var sqlGenerator = new MockSqlGenerator(); final StreamConfig stream = new StreamConfig(STREAM_ID, null, DestinationSyncMode.APPEND_DEDUP, null, null, null); - final DestinationHandler handler = Mockito.mock(DestinationHandler.class); + final DestinationHandler handler = Mockito.mock(DestinationHandler.class); final var sql = sqlGenerator.migrateFromV1toV2(STREAM_ID, "v1_raw_namespace", "v1_raw_table"); // All is well final var migrator = noIssuesMigrator(); migrator.migrate(sqlGenerator, handler, stream); Mockito.verify(handler).execute(sql); // Exception thrown when executing sql, TableNotMigratedException thrown - Mockito.doThrow(Exception.class).when(handler).execute(Mockito.anyString()); + Mockito.doThrow(Exception.class).when(handler).execute(any()); final TableNotMigratedException exception = Assertions.assertThrows(TableNotMigratedException.class, () -> migrator.migrate(sqlGenerator, handler, stream)); Assertions.assertEquals("Attempted and failed to migrate stream final_table", exception.getMessage()); @@ -93,12 +94,12 @@ public static BaseDestinationV1V2Migrator makeMockMigrator(final boolean v2Names final boolean v1RawTableSchemaMatches) throws Exception { final BaseDestinationV1V2Migrator migrator = Mockito.spy(BaseDestinationV1V2Migrator.class); - Mockito.when(migrator.doesAirbyteInternalNamespaceExist(Mockito.any())).thenReturn(v2NamespaceExists); + Mockito.when(migrator.doesAirbyteInternalNamespaceExist(any())).thenReturn(v2NamespaceExists); final var existingTable = v2TableExists ? Optional.of("v2_raw") : Optional.empty(); Mockito.when(migrator.getTableIfExists("raw", "raw_table")).thenReturn(existingTable); Mockito.when(migrator.schemaMatchesExpectation("v2_raw", V2_RAW_TABLE_COLUMN_NAMES)).thenReturn(v2RawSchemaMatches); - Mockito.when(migrator.convertToV1RawName(Mockito.any())).thenReturn(new NamespacedTableName("v1_raw_namespace", "v1_raw_table")); + Mockito.when(migrator.convertToV1RawName(any())).thenReturn(new NamespacedTableName("v1_raw_namespace", "v1_raw_table")); final var existingV1RawTable = v1RawTableExists ? Optional.of("v1_raw") : Optional.empty(); Mockito.when(migrator.getTableIfExists("v1_raw_namespace", "v1_raw_table")).thenReturn(existingV1RawTable); Mockito.when(migrator.schemaMatchesExpectation("v1_raw", LEGACY_RAW_TABLE_COLUMNS)).thenReturn(v1RawTableSchemaMatches); diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java index e08b27fa83f0..3ef59aa91e21 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java @@ -10,7 +10,7 @@ /** * Basic SqlGenerator mock. See {@link DefaultTyperDeduperTest} for example usage. */ -class MockSqlGenerator implements SqlGenerator { +class MockSqlGenerator implements SqlGenerator { @Override public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { @@ -23,44 +23,44 @@ public ColumnId buildColumnId(final String name, final String suffix) { } @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { - return "CREATE TABLE " + stream.id().finalTableId("", suffix); + public Sql createSchema(final String schema) { + return Sql.of("CREATE SCHEMA " + schema); } @Override - public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final String existingTable) throws TableNotMigratedException { - return false; + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + return Sql.of("CREATE TABLE " + stream.id().finalTableId("", suffix)); } @Override - public String updateTable(final StreamConfig stream, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { final String timestampFilter = minRawTimestamp .map(timestamp -> " WHERE extracted_at > " + timestamp) .orElse(""); final String casting = useExpensiveSaferCasting ? " WITH" : " WITHOUT" + " SAFER CASTING"; - return "UPDATE TABLE " + stream.id().finalTableId("", finalSuffix) + casting + timestampFilter; + return Sql.of("UPDATE TABLE " + stream.id().finalTableId("", finalSuffix) + casting + timestampFilter); } @Override - public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return "OVERWRITE TABLE " + stream.finalTableId("") + " FROM " + stream.finalTableId("", finalSuffix); + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return Sql.of("OVERWRITE TABLE " + stream.finalTableId("") + " FROM " + stream.finalTableId("", finalSuffix)); } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - return "MIGRATE TABLE " + String.join(".", namespace, tableName) + " TO " + streamId.rawTableId(""); + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return Sql.of("MIGRATE TABLE " + String.join(".", namespace, tableName) + " TO " + streamId.rawTableId("")); } @Override - public String prepareTablesForSoftReset(final StreamConfig stream) { - return "PREPARE " + String.join(".", stream.id().originalNamespace(), stream.id().originalName()) + " FOR SOFT RESET"; + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + return Sql.of("PREPARE " + String.join(".", stream.id().originalNamespace(), stream.id().originalName()) + " FOR SOFT RESET"); } @Override - public String clearLoadedAt(final StreamId streamId) { + public Sql clearLoadedAt(final StreamId streamId) { return null; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java index 49d4266dec7a..3ada28f544d4 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java @@ -4,13 +4,18 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.IntStream; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class TypeAndDedupeOperationValveTest { @@ -26,23 +31,39 @@ public void setup() { minuteUpdates = () -> start.getAndUpdate(l -> l + (60 * 1000)); } + @AfterEach + public void clearDestinationConfig() { + DestinationConfig.clearInstance(); + } + + private void initializeDestinationConfigOption(final boolean enableIncrementalTypingAndDeduping) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("enable_incremental_final_table_updates", enableIncrementalTypingAndDeduping); + DestinationConfig.initialize(objectNode); + } + private void elapseTime(Supplier timing, int iterations) { IntStream.range(0, iterations).forEach(__ -> { timing.get(); }); } - @Test - public void testAddStream() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testAddStream(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); valve.addStream(STREAM_A); Assertions.assertEquals(-1, valve.getIncrementInterval(STREAM_A)); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); Assertions.assertEquals(valve.get(STREAM_A), 0l); } - @Test - public void testReadyToTypeAndDedupe() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReadyToTypeAndDedupe(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); final var valve = new TypeAndDedupeOperationValve(minuteUpdates); // method call increments time valve.addStream(STREAM_A); @@ -50,9 +71,9 @@ public void testReadyToTypeAndDedupe() { // method call increments time valve.addStream(STREAM_B); // method call increments time - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); elapseTime(minuteUpdates, 1); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_B)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_B), enableIncrementalTypingAndDeduping); valve.updateTimeAndIncreaseInterval(STREAM_A); Assertions.assertEquals(1000 * 60 * 60 * 6, valve.getIncrementInterval(STREAM_A)); @@ -60,18 +81,22 @@ public void testReadyToTypeAndDedupe() { Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A)); // More than enough time has passed now elapseTime(minuteUpdates, 60 * 6); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); } - @Test - public void testUpdateTimeAndIncreaseInterval() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testUpdateTimeAndIncreaseInterval(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); final var valve = new TypeAndDedupeOperationValve(minuteUpdates); valve.addStream(STREAM_A); - IntStream.range(0, 1).forEach(__ -> Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A))); // start ready to T&D - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + IntStream.range(0, 1).forEach(__ -> Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping)); // start + // ready + // to T&D + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); valve.updateTimeAndIncreaseInterval(STREAM_A); IntStream.range(0, 360).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java index 48e7d3cc6c83..93e62670a99d 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java @@ -12,7 +12,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Streams; @@ -51,7 +50,7 @@ * {@link #getDestinationHandler()} in a {@link org.junit.jupiter.api.BeforeEach} method. */ @Execution(ExecutionMode.CONCURRENT) -public abstract class BaseSqlGeneratorIntegrationTest { +public abstract class BaseSqlGeneratorIntegrationTest { private static final Logger LOGGER = LoggerFactory.getLogger(BaseSqlGeneratorIntegrationTest.class); /** @@ -104,8 +103,8 @@ public abstract class BaseSqlGeneratorIntegrationTest { */ protected StreamConfig cdcIncrementalAppendStream; - protected SqlGenerator generator; - protected DestinationHandler destinationHandler; + protected SqlGenerator generator; + protected DestinationHandler destinationHandler; protected String namespace; protected StreamId streamId; @@ -113,9 +112,9 @@ public abstract class BaseSqlGeneratorIntegrationTest { private ColumnId cursor; private LinkedHashMap COLUMNS; - protected abstract SqlGenerator getSqlGenerator(); + protected abstract SqlGenerator getSqlGenerator(); - protected abstract DestinationHandler getDestinationHandler(); + protected abstract DestinationHandler getDestinationHandler(); /** * Subclasses should override this method if they need to make changes to the stream ID. For @@ -194,6 +193,7 @@ protected Map getFinalMetadataColumnNames() { public void setup() throws Exception { generator = getSqlGenerator(); destinationHandler = getDestinationHandler(); + final ColumnId id1 = generator.buildColumnId("id1"); final ColumnId id2 = generator.buildColumnId("id2"); primaryKey = List.of(id1, id2); @@ -226,7 +226,7 @@ public void setup() throws Exception { Pair.of(id2, AirbyteProtocolType.INTEGER), Pair.of(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE)); - namespace = Strings.addRandomSuffix("sql_generator_test", "_", 5); + namespace = Strings.addRandomSuffix("sql_generator_test", "_", 10); // This is not a typical stream ID would look like, but SqlGenerator isn't allowed to make any // assumptions about StreamId structure. // In practice, the final table would be testDataset.users, and the raw table would be @@ -272,21 +272,24 @@ public void teardown() throws Exception { teardownNamespace(namespace); } + private DestinationInitialState getDestinationInitialState(StreamConfig streamConfig) throws Exception { + final List initialState = + destinationHandler.gatherInitialState(List.of(streamConfig)); + assertEquals(1, initialState.size(), "gatherInitialState returned the wrong number of futures"); + assertTrue(initialState.getFirst().isFinalTablePresent(), "Destination handler could not find existing table"); + return initialState.getFirst(); + } + /** * Create a table and verify that we correctly recognize it as identical to itself. */ @Test public void detectNoSchemaChange() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - - assertTrue( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + final DestinationInitialState destinationInitialState = getDestinationInitialState(incrementalDedupStream); + assertFalse( + destinationInitialState.isSchemaMismatch(), "Unchanged schema was incorrectly detected as a schema change."); } @@ -295,20 +298,14 @@ public void detectNoSchemaChange() throws Exception { */ @Test public void detectColumnAdded() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - incrementalDedupStream.columns().put( generator.buildColumnId("new_column"), AirbyteProtocolType.STRING); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + final DestinationInitialState destinationInitialState = getDestinationInitialState(incrementalDedupStream); + assertTrue( + destinationInitialState.isSchemaMismatch(), "Adding a new column was not detected as a schema change."); } @@ -317,18 +314,12 @@ public void detectColumnAdded() throws Exception { */ @Test public void detectColumnRemoved() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - incrementalDedupStream.columns().remove(generator.buildColumnId("string")); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + final DestinationInitialState destinationInitialState = getDestinationInitialState(incrementalDedupStream); + assertTrue( + destinationInitialState.isSchemaMismatch(), "Removing a column was not detected as a schema change."); } @@ -337,20 +328,14 @@ public void detectColumnRemoved() throws Exception { */ @Test public void detectColumnChanged() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - incrementalDedupStream.columns().put( generator.buildColumnId("string"), AirbyteProtocolType.INTEGER); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + final DestinationInitialState destinationInitialState = getDestinationInitialState(incrementalDedupStream); + assertTrue( + destinationInitialState.isSchemaMismatch(), "Altering a column was not detected as a schema change."); } @@ -388,6 +373,11 @@ public void incrementalDedupSameNameNamespace() throws Exception { verifyRecordCounts(1, rawRecords, 1, finalRecords); } + private DestinationInitialState getOnly(final List initialStates) { + assertEquals(1, initialStates.size()); + return initialStates.getFirst(); + } + /** * Run a full T+D update for an incremental-dedup stream, writing to a final table with "_foo" * suffix, with values for all data types. Verifies all behaviors for all types: @@ -403,13 +393,18 @@ public void incrementalDedupSameNameNamespace() throws Exception { */ @Test public void allTypes() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); createRawTable(streamId); createFinalTable(incrementalDedupStream, ""); insertRawTableRecords( streamId, BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); - assertTrue(destinationHandler.isFinalTableEmpty(streamId), "Final table should be empty before T+D"); + DestinationInitialState initialState = getOnly(destinationHandler.gatherInitialState(List.of(incrementalDedupStream))); + assertTrue(initialState.isFinalTableEmpty(), "Final table should be empty before T+D"); TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, Optional.empty(), ""); @@ -418,7 +413,37 @@ public void allTypes() throws Exception { dumpRawTableRecords(streamId), "sqlgenerator/alltypes_expectedrecords_final.jsonl", dumpFinalTableRecords(streamId, "")); - assertFalse(destinationHandler.isFinalTableEmpty(streamId), "Final table should not be empty after T+D"); + initialState = getOnly(destinationHandler.gatherInitialState(List.of(incrementalDedupStream))); + assertFalse(initialState.isFinalTableEmpty(), "Final table should not be empty after T+D"); + } + + /** + * Run a basic test to verify that we don't throw an exception on basic data values. + */ + @Test + public void allTypesUnsafe() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_unsafe_inputrecords.jsonl")); + + DestinationInitialState initialState = getOnly(destinationHandler.gatherInitialState(List.of(incrementalDedupStream))); + assertTrue(initialState.isFinalTableEmpty(), "Final table should be empty before T+D"); + + // Instead of using the full T+D transaction, explicitly run with useSafeCasting=false. + final Sql unsafeSql = generator.updateTable(incrementalDedupStream, "", Optional.empty(), false); + destinationHandler.execute(unsafeSql); + + initialState = getOnly(destinationHandler.gatherInitialState(List.of(incrementalDedupStream))); + assertFalse(initialState.isFinalTableEmpty(), "Final table should not be empty after T+D"); + } + + private InitialRawTableState getInitialRawTableState(StreamConfig streamConfig) throws Exception { + List initialStates = + destinationHandler.gatherInitialState(List.of(streamConfig)); + assertEquals(1, initialStates.size()); + return initialStates.getFirst().initialRawTableState(); } /** @@ -427,12 +452,12 @@ public void allTypes() throws Exception { */ @Test public void minTimestampBehavesCorrectly() throws Exception { - // When the raw table doesn't exist, there is no timestamp - assertEquals(Optional.empty(), destinationHandler.getMinTimestampForSync(streamId)); + // When the raw table doesn't exist, there are no unprocessed records and no timestamp + assertEquals(new InitialRawTableState(false, Optional.empty()), getInitialRawTableState(incrementalAppendStream)); - // When the raw table is empty, there is no timestamp + // When the raw table is empty, there are still no unprocessed records and no timestamp createRawTable(streamId); - assertEquals(Optional.empty(), destinationHandler.getMinTimestampForSync(streamId)); + assertEquals(new InitialRawTableState(false, Optional.empty()), getInitialRawTableState(incrementalAppendStream)); // If we insert some raw records with null loaded_at, we should get the min extracted_at insertRawTableRecords( @@ -454,20 +479,22 @@ public void minTimestampBehavesCorrectly() throws Exception { "_airbyte_data": {} } """))); - Instant actualTimestamp = destinationHandler.getMinTimestampForSync(streamId).get(); + InitialRawTableState tableState = getInitialRawTableState(incrementalAppendStream); + assertTrue(tableState.hasUnprocessedRecords(), + "When all raw records have null loaded_at, we should recognize that there are unprocessed records"); assertTrue( - actualTimestamp.isBefore(Instant.parse("2023-01-01T00:00:00Z")), + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T00:00:00Z")), "When all raw records have null loaded_at, the min timestamp should be earlier than all of their extracted_at values (2023-01-01). Was actually " - + actualTimestamp); + + tableState.maxProcessedTimestamp().get()); // Execute T+D to set loaded_at on the records createFinalTable(incrementalAppendStream, ""); TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalAppendStream, Optional.empty(), ""); assertEquals( - Instant.parse("2023-01-02T00:00:00Z"), - destinationHandler.getMinTimestampForSync(streamId).get(), - "When all raw records have non-null loaded_at, the min timestamp should be equal to the latest extracted_at"); + getInitialRawTableState(incrementalAppendStream), + new InitialRawTableState(false, Optional.of(Instant.parse("2023-01-02T00:00:00Z"))), + "When all raw records have non-null loaded_at, we should recognize that there are no unprocessed records, and the min timestamp should be equal to the latest extracted_at"); // If we insert another raw record with older extracted_at than the typed records, we should fetch a // timestamp earlier than this new record. @@ -484,7 +511,7 @@ public void minTimestampBehavesCorrectly() throws Exception { "_airbyte_data": {} } """))); - actualTimestamp = destinationHandler.getMinTimestampForSync(streamId).get(); + tableState = getInitialRawTableState(incrementalAppendStream); // this is a pretty confusing pair of assertions. To explain them in more detail: There are three // records in the raw table: // * loaded_at not null, extracted_at = 2023-01-01 00:00Z @@ -493,14 +520,16 @@ public void minTimestampBehavesCorrectly() throws Exception { // We should have a timestamp which is older than the second record, but newer than or equal to // (i.e. not before) the first record. This allows us to query the raw table using // `_airbyte_extracted_at > ?`, which will include the second record and exclude the first record. + assertTrue(tableState.hasUnprocessedRecords(), + "When some raw records have null loaded_at, we should recognize that there are unprocessed records"); assertTrue( - actualTimestamp.isBefore(Instant.parse("2023-01-01T12:00:00Z")), + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T12:00:00Z")), "When some raw records have null loaded_at, the min timestamp should be earlier than the oldest unloaded record (2023-01-01 12:00Z). Was actually " - + actualTimestamp); + + tableState); assertFalse( - actualTimestamp.isBefore(Instant.parse("2023-01-01T00:00:00Z")), + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T00:00:00Z")), "When some raw records have null loaded_at, the min timestamp should be later than the newest loaded record older than the oldest unloaded record (2023-01-01 00:00Z). Was actually " - + actualTimestamp); + + tableState); } /** @@ -510,16 +539,23 @@ public void minTimestampBehavesCorrectly() throws Exception { */ @Test public void handlePreexistingRecords() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); createRawTable(streamId); createFinalTable(incrementalDedupStream, ""); insertRawTableRecords( streamId, BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); - final Optional minTimestampForSync = destinationHandler.getMinTimestampForSync(streamId); - assertTrue(minTimestampForSync.isPresent(), "After writing some raw records, the min timestamp should be present."); + final InitialRawTableState tableState = getInitialRawTableState(incrementalDedupStream); + assertAll( + () -> assertTrue(tableState.hasUnprocessedRecords(), + "After writing some raw records, we should recognize that there are unprocessed records"), + () -> assertTrue(tableState.maxProcessedTimestamp().isPresent(), "After writing some raw records, the min timestamp should be present.")); - TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, minTimestampForSync, ""); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, tableState.maxProcessedTimestamp(), ""); verifyRecords( "sqlgenerator/alltypes_expectedrecords_raw.jsonl", @@ -534,16 +570,22 @@ public void handlePreexistingRecords() throws Exception { */ @Test public void handleNoPreexistingRecords() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); createRawTable(streamId); - final Optional minTimestampForSync = destinationHandler.getMinTimestampForSync(streamId); - assertEquals(Optional.empty(), minTimestampForSync); + final InitialRawTableState tableState = getInitialRawTableState(incrementalDedupStream); + assertAll( + () -> assertFalse(tableState.hasUnprocessedRecords(), "With an empty raw table, we should recognize that there are no unprocessed records"), + () -> assertEquals(Optional.empty(), tableState.maxProcessedTimestamp(), "With an empty raw table, the min timestamp should be empty")); createFinalTable(incrementalDedupStream, ""); insertRawTableRecords( streamId, BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); - TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, minTimestampForSync, ""); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, tableState.maxProcessedTimestamp(), ""); verifyRecords( "sqlgenerator/alltypes_expectedrecords_raw.jsonl", @@ -745,7 +787,7 @@ public void overwriteFinalTable() throws Exception { "_tmp", records); - final String sql = generator.overwriteFinalTable(streamId, "_tmp"); + final Sql sql = generator.overwriteFinalTable(streamId, "_tmp"); destinationHandler.execute(sql); assertEquals(1, dumpFinalTableRecords(streamId, "").size()); @@ -858,8 +900,8 @@ public void testCdcOrdering_updateAfterDelete() throws Exception { streamId, BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl")); - final Optional minTimestampForSync = destinationHandler.getMinTimestampForSync(cdcIncrementalDedupStream.id()); - TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, minTimestampForSync, ""); + final InitialRawTableState tableState = getInitialRawTableState(cdcIncrementalDedupStream); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, tableState.maxProcessedTimestamp(), ""); verifyRecordCounts( 2, @@ -895,8 +937,8 @@ public void testCdcOrdering_insertAfterDelete() throws Exception { "", BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl")); - final Optional minTimestampForSync = destinationHandler.getMinTimestampForSync(cdcIncrementalAppendStream.id()); - TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, minTimestampForSync, ""); + final InitialRawTableState tableState = getInitialRawTableState(cdcIncrementalAppendStream); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, tableState.maxProcessedTimestamp(), ""); verifyRecordCounts( 2, dumpRawTableRecords(streamId), @@ -984,7 +1026,7 @@ public void weirdColumnNames() throws Exception { }); - final String createTable = generator.createTable(stream, "", false); + final Sql createTable = generator.createTable(stream, "", false); destinationHandler.execute(createTable); TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); @@ -1032,7 +1074,7 @@ public void noCrashOnSpecialCharacters(final String specialChars) throws Excepti }); - final String createTable = generator.createTable(stream, "", false); + final Sql createTable = generator.createTable(stream, "", false); destinationHandler.execute(createTable); // Not verifying anything about the data; let's just make sure we don't crash. TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); @@ -1066,7 +1108,7 @@ public void testReservedKeywords() throws Exception { }); - final String createTable = generator.createTable(stream, "", false); + final Sql createTable = generator.createTable(stream, "", false); destinationHandler.execute(createTable); TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); @@ -1100,7 +1142,7 @@ public void noColumns() throws Exception { Optional.empty(), new LinkedHashMap<>()); - final String createTable = generator.createTable(stream, "", false); + final Sql createTable = generator.createTable(stream, "", false); destinationHandler.execute(createTable); TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); @@ -1115,20 +1157,24 @@ public void noColumns() throws Exception { public void testV1V2migration() throws Exception { // This is maybe a little hacky, but it avoids having to refactor this entire class and subclasses // for something that is going away + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); final StreamId v1RawTableStreamId = new StreamId(null, null, streamId.finalNamespace(), "v1_" + streamId.rawName(), null, null); createV1RawTable(v1RawTableStreamId); insertV1RawTableRecords(v1RawTableStreamId, BaseTypingDedupingTest.readRecords( "sqlgenerator/all_types_v1_inputrecords.jsonl")); - final String migration = generator.migrateFromV1toV2(streamId, v1RawTableStreamId.rawNamespace(), v1RawTableStreamId.rawName()); + final Sql migration = generator.migrateFromV1toV2(streamId, v1RawTableStreamId.rawNamespace(), v1RawTableStreamId.rawName()); destinationHandler.execute(migration); final List v1RawRecords = dumpV1RawTableRecords(v1RawTableStreamId); final List v2RawRecords = dumpRawTableRecords(streamId); migrationAssertions(v1RawRecords, v2RawRecords); // And then run T+D on the migrated raw data - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(createTable); - final String updateTable = generator.updateTable(incrementalDedupStream, "", Optional.empty(), true); + final Sql updateTable = generator.updateTable(incrementalDedupStream, "", Optional.empty(), true); destinationHandler.execute(updateTable); verifyRecords( "sqlgenerator/alltypes_expectedrecords_raw.jsonl", @@ -1148,7 +1194,7 @@ public void softResetIgnoresPreexistingTempTable() throws Exception { // Create a soft reset table. Use incremental append mode, in case the destination connector uses // different // indexing/partitioning/etc. - final String createOldTempTable = generator.createTable(incrementalAppendStream, TypeAndDedupeTransaction.SOFT_RESET_SUFFIX, false); + final Sql createOldTempTable = generator.createTable(incrementalAppendStream, TypeAndDedupeTransaction.SOFT_RESET_SUFFIX, false); destinationHandler.execute(createOldTempTable); // Execute a soft reset. This should not crash. @@ -1160,8 +1206,8 @@ protected void migrationAssertions(final List v1RawRecords, final List record -> record.get("_airbyte_raw_id").asText(), Function.identity())); assertAll( - () -> assertEquals(5, v1RawRecords.size()), - () -> assertEquals(5, v2RawRecords.size())); + () -> assertEquals(6, v1RawRecords.size()), + () -> assertEquals(6, v2RawRecords.size())); v1RawRecords.forEach(v1Record -> { final var v1id = v1Record.get("_airbyte_ab_id").asText(); assertAll( @@ -1188,19 +1234,19 @@ protected List dumpV1RawTableRecords(final StreamId streamId) throws E @Test public void testCreateTableForce() throws Exception { - final String createTableNoForce = generator.createTable(incrementalDedupStream, "", false); - final String createTableForce = generator.createTable(incrementalDedupStream, "", true); + final Sql createTableNoForce = generator.createTable(incrementalDedupStream, "", false); + final Sql createTableForce = generator.createTable(incrementalDedupStream, "", true); destinationHandler.execute(createTableNoForce); assertThrows(Exception.class, () -> destinationHandler.execute(createTableNoForce)); // This should not throw an exception destinationHandler.execute(createTableForce); - - assertTrue(destinationHandler.findExistingTable(streamId).isPresent()); + // This method call ensures assertion than finalTable exists + getDestinationInitialState(incrementalDedupStream); } protected void createFinalTable(final StreamConfig stream, final String suffix) throws Exception { - final String createTable = generator.createTable(stream, suffix, false); + final Sql createTable = generator.createTable(stream, suffix, false); destinationHandler.execute(createTable); } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java index 2eceef997ccf..e52c669dc798 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertAll; import com.fasterxml.jackson.databind.JsonNode; @@ -35,6 +36,9 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; import java.util.function.Function; import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; @@ -118,7 +122,7 @@ public abstract class BaseTypingDedupingTest { * @return * @throws Exception */ - protected boolean checkTableExists(String streamNamespace, String streamName) { + protected boolean checkTableExists(final String streamNamespace, final String streamName) { // Implementation is specific to destination's tests. return true; } @@ -152,7 +156,7 @@ protected boolean checkTableExists(String streamNamespace, String streamName) { */ protected abstract void teardownStreamAndNamespace(String streamNamespace, String streamName) throws Exception; - protected abstract SqlGenerator getSqlGenerator(); + protected abstract SqlGenerator getSqlGenerator(); /** * Destinations which need to clean up resources after an entire test finishes should override this @@ -185,7 +189,7 @@ protected Map getFinalMetadataColumnNames() { */ protected synchronized String getUniqueSuffix() { if (randomSuffix == null) { - randomSuffix = "_" + RandomStringUtils.randomAlphabetic(5).toLowerCase(); + randomSuffix = "_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); } return randomSuffix; } @@ -211,7 +215,7 @@ public void setup() throws Exception { streamName = "test_stream" + getUniqueSuffix(); streamsToTearDown = new ArrayList<>(); - final SqlGenerator generator = getSqlGenerator(); + final SqlGenerator generator = getSqlGenerator(); DIFFER = new RecordDiffer( getRawMetadataColumnNames(), getFinalMetadataColumnNames(), @@ -377,6 +381,36 @@ public void incrementalDedup() throws Exception { verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); } + /** + * Run the first sync from {@link #incrementalDedup()}, but repeat the messages many times. Some + * destinations behave differently with small vs large record count, so this test case tries to + * exercise that behavior. + */ + @Test + public void largeDedupSync() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // Run a sync with 25K copies of the input messages + final List messages1 = repeatList(25_000, readMessages("dat/sync1_messages.jsonl")); + + runSync(catalog, messages1); + + // The raw table will contain 25K copies of each record + final List expectedRawRecords1 = repeatList(25_000, readRecords("dat/sync1_expectedrecords_raw.jsonl")); + // But the final table should be fully deduped + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + } + /** * Identical to {@link #incrementalDedup()}, except that the stream has no namespace. */ @@ -596,20 +630,21 @@ public void identicalNameSimultaneousSync() throws Exception { // Start two concurrent syncs final AirbyteDestination sync1 = startSync(catalog1); final AirbyteDestination sync2 = startSync(catalog2); + CompletableFuture> outFuture1 = destinationOutputFuture(sync1); + CompletableFuture> outFuture2 = destinationOutputFuture(sync2); + // Write some messages to both syncs. Write a lot of data to sync 2 to try and force a flush. pushMessages(messages1, sync1); for (int i = 0; i < 100_000; i++) { pushMessages(messages2, sync2); } - // This will dump sync1's entire stdout to our stdout - endSync(sync1); + endSync(sync1, outFuture1); // Write some more messages to the second sync. It should not be affected by the first sync's // shutdown. for (int i = 0; i < 100_000; i++) { pushMessages(messages2, sync2); } - // And this will dump sync2's entire stdout to our stdout - endSync(sync2); + endSync(sync2, outFuture2); // For simplicity, don't verify the raw table. Assume that if the final table is correct, then // the raw data is correct. This is generally a safe assumption. @@ -703,9 +738,17 @@ public void testDataTypes() throws Exception { // this test probably needs some configuration per destination to specify what values are supported? } + private List repeatList(final int n, final List list) { + return Collections + .nCopies(n, list) + .stream() + .flatMap(List::stream) + .collect(toList()); + } + protected void verifySyncResult(final List expectedRawRecords, final List expectedFinalRecords, - boolean disableFinalTableComparison) + final boolean disableFinalTableComparison) throws Exception { verifySyncResult(expectedRawRecords, expectedFinalRecords, streamNamespace, streamName, disableFinalTableComparison); } @@ -714,7 +757,7 @@ private void verifySyncResult(final List expectedRawRecords, final List expectedFinalRecords, final String streamNamespace, final String streamName, - boolean disableFinalTableComparison) + final boolean disableFinalTableComparison) throws Exception { final List actualRawRecords = dumpRawTableRecords(streamNamespace, streamName); if (disableFinalTableComparison) { @@ -764,11 +807,29 @@ protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages, final String imageName, - Function configTransformer) + final Function configTransformer) throws Exception { final AirbyteDestination destination = startSync(catalog, imageName, configTransformer); + final CompletableFuture> outputFuture = destinationOutputFuture(destination); pushMessages(messages, destination); - endSync(destination); + endSync(destination, outputFuture); + } + + // In the background, read messages from the destination until it terminates. We need to clear + // stdout in real time, to prevent the buffer from filling up and blocking the destination. + private CompletableFuture> destinationOutputFuture(final AirbyteDestination destination) { + final CompletableFuture> outputFuture = new CompletableFuture<>(); + Executors.newSingleThreadExecutor().submit((Callable) () -> { + final List destinationMessages = new ArrayList<>(); + while (!destination.isFinished()) { + // attemptRead isn't threadsafe, we read stdout fully here. + // i.e. we shouldn't call attemptRead anywhere else. + destination.attemptRead().ifPresent(destinationMessages::add); + } + outputFuture.complete(destinationMessages); + return null; + }); + return outputFuture; } protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog) throws Exception { @@ -790,7 +851,7 @@ protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog, f */ protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog, final String imageName, - Function configTransformer) + final Function configTransformer) throws Exception { synchronized (this) { catalog.getStreams().forEach(s -> streamsToTearDown.add(AirbyteStreamNameNamespacePair.fromAirbyteStream(s.getStream()))); @@ -833,14 +894,13 @@ protected static void pushMessages(final List messages, final Ai message -> Exceptions.toRuntime(() -> destination.accept(convertProtocolObject(message, io.airbyte.protocol.models.AirbyteMessage.class)))); } - // TODO Eventually we'll want to somehow extract the state messages while a sync is running, to - // verify checkpointing. - // That's going to require some nontrivial changes to how attemptRead() works. - protected static void endSync(final AirbyteDestination destination) throws Exception { + protected void endSync(final AirbyteDestination destination, + final CompletableFuture> destinationOutputFuture) + throws Exception { destination.notifyEndOfInput(); - while (!destination.isFinished()) { - destination.attemptRead(); - } + // TODO Eventually we'll want to somehow extract the state messages while a sync is running, to + // verify checkpointing. + destinationOutputFuture.join(); destination.close(); } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl index 4c5dec1a24ea..653e49e39e20 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl @@ -10,3 +10,5 @@ {"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}} // Emit a record with an invalid age. {"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}} +// Emit a record with interesting characters in one of the values. +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl index 3938dd7b53d1..e2cde49ad980 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl @@ -4,3 +4,4 @@ // Note that array and struct have invalid values ({} and [] respectively). {"_airbyte_ab_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_ab_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_ab_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl index 0491c86d495c..c21fc0bbb6ab 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl @@ -4,3 +4,4 @@ // Note that array and struct have invalid values ({} and [] respectively). {"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl new file mode 100644 index 000000000000..55a509408d14 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl @@ -0,0 +1,3 @@ +// this is a strict subset of the alltypes_inputrecords file. All these records have valid values, i.e. can be processed with unsafe casting. +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index ec15406cc32c..cae50f52aa81 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.57.2 +current_version = 0.67.0 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index a2ac0ce2514e..aafd9d33a8a7 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,113 @@ # Changelog +## 0.67.0 +Low-code: Add CustomRecordFilter + +## 0.66.0 +Low-code: Add interpolation for request options + +## 0.65.0 +low-code: Allow connectors to ignore stream slicer request options on paginated requests + +## 0.64.1 + + +## 0.64.0 +Low-code: Add filter to RemoveFields + +## 0.63.2 +Correct handling of custom max_records limits in connector_builder + +## 0.63.1 +File-based CDK: fix record enqueuing + +## 0.63.0 +Per-stream error reporting and continue syncing on error by default + +## 0.62.2 +mask access key when logging refresh response + +## 0.62.1 +[ISSUE #34910] add headers to HttpResponse for test framework + +## 0.62.0 +File-based CDK: functionality to make incremental syncs concurrent + +## 0.61.2 +[ISSUE #34755] do not propagate parameters on JSON schemas + +## 0.61.1 +Align version in CDK Dockerfile to be consistent. Before this change, the docker images was mistakenly pinned to version 0.58.5. + +## 0.61.0 +File-based CDK: log warning on no sync mode instead of raising exception + +## 0.60.2 +Improve error messages for concurrent CDK + +## 0.60.1 +Emit state when no partitions are generated for ccdk and update StateBuilder + +## 0.60.0 +File-based CDK: run full refresh syncs with concurrency + +## 0.59.2 +Fix CCDK overlapping message due to print in entrypoint + +## 0.59.1 +Fix concurrent CDK deadlock + +## 0.59.0 +Fix state message handling when running concurrent syncs + +## 0.58.9 +concurrent-cdk: improve resource usage when reading from substreams + +## 0.58.8 +CDK: HttpRequester can accept http_method in str format, which is required by custom low code components + +## 0.58.7 + + +## 0.58.6 +File CDK: Added logic to emit logged `RecordParseError` errors and raise the single `AirbyteTracebackException` in the end of the sync, instead of silent skipping the parsing errors. PR: https://github.com/airbytehq/airbyte/pull/32589 + +## 0.58.5 +Handle private network exception as config error + +## 0.58.4 +Add POST method to HttpMocker + +## 0.58.3 +fix declarative oauth initialization + +## 0.58.2 +Integration tests: adding debug mode to improve logging + +## 0.58.1 +Add schema normalization to declarative stream + +## 0.58.0 +Concurrent CDK: add state converter for ISO timestamps with millisecond granularity + +## 0.57.8 +add SelectiveAuthenticator + +## 0.57.7 +File CDK: Support raw txt file + +## 0.57.6 +Adding more tooling to cover source-stripe events stream + +## 0.57.5 +Raise error on passing unsupported value formats as query parameters + +## 0.57.4 +Vector DB CDK: Refactor embedders, File based CDK: Handle 422 errors properly in document file type parser + +## 0.57.3 +Vector DB CDK: Refactor embedders, File based CDK: Handle 422 errors properly in document file type parser + ## 0.57.2 Update airbyte-protocol @@ -8,6 +116,9 @@ Improve integration tests tooling ## 0.57.0 low-code: cache requests sent for parent streams +File-based CDK: Add support for automatic primary key for document file type format +File-based CDK: Add support for remote parsing of document file type format via API +Vector DB CDK: Fix bug with embedding tokens with special meaning like `<|endoftext|>` ## 0.56.1 no-op to verify pypi publish flow diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 7b49a505e656..2c1ea428129c 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.57.2 +RUN pip install --prefix=/install airbyte-cdk==0.67.0 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.57.2 +LABEL io.airbyte.version=0.67.0 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index c3ac3221b622..b8998a9d8d83 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -1,10 +1,9 @@ # Connector Development Kit \(Python\) -The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently offers helpers specific for creating Airbyte source connectors for: +The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors.The CDK currently offers helpers specific for creating Airbyte source connectors for: -* HTTP APIs \(REST APIs, GraphQL, etc..\) -* Singer Taps -* Generic Python sources \(anything not covered by the above\) +- HTTP APIs \(REST APIs, GraphQL, etc..\) +- Generic Python sources \(anything not covered by the above\) The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate. @@ -14,14 +13,14 @@ This document is a general introduction to the CDK. Readers should have basic fa Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run -```text +```bash cd airbyte-integrations/connector-templates/generator ./generate.sh ``` then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. -Additionally, you can follow [this tutorial](https://docs.airbyte.io/connector-development/tutorials/cdk-tutorial-python-http) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. +Additionally, you can follow [this tutorial](https://docs.airbyte.com/connector-development/cdk-python/) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. ### Concepts & Documentation @@ -31,29 +30,23 @@ See the [concepts docs](docs/concepts/) for a tour through what the API offers. **HTTP Connectors**: -* [Exchangerates API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py) -* [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) -* [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) - -**Singer connectors**: - -* [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py) -* [Github](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-github-singer/source_github_singer/source.py) +- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) +- [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) **Simple Python connectors using the barebones `Source` abstraction**: -* [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py) -* [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py) +- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py) +- [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py) ## Contributing ### First time setup -We assume `python` points to python >=3.8. +We assume `python` points to Python 3.9 or higher. Setup a virtual env: -```text +```bash python -m venv .venv source .venv/bin/activate pip install -e ".[dev]" # [dev] installs development-only dependencies @@ -61,17 +54,20 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies #### Iteration -* Iterate on the code locally -* Run tests via `python -m pytest -s unit_tests` -* Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`. - * Run `mypy ` to only check specific files. This is useful as the CDK still contains code that is not compliant. -* The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! +- Iterate on the code locally +- Run tests via `python -m pytest -s unit_tests` +- Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`. +- Run `mypy ` to only check specific files. This is useful as the CDK still contains code that is not compliant. +- The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! ##### Autogenerated files + If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run: -```commandline -./gradlew :airbyte-cdk:python:format + +```bash +./gradlew :airbyte-cdk:python:build ``` + This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`. @@ -82,14 +78,16 @@ All tests are located in the `unit_tests` directory. Run `python -m pytest --cov #### Building and testing a connector with your local CDK When developing a new feature in the CDK, you may find it helpful to run a connector that uses that new feature. You can test this in one of two ways: -* Running a connector locally -* Building and running a source via Docker + +- Running a connector locally +- Building and running a source via Docker ##### Installing your local CDK into a local Python connector In order to get a local Python connector running your local CDK, do the following. First, make sure you have your connector's virtual environment active: + ```bash # from the `airbyte/airbyte-integrations/connectors/` directory source .venv/bin/activate @@ -99,6 +97,7 @@ pip install -e . ``` Then, navigate to the CDK and install it in editable mode: + ```bash cd ../../../airbyte-cdk/python pip install -e . @@ -107,28 +106,35 @@ pip install -e . You should see that `pip` has uninstalled the version of `airbyte-cdk` defined by your connector's `setup.py` and installed your local CDK. Any changes you make will be immediately reflected in your editor, so long as your editor's interpreter is set to your connector's virtual environment. ##### Building a Python connector in Docker with your local CDK installed + _Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_ You can build your connector image with the local CDK using + ```bash # from the airbytehq/airbyte base directory airbyte-ci connectors --use-local-cdk --name= build ``` + Note that the local CDK is injected at build time, so if you make changes, you will have to run the build command again to see them reflected. ##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed + _Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_ To run acceptance tests for a single connectors using the local CDK, from the connector directory, run + ```bash airbyte-ci connectors --use-local-cdk --name= test ``` #### When you don't have access to the API + There can be some time where you do not have access to the API (either because you don't have the credentials, network access, etc...) You will probably still want to do end-to-end testing at least once. In order to do so, you can emulate the server you would be reaching using a server stubbing tool. For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file like this: -``` + +```json { "httpRequest": { "method": "GET", @@ -141,7 +147,10 @@ For example, using [mockserver](https://www.mock-server.com/), you can set up an ``` Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the following command will allow to match any requests on path `/data` to return the response defined in the expectation file: -`docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0` + +```bash +docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0 +``` HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file. To test this, the implementer either has to change the code which defines the base URL for Python source or update the `url_base` from low-code. With the Connector Builder running in docker, you will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed within docker. @@ -149,11 +158,3 @@ HTTP requests to `localhost:8113/data` should now return the body defined in the 1. Open a PR 2. Once it is approved and **merged**, an Airbyte member must run the `Publish CDK Manually` workflow from master using `release-type=major|manor|patch` and setting the changelog message. - -## Coming Soon - -* Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) -* Airbyte Java HTTP CDK -* CDK for Async HTTP endpoints \(request-poll-wait style endpoints\) -* CDK for other protocols -* Don't see a feature you need? [Create an issue and let us know how we can help!](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement&template=feature-request.md&title=) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md b/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md index 3f3402fab536..6788a6f226b1 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md @@ -22,6 +22,7 @@ Note: *See [ConnectionSpecification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#actor-specification) for details on the `"config"` key if needed. - When the `__command` is `resolve_manifest`, the argument to `catalog` should be an empty string. +- The config can optionally contain an object under the `__test_read_config` key which can define custom test read limits with `max_records`, `max_slices`, and `max_pages_per_slice` properties. All custom limits are optional; a default will be used for any limit that is not provided. ### Locally running the docker image diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py index c53bda0dfc62..4dfe4a3dd05d 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -57,7 +57,7 @@ def read_stream( source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, limits: TestReadLimits ) -> AirbyteMessage: try: - handler = MessageGrouper(limits.max_pages_per_slice, limits.max_slices) + handler = MessageGrouper(limits.max_pages_per_slice, limits.max_slices, limits.max_records) stream_name = configured_catalog.streams[0].stream.name # The connector builder only supports a single stream stream_read = handler.get_message_groups(source, config, configured_catalog, limits.max_records) return AirbyteMessage( diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index 42a0e1051b52..b30a3a3744f4 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -52,8 +52,8 @@ def get_message_groups( configured_catalog: ConfiguredAirbyteCatalog, record_limit: Optional[int] = None, ) -> StreamRead: - if record_limit is not None and not (1 <= record_limit <= 1000): - raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}") + if record_limit is not None and not (1 <= record_limit <= self._max_record_limit): + raise ValueError(f"Record limit must be between 1 and {self._max_record_limit}. Got {record_limit}") schema_inferrer = SchemaInferrer() datetime_format_inferrer = DatetimeFormatInferrer() diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py index a1f89b05648b..7fb880fadaae 100644 --- a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py @@ -4,6 +4,7 @@ import os from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import List, Optional, Union, cast from airbyte_cdk.destinations.vector_db_based.config import ( @@ -15,8 +16,8 @@ OpenAIEmbeddingConfigModel, ProcessingConfigModel, ) -from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType from langchain.embeddings.cohere import CohereEmbeddings from langchain.embeddings.fake import FakeEmbeddings @@ -24,6 +25,12 @@ from langchain.embeddings.openai import OpenAIEmbeddings +@dataclass +class Document: + page_content: str + record: AirbyteRecordMessage + + class Embedder(ABC): """ Embedder is an abstract class that defines the interface for embedding text. @@ -41,7 +48,7 @@ def check(self) -> Optional[str]: pass @abstractmethod - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: """ Embed the text of each chunk and return the resulting embedding vectors. If a chunk cannot be embedded or is configured to not be embedded, return None for that chunk. @@ -72,7 +79,7 @@ def check(self) -> Optional[str]: return format_exception(e) return None - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: """ Embed the text of each chunk and return the resulting embedding vectors. @@ -80,9 +87,9 @@ def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: It's still possible to run into the rate limit between each embed call because the available token budget hasn't recovered between the calls, but the built-in retry mechanism of the OpenAI client handles that. """ - # Each chunk can hold at most self.chunk_size tokens, so tokens-per-minute by maximum tokens per chunk is the number of chunks that can be embedded at once without exhausting the limit in a single request + # Each chunk can hold at most self.chunk_size tokens, so tokens-per-minute by maximum tokens per chunk is the number of documents that can be embedded at once without exhausting the limit in a single request embedding_batch_size = OPEN_AI_TOKEN_LIMIT // self.chunk_size - batches = create_chunks(chunks, batch_size=embedding_batch_size) + batches = create_chunks(documents, batch_size=embedding_batch_size) embeddings: List[Optional[List[float]]] = [] for batch in batches: embeddings.extend(self.embeddings.embed_documents([chunk.page_content for chunk in batch])) @@ -121,8 +128,8 @@ def check(self) -> Optional[str]: return format_exception(e) return None - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: - return cast(List[Optional[List[float]]], self.embeddings.embed_documents([chunk.page_content or "" for chunk in chunks])) + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) @property def embedding_dimensions(self) -> int: @@ -142,8 +149,8 @@ def check(self) -> Optional[str]: return format_exception(e) return None - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: - return cast(List[Optional[List[float]]], self.embeddings.embed_documents([chunk.page_content or "" for chunk in chunks])) + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) @property def embedding_dimensions(self) -> int: @@ -173,8 +180,8 @@ def check(self) -> Optional[str]: return format_exception(e) return None - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: - return cast(List[Optional[List[float]]], self.embeddings.embed_documents([chunk.page_content or "" for chunk in chunks])) + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) @property def embedding_dimensions(self) -> int: @@ -190,32 +197,32 @@ def __init__(self, config: FromFieldEmbeddingConfigModel): def check(self) -> Optional[str]: return None - def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]: + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: """ From each chunk, pull the embedding from the field specified in the config. Check that the field exists, is a list of numbers and is the correct size. If not, raise an AirbyteTracedException explaining the problem. """ embeddings: List[Optional[List[float]]] = [] - for chunk in chunks: - data = chunk.record.data + for document in documents: + data = document.record.data if self.config.field_name not in data: raise AirbyteTracedException( internal_message="Embedding vector field not found", failure_type=FailureType.config_error, - message=f"Record {str(data)[:250]}... in stream {chunk.record.stream} does not contain embedding vector field {self.config.field_name}. Please check your embedding configuration, the embedding vector field has to be set correctly on every record.", + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does not contain embedding vector field {self.config.field_name}. Please check your embedding configuration, the embedding vector field has to be set correctly on every record.", ) field = data[self.config.field_name] if not isinstance(field, list) or not all(isinstance(x, (int, float)) for x in field): raise AirbyteTracedException( internal_message="Embedding vector field not a list of numbers", failure_type=FailureType.config_error, - message=f"Record {str(data)[:250]}... in stream {chunk.record.stream} does contain embedding vector field {self.config.field_name}, but it is not a list of numbers. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it is not a list of numbers. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", ) if len(field) != self.config.dimensions: raise AirbyteTracedException( internal_message="Embedding vector field has wrong length", failure_type=FailureType.config_error, - message=f"Record {str(data)[:250]}... in stream {chunk.record.stream} does contain embedding vector field {self.config.field_name}, but it has length {len(field)} instead of the configured {self.config.dimensions}. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it has length {len(field)} instead of the configured {self.config.dimensions}. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", ) embeddings.append(field) diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py index e8d58abb4ad6..0f764c366b54 100644 --- a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py @@ -8,7 +8,7 @@ from airbyte_cdk.destinations.vector_db_based.config import ProcessingConfigModel from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk, DocumentProcessor -from airbyte_cdk.destinations.vector_db_based.embedder import Embedder +from airbyte_cdk.destinations.vector_db_based.embedder import Document, Embedder from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type @@ -16,14 +16,14 @@ class Writer: """ The Writer class is orchestrating the document processor, the embedder and the indexer: - * Incoming records are passed through the document processor to generate documents - * One the configured batch size is reached, the documents are passed to the embedder to generate embeddings - * The embedder embeds the documents - * The indexer deletes old documents by the associated record id before indexing the new ones + * Incoming records are passed through the document processor to generate chunks + * One the configured batch size is reached, the chunks are passed to the embedder to generate embeddings + * The embedder embeds the chunks + * The indexer deletes old chunks by the associated record id before indexing the new ones The destination connector is responsible to create a writer instance and pass the input messages iterable to the write method. The batch size can be configured by the destination connector to give the freedom of either letting the user configure it or hardcoding it to a sensible value depending on the destination. - The omit_raw_text parameter can be used to omit the raw text from the documents. This can be useful if the raw text is very large and not needed for the destination. + The omit_raw_text parameter can be used to omit the raw text from the chunks. This can be useful if the raw text is very large and not needed for the destination. """ def __init__( @@ -37,21 +37,29 @@ def __init__( self._init_batch() def _init_batch(self) -> None: - self.documents: Dict[Tuple[str, str], List[Chunk]] = defaultdict(list) + self.chunks: Dict[Tuple[str, str], List[Chunk]] = defaultdict(list) self.ids_to_delete: Dict[Tuple[str, str], List[str]] = defaultdict(list) - self.number_of_documents = 0 + self.number_of_chunks = 0 + + def _convert_to_document(self, chunk: Chunk) -> Document: + """ + Convert a chunk to a document for the embedder. + """ + if chunk.page_content is None: + raise ValueError("Cannot embed a chunk without page content") + return Document(page_content=chunk.page_content, record=chunk.record) def _process_batch(self) -> None: for (namespace, stream), ids in self.ids_to_delete.items(): self.indexer.delete(ids, namespace, stream) - for (namespace, stream), documents in self.documents.items(): - embeddings = self.embedder.embed_chunks(documents) - for i, document in enumerate(documents): + for (namespace, stream), chunks in self.chunks.items(): + embeddings = self.embedder.embed_documents([self._convert_to_document(chunk) for chunk in chunks]) + for i, document in enumerate(chunks): document.embedding = embeddings[i] if self.omit_raw_text: document.page_content = None - self.indexer.index(documents, namespace, stream) + self.indexer.index(chunks, namespace, stream) self._init_batch() @@ -65,12 +73,12 @@ def write(self, configured_catalog: ConfiguredAirbyteCatalog, input_messages: It self._process_batch() yield message elif message.type == Type.RECORD: - record_documents, record_id_to_delete = self.processor.process(message.record) - self.documents[(message.record.namespace, message.record.stream)].extend(record_documents) + record_chunks, record_id_to_delete = self.processor.process(message.record) + self.chunks[(message.record.namespace, message.record.stream)].extend(record_chunks) if record_id_to_delete is not None: self.ids_to_delete[(message.record.namespace, message.record.stream)].append(record_id_to_delete) - self.number_of_documents += len(record_documents) - if self.number_of_documents >= self.batch_size: + self.number_of_chunks += len(record_chunks) + if self.number_of_chunks >= self.batch_size: self._process_batch() self._process_batch() diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index f89e0ef0ec29..3852cb7e9890 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -26,6 +26,7 @@ from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import FailureType from requests import PreparedRequest, Response, Session logger = init_logger("airbyte") @@ -211,7 +212,9 @@ def launch(source: Source, args: List[str]) -> None: source_entrypoint = AirbyteEntrypoint(source) parsed_args = source_entrypoint.parse_args(args) for message in source_entrypoint.run(parsed_args): - print(message) + # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and + # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time + print(f"{message}\n", end="") def _init_internal_request_filter() -> None: @@ -236,9 +239,10 @@ def filtered_send(self: Any, request: PreparedRequest, **kwargs: Any) -> Respons try: is_private = _is_private_url(parsed_url.hostname, parsed_url.port) # type: ignore [arg-type] if is_private: - raise ValueError( - "Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source " - + "connectors only support requesting data from public API endpoints." + raise AirbyteTracedException( + internal_message=f"Invalid URL endpoint: `{parsed_url.hostname!r}` belongs to a private network", + failure_type=FailureType.config_error, + message="Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source connectors only support requesting data from public API endpoints.", ) except socket.gaierror as exception: # This is a special case where the developer specifies an IP address string that is not formatted correctly like trailing diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index 0f8bf716cc10..a5c4b847f182 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -14,7 +14,9 @@ AirbyteStreamStatus, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, + FailureType, Status, + StreamDescriptor, SyncMode, ) from airbyte_cdk.models import Type as MessageType @@ -27,6 +29,7 @@ from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger +from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets from airbyte_cdk.utils.event_timing import create_timer from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -99,7 +102,7 @@ def read( # TODO assert all streams exist in the connector # get the streams once in case the connector needs to make any queries to generate them stream_instances = {s.name: s for s in self.streams(config)} - state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) + state_manager = ConnectorStateManager(stream_instance_map={s.stream.name: s.stream for s in catalog.streams}, state=state) self._stream_to_instance_map = stream_instances stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {} @@ -133,11 +136,16 @@ def read( logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE) except AirbyteTracedException as e: + logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") + logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) - if self.continue_sync_on_stream_failure: - stream_name_to_exception[stream_instance.name] = e - else: - raise e + yield e.as_sanitized_airbyte_message(stream_descriptor=StreamDescriptor(name=configured_stream.stream.name)) + stream_name_to_exception[stream_instance.name] = e + if self.stop_sync_on_stream_failure: + logger.info( + f"Stopping sync on error from stream {configured_stream.stream.name} because {self.name} does not support continuing syncs on error." + ) + break except Exception as e: yield from self._emit_queued_messages() logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") @@ -145,15 +153,28 @@ def read( yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) display_message = stream_instance.get_error_display_message(e) if display_message: - raise AirbyteTracedException.from_exception(e, message=display_message) from e - raise e + traced_exception = AirbyteTracedException.from_exception(e, message=display_message) + else: + traced_exception = AirbyteTracedException.from_exception(e) + yield traced_exception.as_sanitized_airbyte_message( + stream_descriptor=StreamDescriptor(name=configured_stream.stream.name) + ) + stream_name_to_exception[stream_instance.name] = traced_exception + if self.stop_sync_on_stream_failure: + logger.info(f"{self.name} does not support continuing syncs on error from stream {configured_stream.stream.name}") + break finally: timer.finish_event() logger.info(f"Finished syncing {configured_stream.stream.name}") logger.info(timer.report()) - if self.continue_sync_on_stream_failure and len(stream_name_to_exception) > 0: - raise AirbyteTracedException(message=self._generate_failed_streams_error_message(stream_name_to_exception)) + if len(stream_name_to_exception) > 0: + error_message = self._generate_failed_streams_error_message(stream_name_to_exception) + logger.info(error_message) + # We still raise at least one exception when a stream raises an exception because the platform currently relies + # on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error + # type because this combined error isn't actionable, but rather the previously emitted individual errors. + raise AirbyteTracedException(message=error_message, failure_type=FailureType.config_error) logger.info(f"Finished syncing {self.name}") @property @@ -282,17 +303,17 @@ def message_repository(self) -> Union[None, MessageRepository]: return _default_message_repository @property - def continue_sync_on_stream_failure(self) -> bool: + def stop_sync_on_stream_failure(self) -> bool: """ WARNING: This function is in-development which means it is subject to change. Use at your own risk. - By default, a source should raise an exception and stop the sync when it encounters an error while syncing a stream. This - method can be overridden on a per-source basis so that a source will continue syncing streams other streams even if an - exception is raised for a stream. + By default, when a source encounters an exception while syncing a stream, it will emit an error trace message and then + continue syncing the next stream. This can be overwritten on a per-source basis so that the source will stop the sync + on the first error seen and emit a single error trace message for that stream. """ return False @staticmethod def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str: - failures = ", ".join([f"{stream}: {exception.__repr__()}" for stream, exception in stream_failures.items()]) + failures = ", ".join([f"{stream}: {filter_secrets(exception.__repr__())}" for stream, exception in stream_failures.items()]) return f"During the sync, the following streams did not sync successfully: {failures}" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py index d73a524f9625..24ac315c526e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py @@ -43,9 +43,9 @@ def __init__( """ self._stream_name_to_instance = {s.name: s for s in stream_instances_to_read_from} self._record_counter = {} - self._streams_to_partitions: Dict[str, Set[Partition]] = {} + self._streams_to_running_partitions: Dict[str, Set[Partition]] = {} for stream in stream_instances_to_read_from: - self._streams_to_partitions[stream.name] = set() + self._streams_to_running_partitions[stream.name] = set() self._record_counter[stream.name] = 0 self._thread_pool_manager = thread_pool_manager self._partition_enqueuer = partition_enqueuer @@ -55,6 +55,7 @@ def __init__( self._slice_logger = slice_logger self._message_repository = message_repository self._partition_reader = partition_reader + self._streams_done: Set[str] = set() def on_partition_generation_completed(self, sentinel: PartitionGenerationCompletedSentinel) -> Iterable[AirbyteMessage]: """ @@ -65,13 +66,12 @@ def on_partition_generation_completed(self, sentinel: PartitionGenerationComplet """ stream_name = sentinel.stream.name self._streams_currently_generating_partitions.remove(sentinel.stream.name) - ret = [] # It is possible for the stream to already be done if no partitions were generated - if self._is_stream_done(stream_name): - ret.append(self._on_stream_is_done(stream_name)) + # If the partition generation process was completed and there are no partitions left to process, the stream is done + if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0: + yield from self._on_stream_is_done(stream_name) if self._stream_instances_to_start_partition_generation: - ret.append(self.start_next_partition_generator()) - return ret + yield self.start_next_partition_generator() def on_partition(self, partition: Partition) -> None: """ @@ -81,7 +81,7 @@ def on_partition(self, partition: Partition) -> None: 3. Submit the partition to the thread pool manager """ stream_name = partition.stream_name() - self._streams_to_partitions[stream_name].add(partition) + self._streams_to_running_partitions[stream_name].add(partition) if self._slice_logger.should_log_slice_message(self._logger): self._message_repository.emit_message(self._slice_logger.create_slice_log_message(partition.to_slice())) self._thread_pool_manager.submit(self._partition_reader.process_partition, partition) @@ -95,8 +95,12 @@ def on_partition_complete_sentinel(self, sentinel: PartitionCompleteSentinel) -> """ partition = sentinel.partition partition.close() - if self._is_stream_done(partition.stream_name()): - yield self._on_stream_is_done(partition.stream_name()) + partitions_running = self._streams_to_running_partitions[partition.stream_name()] + if partition in partitions_running: + partitions_running.remove(partition) + # If all partitions were generated and this was the last one, the stream is done + if partition.stream_name() not in self._streams_currently_generating_partitions and len(partitions_running) == 0: + yield from self._on_stream_is_done(partition.stream_name()) yield from self._message_repository.consume_queue() def on_record(self, record: Record) -> Iterable[AirbyteMessage]: @@ -114,11 +118,10 @@ def on_record(self, record: Record) -> Iterable[AirbyteMessage]: message = stream_data_to_airbyte_message(record.stream_name, record.data) stream = self._stream_name_to_instance[record.stream_name] - if self._record_counter[stream.name] == 0: - self._logger.info(f"Marking stream {stream.name} as RUNNING") - yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING) - if message.type == MessageType.RECORD: + if self._record_counter[stream.name] == 0: + self._logger.info(f"Marking stream {stream.name} as RUNNING") + yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING) self._record_counter[stream.name] += 1 yield message yield from self._message_repository.consume_queue() @@ -161,30 +164,26 @@ def is_done(self) -> bool: 2. There are no more streams to read from 3. All partitions for all streams are closed """ - return ( - not self._streams_currently_generating_partitions - and not self._stream_instances_to_start_partition_generation - and all([all(p.is_closed() for p in partitions) for partitions in self._streams_to_partitions.values()]) - ) + return all([self._is_stream_done(stream_name) for stream_name in self._stream_name_to_instance.keys()]) def _is_stream_done(self, stream_name: str) -> bool: - return ( - all([p.is_closed() for p in self._streams_to_partitions[stream_name]]) - and stream_name not in self._streams_currently_generating_partitions - ) + return stream_name in self._streams_done - def _on_stream_is_done(self, stream_name: str) -> AirbyteMessage: + def _on_stream_is_done(self, stream_name: str) -> Iterable[AirbyteMessage]: self._logger.info(f"Read {self._record_counter[stream_name]} records from {stream_name} stream") self._logger.info(f"Marking stream {stream_name} as STOPPED") stream = self._stream_name_to_instance[stream_name] + stream.cursor.ensure_at_least_one_state_emitted() + yield from self._message_repository.consume_queue() self._logger.info(f"Finished syncing {stream.name}") - return stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE) + self._streams_done.add(stream_name) + yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE) def _stop_streams(self) -> Iterable[AirbyteMessage]: self._thread_pool_manager.shutdown() - for stream_name, partitions in self._streams_to_partitions.items(): + for stream_name in self._streams_to_running_partitions.keys(): stream = self._stream_name_to_instance[stream_name] - if not all([p.is_closed() for p in partitions]): + if not self._is_stream_done(stream_name): self._logger.info(f"Marking stream {stream.name} as STOPPED") self._logger.info(f"Finished syncing {stream.name}") yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.INCOMPLETE) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py index b5439b230230..f7d65d31aca7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py @@ -25,7 +25,7 @@ class ConcurrentSource: A Source that reads data from multiple AbstractStreams concurrently. It does so by submitting partition generation, and partition read tasks to a thread pool. The tasks asynchronously add their output to a shared queue. - The read is done when all partitions for all streams were generated and read. + The read is done when all partitions for all streams w ere generated and read. """ DEFAULT_TIMEOUT_SECONDS = 900 @@ -39,6 +39,9 @@ def create( message_repository: MessageRepository, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, ) -> "ConcurrentSource": + is_single_threaded = initial_number_of_partitions_to_generate == 1 and num_workers == 1 + too_many_generator = not is_single_threaded and initial_number_of_partitions_to_generate >= num_workers + assert not too_many_generator, "It is required to have more workers than threads generating partitions" threadpool = ThreadPoolManager( concurrent.futures.ThreadPoolExecutor(max_workers=num_workers, thread_name_prefix="workerpool"), logger, @@ -82,10 +85,14 @@ def read( if not stream_instances_to_read_from: return - queue: Queue[QueueItem] = Queue() + # We set a maxsize to for the main thread to process record items when the queue size grows. This assumes that there are less + # threads generating partitions that than are max number of workers. If it weren't the case, we could have threads only generating + # partitions which would fill the queue. This number is arbitrarily set to 10_000 but will probably need to be changed given more + # information and might even need to be configurable depending on the source + queue: Queue[QueueItem] = Queue(maxsize=10_000) concurrent_stream_processor = ConcurrentReadProcessor( stream_instances_to_read_from, - PartitionEnqueuer(queue), + PartitionEnqueuer(queue, self._threadpool), self._threadpool, self._logger, self._slice_logger, @@ -115,7 +122,12 @@ def _consume_from_queue( queue: Queue[QueueItem], concurrent_stream_processor: ConcurrentReadProcessor, ) -> Iterable[AirbyteMessage]: - while airbyte_message_or_record_or_exception := queue.get(block=True, timeout=self._timeout_seconds): + while airbyte_message_or_record_or_exception := queue.get(): + try: + self._threadpool.shutdown_if_exception() + except Exception as exception: + concurrent_stream_processor.on_exception(exception) + yield from self._handle_item( airbyte_message_or_record_or_exception, concurrent_stream_processor, @@ -132,10 +144,8 @@ def _handle_item( # handle queue item and call the appropriate handler depending on the type of the queue item if isinstance(queue_item, Exception): yield from concurrent_stream_processor.on_exception(queue_item) - elif isinstance(queue_item, PartitionGenerationCompletedSentinel): yield from concurrent_stream_processor.on_partition_generation_completed(queue_item) - elif isinstance(queue_item, Partition): concurrent_stream_processor.on_partition(queue_item) elif isinstance(queue_item, PartitionCompleteSentinel): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py index 8e2ea80b79ae..6c3b8aa70efb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py @@ -10,7 +10,7 @@ from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream -from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade class ConcurrentSourceAdapter(AbstractSource, ABC): @@ -58,6 +58,6 @@ def _select_abstract_streams(self, config: Mapping[str, Any], configured_catalog f"The stream {configured_stream.stream.name} no longer exists in the configuration. " f"Refresh the schema in replication settings and remove this stream from future sync attempts." ) - if isinstance(stream_instance, StreamFacade): - abstract_streams.append(stream_instance._abstract_stream) + if isinstance(stream_instance, AbstractStreamFacade): + abstract_streams.append(stream_instance.get_underlying_stream()) return abstract_streams diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py index 6c351850e62d..b6643042b24c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py @@ -1,6 +1,8 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any + from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream @@ -15,3 +17,8 @@ def __init__(self, stream: AbstractStream): :param stream: The stream that was processed """ self.stream = stream + + def __eq__(self, other: Any) -> bool: + if isinstance(other, PartitionGenerationCompletedSentinel): + return self.stream == other.stream + return False diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py index 0c269cf0b3ee..560989af0a6c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py @@ -2,9 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import logging -import time +import threading from concurrent.futures import Future, ThreadPoolExecutor -from typing import Any, Callable, List +from typing import Any, Callable, List, Optional class ThreadPoolManager: @@ -12,7 +12,6 @@ class ThreadPoolManager: Wrapper to abstract away the threadpool and the logic to wait for pending tasks to be completed. """ - DEFAULT_SLEEP_TIME = 0.1 DEFAULT_MAX_QUEUE_SIZE = 10_000 def __init__( @@ -20,53 +19,57 @@ def __init__( threadpool: ThreadPoolExecutor, logger: logging.Logger, max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE, - sleep_time: float = DEFAULT_SLEEP_TIME, ): """ :param threadpool: The threadpool to use :param logger: The logger to use :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time - :param sleep_time: How long to sleep if there are too many pending tasks """ self._threadpool = threadpool self._logger = logger self._max_concurrent_tasks = max_concurrent_tasks - self._sleep_time = sleep_time self._futures: List[Future[Any]] = [] + self._lock = threading.Lock() + self._most_recently_seen_exception: Optional[Exception] = None + + self._logging_threshold = max_concurrent_tasks * 2 + + def prune_to_validate_has_reached_futures_limit(self) -> bool: + self._prune_futures(self._futures) + if len(self._futures) > self._logging_threshold: + self._logger.warning(f"ThreadPoolManager: The list of futures is getting bigger than expected ({len(self._futures)})") + return len(self._futures) >= self._max_concurrent_tasks def submit(self, function: Callable[..., Any], *args: Any) -> None: - # Submit a task to the threadpool, waiting if there are too many pending tasks - self._wait_while_too_many_pending_futures(self._futures) self._futures.append(self._threadpool.submit(function, *args)) - def _wait_while_too_many_pending_futures(self, futures: List[Future[Any]]) -> None: - # Wait until the number of pending tasks is < self._max_concurrent_tasks - while True: - self._prune_futures(futures) - if len(futures) < self._max_concurrent_tasks: - break - self._logger.info("Main thread is sleeping because the task queue is full...") - time.sleep(self._sleep_time) - def _prune_futures(self, futures: List[Future[Any]]) -> None: """ Take a list in input and remove the futures that are completed. If a future has an exception, it'll raise and kill the stream operation. - Pruning this list safely relies on the assumptions that only the main thread can modify the list of futures. + We are using a lock here as without it, the algorithm would not be thread safe """ - if len(futures) < self._max_concurrent_tasks: - return - - for index in reversed(range(len(futures))): - future = futures[index] - optional_exception = future.exception() - if optional_exception: - exception = RuntimeError(f"Failed reading with error: {optional_exception}") - self._stop_and_raise_exception(exception) + with self._lock: + if len(futures) < self._max_concurrent_tasks: + return - if future.done(): - futures.pop(index) + for index in reversed(range(len(futures))): + future = futures[index] + + if future.done(): + # Only call future.exception() if the future is known to be done because it will block until the future is done. + # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Future.exception + optional_exception = future.exception() + if optional_exception: + # Exception handling should be done in the main thread. Hence, we only store the exception and expect the main + # thread to call raise_if_exception + # We do not expect this error to happen. The futures created during concurrent syncs should catch the exception and + # push it to the queue. If this exception occurs, please review the futures and how they handle exceptions. + self._most_recently_seen_exception = RuntimeError( + f"Failed processing a future: {optional_exception}. Please contact the Airbyte team." + ) + futures.pop(index) def shutdown(self) -> None: self._threadpool.shutdown(wait=False, cancel_futures=True) @@ -74,12 +77,21 @@ def shutdown(self) -> None: def is_done(self) -> bool: return all([f.done() for f in self._futures]) + def shutdown_if_exception(self) -> None: + """ + This method will raise if there is an exception so that the caller can use it. + """ + if self._most_recently_seen_exception: + self._stop_and_raise_exception(self._most_recently_seen_exception) + def check_for_errors_and_shutdown(self) -> None: """ Check if any of the futures have an exception, and raise it if so. If all futures are done, shutdown the threadpool. If the futures are not done, raise an exception. :return: """ + self.shutdown_if_exception() + exceptions_from_futures = [f for f in [future.exception() for future in self._futures] if f is not None] if exceptions_from_futures: exception = RuntimeError(f"Failed reading with errors: {exceptions_from_futures}") diff --git a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py index 575756f5a706..62f979f58540 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py @@ -5,7 +5,15 @@ import copy from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStream, + AirbyteStreamState, + StreamDescriptor, +) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.streams import Stream from pydantic import Extra @@ -29,7 +37,9 @@ class ConnectorStateManager: """ def __init__( - self, stream_instance_map: Mapping[str, Stream], state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None + self, + stream_instance_map: Mapping[str, Union[Stream, AirbyteStream]], + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, ): shared_state, per_stream_states = self._extract_from_state_message(state, stream_instance_map) @@ -97,7 +107,9 @@ def create_state_message(self, stream_name: str, namespace: Optional[str], send_ @classmethod def _extract_from_state_message( - cls, state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]], stream_instance_map: Mapping[str, Stream] + cls, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]], + stream_instance_map: Mapping[str, Union[Stream, AirbyteStream]], ) -> Tuple[Optional[AirbyteStateBlob], MutableMapping[HashableStreamDescriptor, Optional[AirbyteStateBlob]]]: """ Takes an incoming list of state messages or the legacy state format and extracts state attributes according to type @@ -149,7 +161,7 @@ def _extract_from_state_message( @staticmethod def _create_descriptor_to_stream_state_mapping( - state: MutableMapping[str, Any], stream_to_instance_map: Mapping[str, Stream] + state: MutableMapping[str, Any], stream_to_instance_map: Mapping[str, Union[Stream, AirbyteStream]] ) -> MutableMapping[HashableStreamDescriptor, Optional[AirbyteStateBlob]]: """ Takes incoming state received in the legacy format and transforms it into a mapping of StreamDescriptor to AirbyteStreamState diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index 4e83c570be6e..d858677b6324 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -46,8 +46,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut refresh_token: Optional[Union[InterpolatedString, str]] = None scopes: Optional[List[str]] = None token_expiry_date: Optional[Union[InterpolatedString, str]] = None - _token_expiry_date: pendulum.DateTime = field(init=False, repr=False, default=None) - token_expiry_date_format: str = None + _token_expiry_date: Optional[pendulum.DateTime] = field(init=False, repr=False, default=None) + token_expiry_date_format: Optional[str] = None token_expiry_is_time_of_expiration: bool = False access_token_name: Union[InterpolatedString, str] = "access_token" expires_in_name: Union[InterpolatedString, str] = "expires_in" @@ -55,65 +55,79 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut grant_type: Union[InterpolatedString, str] = "refresh_token" message_repository: MessageRepository = NoopMessageRepository() - def __post_init__(self, parameters: Mapping[str, Any]): - self.token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, parameters=parameters) - self.client_id = InterpolatedString.create(self.client_id, parameters=parameters) - self.client_secret = InterpolatedString.create(self.client_secret, parameters=parameters) + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__init__() + self._token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, parameters=parameters) + self._client_id = InterpolatedString.create(self.client_id, parameters=parameters) + self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters) if self.refresh_token is not None: - self.refresh_token = InterpolatedString.create(self.refresh_token, parameters=parameters) + self._refresh_token = InterpolatedString.create(self.refresh_token, parameters=parameters) + else: + self._refresh_token = None self.access_token_name = InterpolatedString.create(self.access_token_name, parameters=parameters) self.expires_in_name = InterpolatedString.create(self.expires_in_name, parameters=parameters) self.grant_type = InterpolatedString.create(self.grant_type, parameters=parameters) self._refresh_request_body = InterpolatedMapping(self.refresh_request_body or {}, parameters=parameters) - self._token_expiry_date = ( - pendulum.parse(InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(self.config)) + self._token_expiry_date: pendulum.DateTime = ( + pendulum.parse(InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(self.config)) # type: ignore # pendulum.parse returns a datetime in this context if self.token_expiry_date - else pendulum.now().subtract(days=1) + else pendulum.now().subtract(days=1) # type: ignore # substract does not have type hints ) - self._access_token = None + self._access_token: Optional[str] = None # access_token is initialized by a setter - if self.get_grant_type() == "refresh_token" and self.refresh_token is None: + if self.get_grant_type() == "refresh_token" and self._refresh_token is None: raise ValueError("OAuthAuthenticator needs a refresh_token parameter if grant_type is set to `refresh_token`") def get_token_refresh_endpoint(self) -> str: - return self.token_refresh_endpoint.eval(self.config) + refresh_token: str = self._token_refresh_endpoint.eval(self.config) + if not refresh_token: + raise ValueError("OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter") + return refresh_token def get_client_id(self) -> str: - return self.client_id.eval(self.config) + client_id: str = self._client_id.eval(self.config) + if not client_id: + raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter") + return client_id def get_client_secret(self) -> str: - return self.client_secret.eval(self.config) + client_secret: str = self._client_secret.eval(self.config) + if not client_secret: + raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter") + return client_secret def get_refresh_token(self) -> Optional[str]: - return None if self.refresh_token is None else self.refresh_token.eval(self.config) + return None if self._refresh_token is None else self._refresh_token.eval(self.config) - def get_scopes(self) -> [str]: - return self.scopes + def get_scopes(self) -> List[str]: + return self.scopes or [] - def get_access_token_name(self) -> InterpolatedString: - return self.access_token_name.eval(self.config) + def get_access_token_name(self) -> str: + return self.access_token_name.eval(self.config) # type: ignore # eval returns a string in this context - def get_expires_in_name(self) -> InterpolatedString: - return self.expires_in_name.eval(self.config) + def get_expires_in_name(self) -> str: + return self.expires_in_name.eval(self.config) # type: ignore # eval returns a string in this context - def get_grant_type(self) -> InterpolatedString: - return self.grant_type.eval(self.config) + def get_grant_type(self) -> str: + return self.grant_type.eval(self.config) # type: ignore # eval returns a string in this context def get_refresh_request_body(self) -> Mapping[str, Any]: - return self._refresh_request_body.eval(self.config) + return self._refresh_request_body.eval(self.config) # type: ignore # eval should return a Mapping in this context def get_token_expiry_date(self) -> pendulum.DateTime: - return self._token_expiry_date + return self._token_expiry_date # type: ignore # _token_expiry_date is a pendulum.DateTime. It is never None despite what mypy thinks - def set_token_expiry_date(self, value: Union[str, int]): + def set_token_expiry_date(self, value: Union[str, int]) -> None: self._token_expiry_date = self._parse_token_expiration_date(value) @property def access_token(self) -> str: + if self._access_token is None: + raise ValueError("access_token is not set") return self._access_token @access_token.setter - def access_token(self, value: str): + def access_token(self, value: str) -> None: self._access_token = value @property @@ -130,5 +144,5 @@ class DeclarativeSingleUseRefreshTokenOauth2Authenticator(SingleUseRefreshTokenO Declarative version of SingleUseRefreshTokenOauth2Authenticator which can be used in declarative connectors. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py new file mode 100644 index 000000000000..6a9d6128706b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping + +import dpath +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator + + +@dataclass +class SelectiveAuthenticator(DeclarativeAuthenticator): + """Authenticator that selects concrete implementation based on specific config value.""" + + config: Mapping[str, Any] + authenticators: Mapping[str, DeclarativeAuthenticator] + authenticator_selection_path: List[str] + + # returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator" + def __new__( # type: ignore[misc] + cls, + config: Mapping[str, Any], + authenticators: Mapping[str, DeclarativeAuthenticator], + authenticator_selection_path: List[str], + *arg: Any, + **kwargs: Any, + ) -> DeclarativeAuthenticator: + try: + selected_key = str(dpath.util.get(config, authenticator_selection_path)) + except KeyError as err: + raise ValueError("The path from `authenticator_selection_path` is not found in the config.") from err + + try: + return authenticators[selected_key] + except KeyError as err: + raise ValueError(f"The authenticator `{selected_key}` is not found.") from err diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 8d4af292f68a..69cf2f8d1bbb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -224,6 +224,49 @@ definitions: $parameters: type: object additionalProperties: true + SelectiveAuthenticator: + title: Selective Authenticator + description: Authenticator that selects concrete authenticator based on config property. + type: object + additionalProperties: true + required: + - type + - authenticators + - authenticator_selection_path + properties: + type: + type: string + enum: [SelectiveAuthenticator] + authenticator_selection_path: + title: Authenticator Selection Path + description: Path of the field in config with selected authenticator name + type: array + items: + type: string + examples: + - ["auth"] + - ["auth", "type"] + authenticators: + title: Authenticators + description: Authenticators to select from. + type: object + additionalProperties: + anyOf: + - "$ref": "#/definitions/ApiKeyAuthenticator" + - "$ref": "#/definitions/BasicHttpAuthenticator" + - "$ref": "#/definitions/BearerAuthenticator" + - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/OAuthAuthenticator" + - "$ref": "#/definitions/NoAuth" + - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + examples: + - authenticators: + token: "#/definitions/ApiKeyAuthenticator" + oauth: "#/definitions/OAuthAuthenticator" + $parameters: + type: object + additionalProperties: true CheckStream: title: Streams to Check description: Defines the streams to try reading when running a check operation. @@ -473,6 +516,27 @@ definitions: $parameters: type: object additionalProperties: true + CustomRecordFilter: + title: Custom Record Filter + description: Record filter component whose behavior is derived from a custom code implementation of the connector. + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + type: string + enum: [CustomRecordFilter] + class_name: + title: Class Name + description: Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_..`. + type: string + examples: + - "source_railz.components.MyCustomCustomRecordFilter" + $parameters: + type: object + additionalProperties: true CustomRequester: title: Custom Requester description: Requester component whose behavior is derived from a custom code implementation of the connector. @@ -1149,6 +1213,7 @@ definitions: - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + - "$ref": "#/definitions/SelectiveAuthenticator" error_handler: title: Error Handler description: Error handler component that defines how to handle errors. @@ -1159,12 +1224,10 @@ definitions: http_method: title: HTTP Method description: The HTTP method used to fetch data from the source (can be GET or POST). - anyOf: - - type: string - - type: string - enum: - - GET - - POST + type: string + enum: + - GET + - POST default: GET examples: - GET @@ -1777,10 +1840,25 @@ definitions: record_filter: title: Record Filter description: Responsible for filtering records to be emitted by the Source. - "$ref": "#/definitions/RecordFilter" + anyOf: + - "$ref": "#/definitions/CustomRecordFilter" + - "$ref": "#/definitions/RecordFilter" + schema_normalization: + "$ref": "#/definitions/SchemaNormalization" + default: None $parameters: type: object additionalProperties: true + SchemaNormalization: + title: Schema Normalization + description: Responsible for normalization according to the schema. + type: string + enum: + - None + - Default + examples: + - None + - Default RemoveFields: title: Remove Fields description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown. @@ -1792,6 +1870,19 @@ definitions: type: type: string enum: [RemoveFields] + condition: + description: The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True., + type: string + default: "" + interpolation_context: + - config + - property + - parameters + examples: + - "{{ property|string == '' }}" + - "{{ property is integer }}" + - "{{ property|length > 5 }}" + - "{{ property == 'some_string_to_match' }}" field_pointers: title: Field Paths description: Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove. @@ -1830,6 +1921,9 @@ definitions: type: string examples: - segment_id + interpolation_context: + - config + - parameters inject_into: title: Inject Into description: Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated. @@ -1933,6 +2027,10 @@ definitions: anyOf: - "$ref": "#/definitions/DefaultPaginator" - "$ref": "#/definitions/NoPagination" + ignore_stream_slicer_parameters_on_paginated_requests: + description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. + type: boolean + default: false partition_router: title: Partition Router description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing. @@ -2082,6 +2180,13 @@ interpolation: examples: - start_date: 2010-01-01 api_key: "*****" + - title: parameters + description: Additional runtime parameters, to be used for string interpolation. Parameters can be passed down from a parent component to its subcomponents using the $parameters key. This can be used to avoid repetitions. + type: object + examples: + - path: "automations" + data_export_path: "automations" + cursor_field: "updated_at" - title: headers description: The HTTP headers from the last response received from the API. The object's keys are the header names from the response. type: object diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index 56d92dfc5639..f74ed377c4ab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -101,7 +101,7 @@ def read_records( """ :param: stream_state We knowingly avoid using stream_state as we want cursors to manage their own state. """ - yield from self.retriever.read_records(stream_slice) + yield from self.retriever.read_records(self.get_json_schema(), stream_slice) def get_json_schema(self) -> Mapping[str, Any]: # type: ignore """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py index 1b52cb03ba4f..0da7125868f1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py @@ -22,6 +22,7 @@ def select_records( self, response: requests.Response, stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: @@ -29,6 +30,7 @@ def select_records( Selects records from the response :param response: The response to select the records from :param stream_state: The stream state + :param records_schema: json schema of records to return :param stream_slice: The stream slice :param next_page_token: The paginator token :return: List of Records selected from the response diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index d08068a952e0..33ad173d5484 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -9,8 +9,15 @@ from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter +from airbyte_cdk.sources.declarative.models import SchemaNormalization from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer + +SCHEMA_TRANSFORMER_TYPE_MAPPING = { + SchemaNormalization.None_: TransformConfig.NoTransform, + SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization, +} @dataclass @@ -21,6 +28,7 @@ class RecordSelector(HttpSelector): Attributes: extractor (RecordExtractor): The record extractor responsible for extracting records from a response + schema_normalization (TypeTransformer): The record normalizer responsible for casting record values to stream schema types record_filter (RecordFilter): The record filter responsible for filtering extracted records transformations (List[RecordTransformation]): The transformations to be done on the records """ @@ -28,6 +36,7 @@ class RecordSelector(HttpSelector): extractor: RecordExtractor config: Config parameters: InitVar[Mapping[str, Any]] + schema_normalization: TypeTransformer record_filter: Optional[RecordFilter] = None transformations: List[RecordTransformation] = field(default_factory=lambda: []) @@ -38,14 +47,31 @@ def select_records( self, response: requests.Response, stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: + """ + Selects records from the response + :param response: The response to select the records from + :param stream_state: The stream state + :param records_schema: json schema of records to return + :param stream_slice: The stream slice + :param next_page_token: The paginator token + :return: List of Records selected from the response + """ all_data = self.extractor.extract_records(response) filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token) self._transform(filtered_data, stream_state, stream_slice) + self._normalize_by_schema(filtered_data, schema=records_schema) return [Record(data, stream_slice) for data in filtered_data] + def _normalize_by_schema(self, records: List[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]) -> List[Mapping[str, Any]]: + if schema: + # record has type Mapping[str, Any], but dict[str, Any] expected + return [self.schema_normalization.transform(record, schema) for record in records] # type: ignore + return records + def _filter( self, records: List[Mapping[str, Any]], @@ -67,4 +93,5 @@ def _transform( ) -> None: for record in records: for transformation in self.transformations: - transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice) + # record has type Mapping[str, Any], but Record expected + transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice) # type: ignore diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py index 685f0b7e6876..e2a5f27d1ef3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py @@ -247,9 +247,11 @@ def request_kwargs(self) -> Mapping[str, Any]: def _get_request_options(self, option_type: RequestOptionType, stream_slice: StreamSlice): options = {} if self.start_time_option and self.start_time_option.inject_into == option_type: - options[self.start_time_option.field_name] = stream_slice.get(self.partition_field_start.eval(self.config)) + options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( + self.partition_field_start.eval(self.config) + ) if self.end_time_option and self.end_time_option.inject_into == option_type: - options[self.end_time_option.field_name] = stream_slice.get(self.partition_field_end.eval(self.config)) + options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(self.partition_field_end.eval(self.config)) return options def should_be_synced(self, record: Record) -> bool: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 9981f26169e8..f70d3aef7523 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -11,142 +11,156 @@ class AuthFlowType(Enum): - oauth2_0 = 'oauth2.0' - oauth1_0 = 'oauth1.0' + oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class BasicHttpAuthenticator(BaseModel): - type: Literal['BasicHttpAuthenticator'] + type: Literal["BasicHttpAuthenticator"] username: str = Field( ..., - description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', + description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", + examples=["{{ config['password'] }}", ""], + title="Password", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class BearerAuthenticator(BaseModel): - type: Literal['BearerAuthenticator'] + type: Literal["BearerAuthenticator"] api_token: str = Field( ..., - description='Token to inject as request header for authenticating with the API.', + description="Token to inject as request header for authenticating with the API.", examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], - title='Bearer Token', + title="Bearer Token", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CheckStream(BaseModel): - type: Literal['CheckStream'] + type: Literal["CheckStream"] stream_names: List[str] = Field( ..., - description='Names of the streams to try reading from when running a check operation.', - examples=[['users'], ['users', 'contacts']], - title='Stream Names', + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", ) class ConstantBackoffStrategy(BaseModel): - type: Literal['ConstantBackoffStrategy'] + type: Literal["ConstantBackoffStrategy"] backoff_time_in_seconds: Union[float, str] = Field( ..., - description='Backoff time in seconds.', + description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], - title='Backoff Time', + title="Backoff Time", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomAuthenticator'] + type: Literal["CustomAuthenticator"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', - examples=['source_railz.components.ShortLivedTokenAuthenticator'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", + examples=["source_railz.components.ShortLivedTokenAuthenticator"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomBackoffStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomBackoffStrategy'] + type: Literal["CustomBackoffStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomBackoffStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomBackoffStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomErrorHandler(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomErrorHandler'] + type: Literal["CustomErrorHandler"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', - examples=['source_railz.components.MyCustomErrorHandler'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.", + examples=["source_railz.components.MyCustomErrorHandler"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomIncrementalSync(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomIncrementalSync'] + type: Literal["CustomIncrementalSync"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', - examples=['source_railz.components.MyCustomIncrementalSync'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.", + examples=["source_railz.components.MyCustomIncrementalSync"], + title="Class Name", ) cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync.', + description="The location of the value on a record that will be used as a bookmark during sync.", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPaginationStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPaginationStrategy'] + type: Literal["CustomPaginationStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomPaginationStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomPaginationStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRecordExtractor(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRecordExtractor'] + type: Literal["CustomRecordExtractor"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], + description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class CustomRecordFilter(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomRecordFilter'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom record filtering. The format is `source_..`.', + examples=['source_railz.components.MyCustomRecordFilter'], title='Class Name', ) parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') @@ -156,456 +170,465 @@ class CustomRequester(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRequester'] + type: Literal["CustomRequester"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRetriever(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRetriever'] + type: Literal["CustomRetriever"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRetriever'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRetriever"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPartitionRouter(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPartitionRouter'] + type: Literal["CustomPartitionRouter"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', - examples=['source_railz.components.MyCustomPartitionRouter'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.", + examples=["source_railz.components.MyCustomPartitionRouter"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomTransformation(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomTransformation'] + type: Literal["CustomTransformation"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', - examples=['source_railz.components.MyCustomTransformation'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.", + examples=["source_railz.components.MyCustomTransformation"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RefreshTokenUpdater(BaseModel): refresh_token_name: Optional[str] = Field( - 'refresh_token', - description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', - examples=['refresh_token'], - title='Refresh Token Property Name', + "refresh_token", + description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", + examples=["refresh_token"], + title="Refresh Token Property Name", ) access_token_config_path: Optional[List[str]] = Field( - ['credentials', 'access_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'access_token'], ['access_token']], - title='Config Path To Access Token', + ["credentials", "access_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", ) refresh_token_config_path: Optional[List[str]] = Field( - ['credentials', 'refresh_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'refresh_token'], ['refresh_token']], - title='Config Path To Refresh Token', + ["credentials", "refresh_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", ) token_expiry_date_config_path: Optional[List[str]] = Field( - ['credentials', 'token_expiry_date'], - description='Config path to the expiry date. Make sure actually exists in the config.', - examples=[['credentials', 'token_expiry_date']], - title='Config Path To Expiry Date', + ["credentials", "token_expiry_date"], + description="Config path to the expiry date. Make sure actually exists in the config.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", ) class OAuthAuthenticator(BaseModel): - type: Literal['OAuthAuthenticator'] + type: Literal["OAuthAuthenticator"] client_id: str = Field( ..., - description='The OAuth client ID. Fill it in the user inputs.', + description="The OAuth client ID. Fill it in the user inputs.", examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], - title='Client ID', + title="Client ID", ) client_secret: str = Field( ..., - description='The OAuth client secret. Fill it in the user inputs.', + description="The OAuth client secret. Fill it in the user inputs.", examples=[ "{{ config['client_secret }}", "{{ config['credentials']['client_secret }}", ], - title='Client Secret', + title="Client Secret", ) refresh_token: Optional[str] = Field( None, - description='Credential artifact used to get a new access token.', + description="Credential artifact used to get a new access token.", examples=[ "{{ config['refresh_token'] }}", "{{ config['credentials]['refresh_token'] }}", ], - title='Refresh Token', + title="Refresh Token", ) token_refresh_endpoint: str = Field( ..., - description='The full URL to call to obtain a new access token.', - examples=['https://connect.squareup.com/oauth2/token'], - title='Token Refresh Endpoint', + description="The full URL to call to obtain a new access token.", + examples=["https://connect.squareup.com/oauth2/token"], + title="Token Refresh Endpoint", ) access_token_name: Optional[str] = Field( - 'access_token', - description='The name of the property which contains the access token in the response from the token refresh endpoint.', - examples=['access_token'], - title='Access Token Property Name', + "access_token", + description="The name of the property which contains the access token in the response from the token refresh endpoint.", + examples=["access_token"], + title="Access Token Property Name", ) expires_in_name: Optional[str] = Field( - 'expires_in', - description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', - examples=['expires_in'], - title='Token Expiry Property Name', + "expires_in", + description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", + examples=["expires_in"], + title="Token Expiry Property Name", ) grant_type: Optional[str] = Field( - 'refresh_token', - description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', - examples=['refresh_token', 'client_credentials'], - title='Grant Type', + "refresh_token", + description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", + examples=["refresh_token", "client_credentials"], + title="Grant Type", ) refresh_request_body: Optional[Dict[str, Any]] = Field( None, - description='Body of the request sent to get a new access token.', + description="Body of the request sent to get a new access token.", examples=[ { - 'applicationId': "{{ config['application_id'] }}", - 'applicationSecret': "{{ config['application_secret'] }}", - 'token': "{{ config['token'] }}", + "applicationId": "{{ config['application_id'] }}", + "applicationSecret": "{{ config['application_secret'] }}", + "token": "{{ config['token'] }}", } ], - title='Refresh Request Body', + title="Refresh Request Body", ) scopes: Optional[List[str]] = Field( None, - description='List of scopes that should be granted to the access token.', - examples=[ - ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] - ], - title='Scopes', + description="List of scopes that should be granted to the access token.", + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + title="Scopes", ) token_expiry_date: Optional[str] = Field( None, - description='The access token expiry date.', - examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], - title='Token Expiry Date', + description="The access token expiry date.", + examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], + title="Token Expiry Date", ) token_expiry_date_format: Optional[str] = Field( None, - description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', - examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], - title='Token Expiry Date Format', + description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", + examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], + title="Token Expiry Date Format", ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', - title='Token Updater', + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", + title="Token Updater", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ExponentialBackoffStrategy(BaseModel): - type: Literal['ExponentialBackoffStrategy'] + type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( 5, - description='Multiplicative constant applied on each retry.', - examples=[5, 5.5, '10'], - title='Factor', + description="Multiplicative constant applied on each retry.", + examples=[5, 5.5, "10"], + title="Factor", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestBearerAuthenticator(BaseModel): - type: Literal['Bearer'] + type: Literal["Bearer"] -class HttpMethodEnum(Enum): - GET = 'GET' - POST = 'POST' +class HttpMethod(Enum): + GET = "GET" + POST = "POST" class Action(Enum): - SUCCESS = 'SUCCESS' - FAIL = 'FAIL' - RETRY = 'RETRY' - IGNORE = 'IGNORE' + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" class HttpResponseFilter(BaseModel): - type: Literal['HttpResponseFilter'] + type: Literal["HttpResponseFilter"] action: Action = Field( ..., - description='Action to execute if a response matches the filter.', - examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'], - title='Action', + description="Action to execute if a response matches the filter.", + examples=["SUCCESS", "FAIL", "RETRY", "IGNORE"], + title="Action", ) error_message: Optional[str] = Field( None, - description='Error Message to display if the response matches the filter.', - title='Error Message', + description="Error Message to display if the response matches the filter.", + title="Error Message", ) error_message_contains: Optional[str] = Field( None, - description='Match the response if its error message contains the substring.', - example=['This API operation is not enabled for this site'], - title='Error Message Substring', + description="Match the response if its error message contains the substring.", + example=["This API operation is not enabled for this site"], + title="Error Message Substring", ) http_codes: Optional[List[int]] = Field( None, - description='Match the response if its HTTP code is included in this list.', + description="Match the response if its HTTP code is included in this list.", examples=[[420, 429], [500]], - title='HTTP Codes', + title="HTTP Codes", ) predicate: Optional[str] = Field( None, - description='Match the response if the predicate evaluates to true.', + description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], - title='Predicate', + title="Predicate", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class InlineSchemaLoader(BaseModel): - type: Literal['InlineSchemaLoader'] + type: Literal["InlineSchemaLoader"] schema_: Optional[Dict[str, Any]] = Field( None, - alias='schema', + alias="schema", description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', - title='Schema', + title="Schema", ) class JsonFileSchemaLoader(BaseModel): - type: Literal['JsonFileSchemaLoader'] + type: Literal["JsonFileSchemaLoader"] file_path: Optional[str] = Field( None, description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", - example=['./schemas/users.json'], - title='File Path', + example=["./schemas/users.json"], + title="File Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class JsonDecoder(BaseModel): - type: Literal['JsonDecoder'] + type: Literal["JsonDecoder"] class MinMaxDatetime(BaseModel): - type: Literal['MinMaxDatetime'] + type: Literal["MinMaxDatetime"] datetime: str = Field( ..., - description='Datetime value.', - examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], - title='Datetime', + description="Datetime value.", + examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], + title="Datetime", ) datetime_format: Optional[str] = Field( - '', + "", description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], - title='Datetime Format', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], + title="Datetime Format", ) max_datetime: Optional[str] = Field( None, - description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2021-01-01T00:00:00Z', '2021-01-01'], - title='Max Datetime', + description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2021-01-01T00:00:00Z", "2021-01-01"], + title="Max Datetime", ) min_datetime: Optional[str] = Field( None, - description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2010-01-01T00:00:00Z', '2010-01-01'], - title='Min Datetime', + description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2010-01-01T00:00:00Z", "2010-01-01"], + title="Min Datetime", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoAuth(BaseModel): - type: Literal['NoAuth'] - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + type: Literal["NoAuth"] + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoPagination(BaseModel): - type: Literal['NoPagination'] + type: Literal["NoPagination"] class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[ - Dict[str, Any] - ] = Field( + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", examples=[ - {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, { - 'app_id': { - 'type': 'string', - 'path_in_connector_config': ['info', 'app_id'], + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], } }, ], - title='OAuth user input', + title="OAuth user input", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", examples=[ { - 'refresh_token': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'refresh_token'], + "refresh_token": { + "type": "string,", + "path_in_connector_config": ["credentials", "refresh_token"], } } ], - title='OAuth output specification', + title="OAuth output specification", ) complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, - description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', - examples=[ - {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} - ], - title='OAuth input specification', + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", examples=[ { - 'client_id': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_id'], + "client_id": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_id"], }, - 'client_secret': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_secret'], + "client_secret": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_secret"], }, } ], - title='OAuth server output specification', + title="OAuth server output specification", ) class OffsetIncrement(BaseModel): - type: Literal['OffsetIncrement'] + type: Literal["OffsetIncrement"] page_size: Optional[Union[int, str]] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100, "{{ config['page_size'] }}"], - title='Limit', + title="Limit", ) inject_on_first_request: Optional[bool] = Field( False, - description='Using the `offset` with value `0` during the first request', - title='Inject Offset', + description="Using the `offset` with value `0` during the first request", + title="Inject Offset", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PageIncrement(BaseModel): - type: Literal['PageIncrement'] + type: Literal["PageIncrement"] page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', - examples=[100, '100'], - title='Page Size', + description="The number of records to include in each pages.", + examples=[100, "100"], + title="Page Size", ) start_from_page: Optional[int] = Field( 0, - description='Index of the first page to request.', + description="Index of the first page to request.", examples=[0, 1], - title='Start From Page', + title="Start From Page", ) inject_on_first_request: Optional[bool] = Field( False, - description='Using the `page number` with value defined by `start_from_page` during the first request', - title='Inject Page Number', + description="Using the `page number` with value defined by `start_from_page` during the first request", + title="Inject Page Number", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PrimaryKey(BaseModel): __root__: Union[str, List[str], List[List[str]]] = Field( ..., - description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', - examples=['id', ['code', 'type']], - title='Primary Key', + description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", + examples=["id", ["code", "type"]], + title="Primary Key", ) class RecordFilter(BaseModel): - type: Literal['RecordFilter'] + type: Literal["RecordFilter"] condition: Optional[str] = Field( - '', - description='The predicate to filter a record. Records will be removed if evaluated to False.', + "", + description="The predicate to filter a record. Records will be removed if evaluated to False.", examples=[ "{{ record['created_at'] >= stream_interval['start_time'] }}", "{{ record.status in ['active', 'expired'] }}", ], ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class SchemaNormalization(Enum): + None_ = "None" + Default = "Default" class RemoveFields(BaseModel): - type: Literal['RemoveFields'] + type: Literal["RemoveFields"] + condition: Optional[str] = Field( + "", + description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.", + examples=[ + "{{ property|string == '' }}", + "{{ property is integer }}", + "{{ property|length > 5 }}", + "{{ property == 'some_string_to_match' }}", + ], + ) field_pointers: List[List[str]] = Field( ..., - description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', - examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], - title='Field Paths', + description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", + examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], + title="Field Paths", ) class RequestPath(BaseModel): - type: Literal['RequestPath'] + type: Literal["RequestPath"] class InjectInto(Enum): - request_parameter = 'request_parameter' - header = 'header' - body_data = 'body_data' - body_json = 'body_json' + request_parameter = "request_parameter" + header = "header" + body_data = "body_data" + body_json = "body_json" class RequestOption(BaseModel): - type: Literal['RequestOption'] + type: Literal["RequestOption"] field_name: str = Field( ..., - description='Configures which key should be used in the location that the descriptor is being injected into', - examples=['segment_id'], - title='Request Option', + description="Configures which key should be used in the location that the descriptor is being injected into", + examples=["segment_id"], + title="Request Option", ) inject_into: InjectInto = Field( ..., - description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', - examples=['request_parameter', 'header', 'body_data', 'body_json'], - title='Inject Into', + description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", + examples=["request_parameter", "header", "body_data", "body_json"], + title="Inject Into", ) @@ -617,106 +640,106 @@ class Config: class LegacySessionTokenAuthenticator(BaseModel): - type: Literal['LegacySessionTokenAuthenticator'] + type: Literal["LegacySessionTokenAuthenticator"] header: str = Field( ..., - description='The name of the session token header that will be injected in the request', - examples=['X-Session'], - title='Session Request Header', + description="The name of the session token header that will be injected in the request", + examples=["X-Session"], + title="Session Request Header", ) login_url: str = Field( ..., - description='Path of the login URL (do not include the base URL)', - examples=['session'], - title='Login Path', + description="Path of the login URL (do not include the base URL)", + examples=["session"], + title="Login Path", ) session_token: Optional[str] = Field( None, - description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', + description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", example=["{{ config['session_token'] }}"], - title='Session Token', + title="Session Token", ) session_token_response_key: str = Field( ..., - description='Name of the key of the session token to be extracted from the response', - examples=['id'], - title='Response Token Response Key', + description="Name of the key of the session token to be extracted from the response", + examples=["id"], + title="Response Token Response Key", ) username: Optional[str] = Field( None, - description='Username used to authenticate and obtain a session token', + description="Username used to authenticate and obtain a session token", examples=[" {{ config['username'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='Password used to authenticate and obtain a session token', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="Password used to authenticate and obtain a session token", + examples=["{{ config['password'] }}", ""], + title="Password", ) validate_session_url: str = Field( ..., - description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', - examples=['user/current'], - title='Validate Session Path', + description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", + examples=["user/current"], + title="Validate Session Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ValueType(Enum): - string = 'string' - number = 'number' - integer = 'integer' - boolean = 'boolean' + string = "string" + number = "number" + integer = "integer" + boolean = "boolean" class WaitTimeFromHeader(BaseModel): - type: Literal['WaitTimeFromHeader'] + type: Literal["WaitTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['Retry-After'], - title='Response Header Name', + description="The name of the response header defining how long to wait before retrying.", + examples=["Retry-After"], + title="Response Header Name", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitUntilTimeFromHeader(BaseModel): - type: Literal['WaitUntilTimeFromHeader'] + type: Literal["WaitUntilTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['wait_time'], - title='Response Header', + description="The name of the response header defining how long to wait before retrying.", + examples=["wait_time"], + title="Response Header", ) min_wait: Optional[Union[float, str]] = Field( None, - description='Minimum time to wait before retrying.', - examples=[10, '60'], - title='Minimum Wait Time', + description="Minimum time to wait before retrying.", + examples=[10, "60"], + title="Minimum Wait Time", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddedFieldDefinition(BaseModel): - type: Literal['AddedFieldDefinition'] + type: Literal["AddedFieldDefinition"] path: List[str] = Field( ..., - description='List of strings defining the path where to add the value on the record.', - examples=[['segment_id'], ['metadata', 'segment_id']], - title='Path', + description="List of strings defining the path where to add the value on the record.", + examples=[["segment_id"], ["metadata", "segment_id"]], + title="Path", ) value: str = Field( ..., @@ -726,187 +749,185 @@ class AddedFieldDefinition(BaseModel): "{{ record['MetaData']['LastUpdatedTime'] }}", "{{ stream_partition['segment_id'] }}", ], - title='Value', + title="Value", ) value_type: Optional[ValueType] = Field( None, - description='Type of the value. If not specified, the type will be inferred from the value.', - title='Value Type', + description="Type of the value. If not specified, the type will be inferred from the value.", + title="Value Type", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddFields(BaseModel): - type: Literal['AddFields'] + type: Literal["AddFields"] fields: List[AddedFieldDefinition] = Field( ..., - description='List of transformations (path and corresponding value) that will be added to the record.', - title='Fields', + description="List of transformations (path and corresponding value) that will be added to the record.", + title="Fields", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ApiKeyAuthenticator(BaseModel): - type: Literal['ApiKeyAuthenticator'] + type: Literal["ApiKeyAuthenticator"] api_token: Optional[str] = Field( None, - description='The API key to inject in the request. Fill it in the user inputs.', + description="The API key to inject in the request. Fill it in the user inputs.", examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title='API Key', + title="API Key", ) header: Optional[str] = Field( None, - description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', - examples=['Authorization', 'Api-Token', 'X-Auth-Token'], - title='Header Name', + description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", + examples=["Authorization", "Api-Token", "X-Auth-Token"], + title="Header Name", ) inject_into: Optional[RequestOption] = Field( None, - description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', + description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlow(BaseModel): - auth_flow_type: Optional[AuthFlowType] = Field( - None, description='The type of auth to use', title='Auth flow type' - ) + auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") predicate_key: Optional[List[str]] = Field( None, - description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', - examples=[['credentials', 'auth_type']], - title='Predicate key', + description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + examples=[["credentials", "auth_type"]], + title="Predicate key", ) predicate_value: Optional[str] = Field( None, - description='Value of the predicate_key fields for the advanced auth to be applicable.', - examples=['Oauth'], - title='Predicate value', + description="Value of the predicate_key fields for the advanced auth to be applicable.", + examples=["Oauth"], + title="Predicate value", ) oauth_config_specification: Optional[OAuthConfigSpecification] = None class CursorPagination(BaseModel): - type: Literal['CursorPagination'] + type: Literal["CursorPagination"] cursor_value: str = Field( ..., - description='Value of the cursor defining the next page to fetch.', + description="Value of the cursor defining the next page to fetch.", examples=[ - '{{ headers.link.next.cursor }}', + "{{ headers.link.next.cursor }}", "{{ last_records[-1]['key'] }}", "{{ response['nextPage'] }}", ], - title='Cursor Value', + title="Cursor Value", ) page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100], - title='Page Size', + title="Page Size", ) stop_condition: Optional[str] = Field( None, - description='Template string evaluating when to stop paginating.', + description="Template string evaluating when to stop paginating.", examples=[ - '{{ response.data.has_more is false }}', + "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", ], - title='Stop Condition', + title="Stop Condition", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DatetimeBasedCursor(BaseModel): - type: Literal['DatetimeBasedCursor'] + type: Literal["DatetimeBasedCursor"] cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', - examples=['created_at', "{{ config['record_cursor'] }}"], - title='Cursor Field', + description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", + examples=["created_at", "{{ config['record_cursor'] }}"], + title="Cursor Field", ) datetime_format: str = Field( ..., - description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'], - title='Outgoing Datetime Format', + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"], + title="Outgoing Datetime Format", ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., - description='The datetime that determines the earliest record that should be synced.', - examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], - title='Start Datetime', + description="The datetime that determines the earliest record that should be synced.", + examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], + title="Start Datetime", ) cursor_datetime_formats: Optional[List[str]] = Field( None, - description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', - title='Cursor Datetime Formats', + description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", + title="Cursor Datetime Formats", ) cursor_granularity: Optional[str] = Field( None, - description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', - examples=['PT1S'], - title='Cursor Granularity', + description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", + examples=["PT1S"], + title="Cursor Granularity", ) end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( None, - description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', - examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], - title='End Datetime', + description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", + examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], + title="End Datetime", ) end_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the end datetime will be sent in requests to the source API.', - title='Inject End Time Into Outgoing HTTP Request', + description="Optionally configures how the end datetime will be sent in requests to the source API.", + title="Inject End Time Into Outgoing HTTP Request", ) is_data_feed: Optional[bool] = Field( None, - description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', - title='Whether the target API is formatted as a data feed', + description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", + title="Whether the target API is formatted as a data feed", ) lookback_window: Optional[str] = Field( None, - description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', - examples=['P1D', "P{{ config['lookback_days'] }}D"], - title='Lookback Window', + description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", + examples=["P1D", "P{{ config['lookback_days'] }}D"], + title="Lookback Window", ) partition_field_end: Optional[str] = Field( None, - description='Name of the partition start time field.', - examples=['ending_time'], - title='Partition Field End', + description="Name of the partition start time field.", + examples=["ending_time"], + title="Partition Field End", ) partition_field_start: Optional[str] = Field( None, - description='Name of the partition end time field.', - examples=['starting_time'], - title='Partition Field Start', + description="Name of the partition end time field.", + examples=["starting_time"], + title="Partition Field Start", ) start_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the start datetime will be sent in requests to the source API.', - title='Inject Start Time Into Outgoing HTTP Request', + description="Optionally configures how the start datetime will be sent in requests to the source API.", + title="Inject Start Time Into Outgoing HTTP Request", ) step: Optional[str] = Field( None, - description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', - examples=['P1W', "{{ config['step_increment'] }}"], - title='Step', + description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", + examples=["P1W", "{{ config['step_increment'] }}"], + title="Step", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultErrorHandler(BaseModel): - type: Literal['DefaultErrorHandler'] + type: Literal["DefaultErrorHandler"] backoff_strategies: Optional[ List[ Union[ @@ -919,144 +940,143 @@ class DefaultErrorHandler(BaseModel): ] ] = Field( None, - description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', - title='Backoff Strategies', + description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", + title="Backoff Strategies", ) max_retries: Optional[int] = Field( 5, - description='The maximum number of time to retry a retryable request before giving up and failing.', + description="The maximum number of time to retry a retryable request before giving up and failing.", examples=[5, 0, 10], - title='Max Retry Count', + title="Max Retry Count", ) response_filters: Optional[List[HttpResponseFilter]] = Field( None, description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", - title='Response Filters', + title="Response Filters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultPaginator(BaseModel): - type: Literal['DefaultPaginator'] - pagination_strategy: Union[ - CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement - ] = Field( + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field( ..., - description='Strategy defining how records are paginated.', - title='Pagination Strategy', + description="Strategy defining how records are paginated.", + title="Pagination Strategy", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestOption, RequestPath]] = None - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DpathExtractor(BaseModel): - type: Literal['DpathExtractor'] + type: Literal["DpathExtractor"] field_path: List[str] = Field( ..., description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ['data'], - ['data', 'records'], - ['data', '{{ parameters.name }}'], - ['data', '*', 'record'], + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], ], - title='Field Path', + title="Field Path", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestApiKeyAuthenticator(BaseModel): - type: Literal['ApiKey'] + type: Literal["ApiKey"] inject_into: RequestOption = Field( ..., - description='Configure how the API Key will be sent in requests to the source API.', + description="Configure how the API Key will be sent in requests to the source API.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) class ListPartitionRouter(BaseModel): - type: Literal['ListPartitionRouter'] + type: Literal["ListPartitionRouter"] cursor_field: str = Field( ..., description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', - examples=['section', "{{ config['section_key'] }}"], - title='Current Partition Value Identifier', + examples=["section", "{{ config['section_key'] }}"], + title="Current Partition Value Identifier", ) values: Union[str, List[str]] = Field( ..., - description='The list of attributes being iterated over and used as input for the requests made to the source API.', - examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], - title='Partition Values', + description="The list of attributes being iterated over and used as input for the requests made to the source API.", + examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], + title="Partition Values", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the list value should be injected into and under what field name if applicable.', - title='Inject Partition Value Into Outgoing HTTP Request', + description="A request option describing where the list value should be injected into and under what field name if applicable.", + title="Inject Partition Value Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RecordSelector(BaseModel): - type: Literal['RecordSelector'] + type: Literal["RecordSelector"] extractor: Union[CustomRecordExtractor, DpathExtractor] - record_filter: Optional[RecordFilter] = Field( + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( None, - description='Responsible for filtering records to be emitted by the Source.', - title='Record Filter', + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_ + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class Spec(BaseModel): - type: Literal['Spec'] + type: Literal["Spec"] connection_specification: Dict[str, Any] = Field( ..., - description='A connection specification describing how a the connector can be configured.', - title='Connection Specification', + description="A connection specification describing how a the connector can be configured.", + title="Connection Specification", ) documentation_url: Optional[str] = Field( None, description="URL of the connector's documentation page.", - examples=['https://docs.airbyte.com/integrations/sources/dremio'], - title='Documentation URL', + examples=["https://docs.airbyte.com/integrations/sources/dremio"], + title="Documentation URL", ) advanced_auth: Optional[AuthFlow] = Field( None, - description='Advanced specification for configuring the authentication flow.', - title='Advanced Auth', + description="Advanced specification for configuring the authentication flow.", + title="Advanced Auth", ) class CompositeErrorHandler(BaseModel): - type: Literal['CompositeErrorHandler'] + type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( ..., - description='List of error handlers to iterate on to determine how to handle a failed response.', - title='Error Handlers', + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DeclarativeSource(BaseModel): class Config: extra = Extra.forbid - type: Literal['DeclarativeSource'] + type: Literal["DeclarativeSource"] check: CheckStream streams: List[DeclarativeStream] version: str @@ -1065,109 +1085,138 @@ class Config: spec: Optional[Spec] = None metadata: Optional[Dict[str, Any]] = Field( None, - description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', + description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", ) -class DeclarativeStream(BaseModel): +class SelectiveAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['DeclarativeStream'] - retriever: Union[CustomRetriever, SimpleRetriever] = Field( + type: Literal["SelectiveAuthenticator"] + authenticator_selection_path: List[str] = Field( ..., - description='Component used to coordinate how records are extracted across stream slices and request pages.', - title='Retriever', + description="Path of the field in config with selected authenticator name", + examples=[["auth"], ["auth", "type"]], + title="Authenticator Selection Path", ) - incremental_sync: Optional[ - Union[CustomIncrementalSync, DatetimeBasedCursor] + authenticators: Dict[ + str, + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + NoAuth, + SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, + ], ] = Field( - None, - description='Component used to fetch data incrementally based on a time field in the data.', - title='Incremental Sync', + ..., + description="Authenticators to select from.", + examples=[ + { + "authenticators": { + "token": "#/definitions/ApiKeyAuthenticator", + "oauth": "#/definitions/OAuthAuthenticator", + } + } + ], + title="Authenticators", ) - name: Optional[str] = Field( - '', description='The stream name.', example=['Users'], title='Name' + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class DeclarativeStream(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["DeclarativeStream"] + retriever: Union[CustomRetriever, SimpleRetriever] = Field( + ..., + description="Component used to coordinate how records are extracted across stream slices and request pages.", + title="Retriever", ) - primary_key: Optional[PrimaryKey] = Field( - '', description='The primary key of the stream.', title='Primary Key' + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( None, - description='Component used to retrieve the schema for the current stream.', - title='Schema Loader', + description="Component used to retrieve the schema for the current stream.", + title="Schema Loader", ) - transformations: Optional[ - List[Union[AddFields, CustomTransformation, RemoveFields]] - ] = Field( + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( None, - description='A list of transformations to be applied to each output record.', - title='Transformations', + description="A list of transformations to be applied to each output record.", + title="Transformations", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenAuthenticator(BaseModel): - type: Literal['SessionTokenAuthenticator'] + type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( ..., - description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', + description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", examples=[ { - 'type': 'HttpRequester', - 'url_base': 'https://my_api.com', - 'path': '/login', - 'authenticator': { - 'type': 'BasicHttpAuthenticator', - 'username': '{{ config.username }}', - 'password': '{{ config.password }}', + "type": "HttpRequester", + "url_base": "https://my_api.com", + "path": "/login", + "authenticator": { + "type": "BasicHttpAuthenticator", + "username": "{{ config.username }}", + "password": "{{ config.password }}", }, } ], - title='Login Requester', + title="Login Requester", ) session_token_path: List[str] = Field( ..., - description='The path in the response body returned from the login requester to the session token.', - examples=[['access_token'], ['result', 'token']], - title='Session Token Path', + description="The path in the response body returned from the login requester to the session token.", + examples=[["access_token"], ["result", "token"]], + title="Session Token Path", ) expiration_duration: Optional[str] = Field( None, - description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', - examples=['PT1H', 'P1D'], - title='Expiration Duration', + description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", + examples=["PT1H", "P1D"], + title="Expiration Duration", ) - request_authentication: Union[ - SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator - ] = Field( + request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( ..., - description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', - title='Data Request Authentication', + description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", + title="Data Request Authentication", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class HttpRequester(BaseModel): - type: Literal['HttpRequester'] + type: Literal["HttpRequester"] url_base: str = Field( ..., - description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - 'https://connect.squareup.com/v2', + "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", ], - title='API Base URL', + title="API Base URL", ) path: str = Field( ..., - description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - '/products', + "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", "/trades/{{ config['symbol_id'] }}/history", ], - title='URL Path', + title="URL Path", ) authenticator: Optional[ Union[ @@ -1179,140 +1228,138 @@ class HttpRequester(BaseModel): NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, + SelectiveAuthenticator, ] ] = Field( None, - description='Authentication method to use for requests sent to the API.', - title='Authenticator', + description="Authentication method to use for requests sent to the API.", + title="Authenticator", ) - error_handler: Optional[ - Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] - ] = Field( + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field( None, - description='Error handler component that defines how to handle errors.', - title='Error Handler', + description="Error handler component that defines how to handle errors.", + title="Error Handler", ) - http_method: Optional[Union[str, HttpMethodEnum]] = Field( - 'GET', - description='The HTTP method used to fetch data from the source (can be GET or POST).', - examples=['GET', 'POST'], - title='HTTP Method', + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch data from the source (can be GET or POST).", + examples=["GET", "POST"], + title="HTTP Method", ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', + description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], - title='Request Body Payload (Non-JSON)', + title="Request Body Payload (Non-JSON)", ) request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( None, - description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', + description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", examples=[ - {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, - {'key': "{{ config['value'] }}"}, - {'sort': {'field': 'updated_at', 'order': 'ascending'}}, + {"sort_order": "ASC", "sort_field": "CREATED_AT"}, + {"key": "{{ config['value'] }}"}, + {"sort": {"field": "updated_at", "order": "ascending"}}, ], - title='Request Body JSON Payload', + title="Request Body JSON Payload", ) request_headers: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', - examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], - title='Request Headers', + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", ) request_parameters: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", examples=[ - {'unit': 'day'}, + {"unit": "day"}, { - 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' }, - {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, - {'sort_by[asc]': 'updated_at'}, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, ], - title='Query Parameters', + title="Query Parameters", ) use_cache: Optional[bool] = Field( False, - description='Enables stream requests caching. This field is automatically set by the CDK.', - title='Use Cache', + description="Enables stream requests caching. This field is automatically set by the CDK.", + title="Use Cache", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ParentStreamConfig(BaseModel): - type: Literal['ParentStreamConfig'] + type: Literal["ParentStreamConfig"] parent_key: str = Field( ..., - description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', - examples=['id', "{{ config['parent_record_id'] }}"], - title='Parent Key', - ) - stream: DeclarativeStream = Field( - ..., description='Reference to the parent stream.', title='Parent Stream' + description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", + examples=["id", "{{ config['parent_record_id'] }}"], + title="Parent Key", ) + stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream") partition_field: str = Field( ..., - description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', - examples=['parent_id', "{{ config['parent_partition_field'] }}"], - title='Current Parent Key Value Identifier', + description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", + examples=["parent_id", "{{ config['parent_partition_field'] }}"], + title="Current Parent Key Value Identifier", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the parent key value should be injected into and under what field name if applicable.', - title='Request Option', + description="A request option describing where the parent key value should be injected into and under what field name if applicable.", + title="Request Option", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SimpleRetriever(BaseModel): - type: Literal['SimpleRetriever'] + type: Literal["SimpleRetriever"] record_selector: RecordSelector = Field( ..., - description='Component that describes how to extract records from a HTTP response.', + description="Component that describes how to extract records from a HTTP response.", ) requester: Union[CustomRequester, HttpRequester] = Field( ..., - description='Requester component that describes how to prepare HTTP requests to send to the source API.', + description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( None, description="Paginator component that describes how to navigate through the API's pages.", ) + ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( + False, + description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.', + ) partition_router: Optional[ Union[ CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, - List[ - Union[ - CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter - ] - ], + List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], ] ] = Field( [], - description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', - title='Partition Router', + description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", + title="Partition Router", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SubstreamPartitionRouter(BaseModel): - type: Literal['SubstreamPartitionRouter'] + type: Literal["SubstreamPartitionRouter"] parent_stream_configs: List[ParentStreamConfig] = Field( ..., - description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', - title='Parent Stream Configs', + description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", + title="Parent Stream Configs", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") CompositeErrorHandler.update_forward_refs() DeclarativeSource.update_forward_refs() +SelectiveAuthenticator.update_forward_refs() DeclarativeStream.update_forward_refs() SessionTokenAuthenticator.update_forward_refs() SimpleRetriever.update_forward_refs() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py index f4f7694ad3ba..7b8b221c68df 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py @@ -103,7 +103,11 @@ def propagate_types_and_parameters( propagated_component["type"] = found_type # When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters - if "type" not in propagated_component: + # When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could + # be json_schema are not objects but we believe this is not likely in our case because: + # * records are Mapping so objects hence SchemaLoader root should be an object + # * connection_specification is a Mapping + if "type" not in propagated_component or self._is_json_schema_object(propagated_component): return propagated_component # Combines parameters defined at the current level with parameters from parent components. Parameters at the current @@ -140,3 +144,7 @@ def propagate_types_and_parameters( if current_parameters: propagated_component[PARAMETERS_STR] = current_parameters return propagated_component + + @staticmethod + def _is_json_schema_object(propagated_component: Mapping[str, Any]) -> bool: + return propagated_component.get("type") == "object" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 33138409afd2..45a44a97b04b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -11,8 +11,9 @@ from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, BasicHttpAuthenticator, @@ -25,6 +26,7 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector +from airbyte_cdk.sources.declarative.extractors.record_selector import SCHEMA_TRANSFORMER_TYPE_MAPPING from airbyte_cdk.sources.declarative.incremental import Cursor, CursorFactory, DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping @@ -44,6 +46,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomPaginationStrategy as CustomPaginationStrategyModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomPartitionRouter as CustomPartitionRouterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordExtractor as CustomRecordExtractorModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordFilter as CustomRecordFilterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRequester as CustomRequesterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRetriever as CustomRetrieverModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomTransformation as CustomTransformationModel @@ -76,6 +79,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import RemoveFields as RemoveFieldsModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestOption as RequestOptionModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestPath as RequestPathModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator as SelectiveAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SessionTokenAuthenticator as SessionTokenAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel @@ -105,6 +109,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath +from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.spec import Spec @@ -113,6 +118,7 @@ from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.types import Config from airbyte_cdk.sources.message import InMemoryMessageRepository, LogAppenderMessageRepositoryDecorator, MessageRepository +from airbyte_cdk.sources.utils.transform import TypeTransformer from isodate import parse_duration from pydantic import BaseModel @@ -156,6 +162,7 @@ def _init_mappings(self) -> None: CustomErrorHandlerModel: self.create_custom_component, CustomIncrementalSyncModel: self.create_custom_component, CustomRecordExtractorModel: self.create_custom_component, + CustomRecordFilterModel: self.create_custom_component, CustomRequesterModel: self.create_custom_component, CustomRetrieverModel: self.create_custom_component, CustomPaginationStrategyModel: self.create_custom_component, @@ -187,6 +194,7 @@ def _init_mappings(self) -> None: RequestPathModel: self.create_request_path, RequestOptionModel: self.create_request_option, LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, + SelectiveAuthenticatorModel: self.create_selective_authenticator, SimpleRetrieverModel: self.create_simple_retriever, SpecModel: self.create_spec, SubstreamPartitionRouterModel: self.create_substream_partition_router, @@ -707,9 +715,10 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na parameters=model.parameters or {}, ) - model_http_method = ( - model.http_method if isinstance(model.http_method, str) else model.http_method.value if model.http_method is not None else "GET" - ) + assert model.use_cache is not None # for mypy + assert model.http_method is not None # for mypy + + assert model.use_cache is not None # for mypy return HttpRequester( name=name, @@ -717,7 +726,7 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na path=model.path, authenticator=authenticator, error_handler=error_handler, - http_method=model_http_method, + http_method=HttpMethod[model.http_method.value], request_options_provider=request_options_provider, config=config, disable_retries=self._disable_retries, @@ -879,22 +888,40 @@ def create_request_option(model: RequestOptionModel, config: Config, **kwargs: A return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={}) def create_record_selector( - self, model: RecordSelectorModel, config: Config, *, transformations: List[RecordTransformation], **kwargs: Any + self, + model: RecordSelectorModel, + config: Config, + *, + transformations: List[RecordTransformation], + **kwargs: Any, ) -> RecordSelector: + assert model.schema_normalization is not None # for mypy extractor = self._create_component_from_model(model=model.extractor, config=config) record_filter = self._create_component_from_model(model.record_filter, config=config) if model.record_filter else None + schema_normalization = TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) return RecordSelector( extractor=extractor, config=config, record_filter=record_filter, transformations=transformations, + schema_normalization=schema_normalization, parameters=model.parameters or {}, ) @staticmethod def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any) -> RemoveFields: - return RemoveFields(field_pointers=model.field_pointers, parameters={}) + return RemoveFields(field_pointers=model.field_pointers, condition=model.condition or "", parameters={}) + + def create_selective_authenticator(self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeAuthenticator: + authenticators = {name: self._create_component_from_model(model=auth, config=config) for name, auth in model.authenticators.items()} + # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error + return SelectiveAuthenticator( # type: ignore[abstract] + config=config, + authenticators=authenticators, + authenticator_selection_path=model.authenticator_selection_path, + **kwargs, + ) @staticmethod def create_legacy_session_token_authenticator( @@ -933,12 +960,17 @@ def create_simple_retriever( cursor_used_for_stop_condition = cursor if stop_condition_on_cursor else None paginator = ( self._create_component_from_model( - model=model.paginator, config=config, url_base=url_base, cursor_used_for_stop_condition=cursor_used_for_stop_condition + model=model.paginator, + config=config, + url_base=url_base, + cursor_used_for_stop_condition=cursor_used_for_stop_condition, ) if model.paginator else NoPagination(parameters={}) ) + ignore_stream_slicer_parameters_on_paginated_requests = model.ignore_stream_slicer_parameters_on_paginated_requests or False + if self._limit_slices_fetched or self._emit_connector_builder_messages: return SimpleRetrieverTestReadDecorator( name=name, @@ -950,6 +982,7 @@ def create_simple_retriever( cursor=cursor, config=config, maximum_number_of_slices=self._limit_slices_fetched or 5, + ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, parameters=model.parameters or {}, ) return SimpleRetriever( @@ -961,6 +994,7 @@ def create_simple_retriever( stream_slicer=stream_slicer, cursor=cursor, config=config, + ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, parameters=model.parameters or {}, ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py index 9841bbd51dba..5413709d9615 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py @@ -81,7 +81,7 @@ def _get_request_option(self, request_option_type: RequestOptionType, stream_sli if self.request_option and self.request_option.inject_into == request_option_type and stream_slice: slice_value = stream_slice.get(self.cursor_field.eval(self.config)) if slice_value: - return {self.request_option.field_name: slice_value} + return {self.request_option.field_name.eval(self.config): slice_value} else: return {} else: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py index c080e56a49ce..3e915168c059 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py @@ -100,7 +100,7 @@ def _get_request_option(self, option_type: RequestOptionType, stream_slice: Stre key = parent_config.partition_field.eval(self.config) value = stream_slice.get(key) if value: - params.update({parent_config.request_option.field_name: value}) + params.update({parent_config.request_option.field_name.eval(config=self.config): value}) return params def stream_slices(self) -> Iterable[StreamSlice]: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index c6be905b483c..20c18ec9ba6c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -334,6 +334,11 @@ def _request_params( ) if isinstance(options, str): raise ValueError("Request params cannot be a string") + + for k, v in options.items(): + if isinstance(v, (list, dict)): + raise ValueError(f"Invalid value for `{k}` parameter. The values of request params cannot be an array or object.") + return options def _request_body_data( diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index e23b948859fc..824efe9aed39 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -164,9 +164,9 @@ def _get_request_options(self, option_type: RequestOptionType) -> MutableMapping and isinstance(self.page_token_option, RequestOption) and self.page_token_option.inject_into == option_type ): - options[self.page_token_option.field_name] = self._token + options[self.page_token_option.field_name.eval(config=self.config)] = self._token if self.page_size_option and self.pagination_strategy.get_page_size() and self.page_size_option.inject_into == option_type: - options[self.page_size_option.field_name] = self.pagination_strategy.get_page_size() + options[self.page_size_option.field_name.eval(config=self.config)] = self.pagination_strategy.get_page_size() return options diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py index 47de679c8944..d13d2056681d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py @@ -4,7 +4,9 @@ from dataclasses import InitVar, dataclass from enum import Enum -from typing import Any, Mapping +from typing import Any, Mapping, Union + +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString class RequestOptionType(Enum): @@ -28,6 +30,9 @@ class RequestOption: inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter """ - field_name: str + field_name: Union[InterpolatedString, str] inject_into: RequestOptionType parameters: InitVar[Mapping[str, Any]] + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self.field_name = InterpolatedString.create(self.field_name, parameters=parameters) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py index d46dc9463487..bf4247a4f441 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py @@ -4,7 +4,7 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Iterable, Optional +from typing import Any, Iterable, Mapping, Optional from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.core import StreamData @@ -19,15 +19,14 @@ class Retriever: @abstractmethod def read_records( self, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, ) -> Iterable[StreamData]: """ Fetch a stream's records from an HTTP API source - :param sync_mode: Unused but currently necessary for integrating with HttpStream - :param cursor_field: Unused but currently necessary for integrating with HttpStream + :param records_schema: json schema to describe record :param stream_slice: The stream slice to read data for - :param stream_state: The initial stream state :return: The records read from the API source """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index f269e35ebeab..a9c946044922 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -3,6 +3,7 @@ # from dataclasses import InitVar, dataclass, field +from functools import partial from itertools import islice from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union @@ -58,6 +59,7 @@ class SimpleRetriever(Retriever): paginator: Optional[Paginator] = None stream_slicer: StreamSlicer = SinglePartitionRouter(parameters={}) cursor: Optional[Cursor] = None + ignore_stream_slicer_parameters_on_paginated_requests: bool = False def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -104,12 +106,12 @@ def _get_request_options( Returned merged mapping otherwise """ # FIXME we should eventually remove the usage of stream_state as part of the interpolation - return combine_mappings( - [ - paginator_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - stream_slicer_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - ] - ) + mappings = [ + paginator_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + ] + if not next_page_token or not self.ignore_stream_slicer_parameters_on_paginated_requests: + mappings.append(stream_slicer_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)) + return combine_mappings(mappings) def _request_headers( self, @@ -215,6 +217,7 @@ def _parse_response( self, response: Optional[requests.Response], stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Record]: @@ -225,7 +228,11 @@ def _parse_response( self._last_response = response records = self.record_selector.select_records( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + response=response, + stream_state=stream_state, + records_schema=records_schema, + stream_slice=stream_slice, + next_page_token=next_page_token, ) self._records_from_last_response = records return records @@ -271,16 +278,15 @@ def _fetch_next_page( # This logic is similar to _read_pages in the HttpStream class. When making changes here, consider making changes there as well. def _read_pages( self, - records_generator_fn: Callable[[Optional[requests.Response], Mapping[str, Any], Mapping[str, Any]], Iterable[StreamData]], + records_generator_fn: Callable[[Optional[requests.Response]], Iterable[StreamData]], stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any], ) -> Iterable[StreamData]: - stream_state = stream_state or {} pagination_complete = False next_page_token = None while not pagination_complete: response = self._fetch_next_page(stream_state, stream_slice, next_page_token) - yield from records_generator_fn(response, stream_state, stream_slice) + yield from records_generator_fn(response) if not response: pagination_complete = True @@ -294,14 +300,28 @@ def _read_pages( def read_records( self, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, ) -> Iterable[StreamData]: + """ + Fetch a stream's records from an HTTP API source + + :param records_schema: json schema to describe record + :param stream_slice: The stream slice to read data for + :return: The records read from the API source + """ stream_slice = stream_slice or {} # None-check # Fixing paginator types has a long tail of dependencies self._paginator.reset() most_recent_record_from_slice = None - for stream_data in self._read_pages(self._parse_records, self.state, stream_slice): + record_generator = partial( + self._parse_records, + stream_state=self.state or {}, + stream_slice=stream_slice, + records_schema=records_schema, + ) + for stream_data in self._read_pages(record_generator, self.state, stream_slice): most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, stream_data, stream_slice) yield stream_data @@ -361,9 +381,15 @@ def _parse_records( self, response: Optional[requests.Response], stream_state: Mapping[str, Any], + records_schema: Mapping[str, Any], stream_slice: Optional[Mapping[str, Any]], ) -> Iterable[StreamData]: - yield from self._parse_response(response, stream_slice=stream_slice, stream_state=stream_state) + yield from self._parse_response( + response, + stream_slice=stream_slice, + stream_state=stream_state, + records_schema=records_schema, + ) def must_deduplicate_query_params(self) -> bool: return True diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py index b0d222273ef3..964483dddd87 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py @@ -7,6 +7,7 @@ import dpath.exceptions import dpath.util +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, FieldPointer, StreamSlice, StreamState @@ -40,6 +41,10 @@ class RemoveFields(RecordTransformation): field_pointers: List[FieldPointer] parameters: InitVar[Mapping[str, Any]] + condition: str = "" + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters=parameters) def transform( self, @@ -55,7 +60,11 @@ def transform( for pointer in self.field_pointers: # the dpath library by default doesn't delete fields from arrays try: - dpath.util.delete(record, pointer) + dpath.util.delete( + record, + pointer, + afilter=(lambda x: self._filter_interpolator.eval(config or {}, property=x)) if self.condition else None, + ) except dpath.exceptions.PathNotFound: # if the (potentially nested) property does not exist, silently skip pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py index 983f4eeb8bf7..a05e5421000a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py @@ -1,4 +1,4 @@ -from .abstract_file_based_availability_strategy import AbstractFileBasedAvailabilityStrategy +from .abstract_file_based_availability_strategy import AbstractFileBasedAvailabilityStrategy, AbstractFileBasedAvailabilityStrategyWrapper from .default_file_based_availability_strategy import DefaultFileBasedAvailabilityStrategy -__all__ = ["AbstractFileBasedAvailabilityStrategy", "DefaultFileBasedAvailabilityStrategy"] +__all__ = ["AbstractFileBasedAvailabilityStrategy", "AbstractFileBasedAvailabilityStrategyWrapper", "DefaultFileBasedAvailabilityStrategy"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py index 1ba12f64febd..ba26745ea57c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py @@ -8,6 +8,12 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.concurrent.availability_strategy import ( + AbstractAvailabilityStrategy, + StreamAvailability, + StreamAvailable, + StreamUnavailable, +) from airbyte_cdk.sources.streams.core import Stream if TYPE_CHECKING: @@ -35,3 +41,17 @@ def check_availability_and_parsability( Returns (True, None) if successful, otherwise (False, ). """ ... + + +class AbstractFileBasedAvailabilityStrategyWrapper(AbstractAvailabilityStrategy): + def __init__(self, stream: "AbstractFileBasedStream"): + self.stream = stream + + def check_availability(self, logger: logging.Logger) -> StreamAvailability: + is_available, reason = self.stream.availability_strategy.check_availability(self.stream, logger, None) + if is_available: + return StreamAvailable() + return StreamUnavailable(reason or "") + + def check_availability_and_parsability(self, logger: logging.Logger) -> Tuple[bool, Optional[str]]: + return self.stream.availability_strategy.check_availability_and_parsability(self.stream, logger, None) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py index 9a8d05c5255b..a5bef76f6176 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic import BaseModel, Field diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py index fab52aeefd28..bf8a57e73e90 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py @@ -150,6 +150,8 @@ class Config(OneOfOptionConfig): @validator("delimiter") def validate_delimiter(cls, v: str) -> str: + if v == r"\t": + return v if len(v) != 1: raise ValueError("delimiter should only be one character") if v in {"\r", "\n"}: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py index 2462df3d14cb..b462e78bba03 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic import BaseModel, Field diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py index 61951fc21472..18073be07b26 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py @@ -3,8 +3,9 @@ # from enum import Enum -from typing import Union +from typing import Any, List, Union +from airbyte_cdk.models import AirbyteMessage, FailureType from airbyte_cdk.utils import AirbyteTracedException @@ -40,6 +41,30 @@ class FileBasedSourceError(Enum): UNDEFINED_VALIDATION_POLICY = "The validation policy defined in the config does not exist for the source." +class FileBasedErrorsCollector: + """ + The placeholder for all errors collected. + """ + + errors: List[AirbyteMessage] = [] + + def yield_and_raise_collected(self) -> Any: + if self.errors: + # emit collected logged messages + yield from self.errors + # clean the collector + self.errors.clear() + # raising the single exception + raise AirbyteTracedException( + internal_message="Please check the logged errors for more information.", + message="Some errors occured while reading from the source.", + failure_type=FailureType.config_error, + ) + + def collect(self, logged_error: AirbyteMessage) -> None: + self.errors.append(logged_error) + + class BaseFileBasedSourceError(Exception): def __init__(self, error: Union[FileBasedSourceError, str], **kwargs): # type: ignore # noqa if isinstance(error, FileBasedSourceError): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py index 01a8e7d0bbce..8bd5cfe9565e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py @@ -8,47 +8,91 @@ from collections import Counter from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Type, Union -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.logger import AirbyteLogFormatter, init_logger +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + FailureType, + Level, + SyncMode, +) +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy -from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedErrorsCollector, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types import default_parsers from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream, DefaultFileBasedStream +from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import ( + AbstractConcurrentFileBasedCursor, + FileBasedConcurrentCursor, + FileBasedNoopCursor, +) from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor -from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField from airbyte_cdk.utils.analytics_message import create_analytics_message +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pydantic.error_wrappers import ValidationError +DEFAULT_CONCURRENCY = 100 +MAX_CONCURRENCY = 100 +INITIAL_N_PARTITIONS = MAX_CONCURRENCY // 2 + + +class FileBasedSource(ConcurrentSourceAdapter, ABC): + # We make each source override the concurrency level to give control over when they are upgraded. + _concurrency_level = None -class FileBasedSource(AbstractSource, ABC): def __init__( self, stream_reader: AbstractFileBasedStreamReader, spec_class: Type[AbstractFileBasedSpec], - catalog_path: Optional[str] = None, + catalog: Optional[ConfiguredAirbyteCatalog], + config: Optional[Mapping[str, Any]], + state: Optional[MutableMapping[str, Any]], availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None, discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(), parsers: Mapping[Type[Any], FileTypeParser] = default_parsers, validation_policies: Mapping[ValidationPolicy, AbstractSchemaValidationPolicy] = DEFAULT_SCHEMA_VALIDATION_POLICIES, - cursor_cls: Type[AbstractFileBasedCursor] = DefaultFileBasedCursor, + cursor_cls: Type[Union[AbstractConcurrentFileBasedCursor, AbstractFileBasedCursor]] = FileBasedConcurrentCursor, ): self.stream_reader = stream_reader self.spec_class = spec_class + self.config = config + self.catalog = catalog + self.state = state self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(stream_reader) self.discovery_policy = discovery_policy self.parsers = parsers self.validation_policies = validation_policies - catalog = self.read_catalog(catalog_path) if catalog_path else None self.stream_schemas = {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {} self.cursor_cls = cursor_cls - self.logger = logging.getLogger(f"airbyte.{self.name}") + self.logger = init_logger(f"airbyte.{self.name}") + self.errors_collector: FileBasedErrorsCollector = FileBasedErrorsCollector() + self._message_repository: Optional[MessageRepository] = None + concurrent_source = ConcurrentSource.create( + MAX_CONCURRENCY, INITIAL_N_PARTITIONS, self.logger, self._slice_logger, self.message_repository + ) + self._state = None + super().__init__(concurrent_source) + + @property + def message_repository(self) -> MessageRepository: + if self._message_repository is None: + self._message_repository = InMemoryMessageRepository(Level(AirbyteLogFormatter.level_mapping[self.logger.level])) + return self._message_repository def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """ @@ -60,7 +104,15 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Otherwise, the "error" object should describe what went wrong. """ - streams = self.streams(config) + try: + streams = self.streams(config) + except Exception as config_exception: + raise AirbyteTracedException( + internal_message="Please check the logged errors for more information.", + message=FileBasedSourceError.CONFIG_VALIDATION_ERROR.value, + exception=AirbyteTracedException(exception=config_exception), + failure_type=FailureType.config_error, + ) if len(streams) == 0: return ( False, @@ -79,7 +131,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> reason, ) = stream.availability_strategy.check_availability_and_parsability(stream, logger, self) except Exception: - errors.append(f"Unable to connect to stream {stream} - {''.join(traceback.format_exc())}") + errors.append(f"Unable to connect to stream {stream.name} - {''.join(traceback.format_exc())}") else: if not stream_is_available and reason: errors.append(reason) @@ -90,29 +142,101 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ Return a list of this source's streams. """ + + if self.catalog: + state_manager = ConnectorStateManager( + stream_instance_map={s.stream.name: s.stream for s in self.catalog.streams}, + state=self.state, + ) + else: + # During `check` operations we don't have a catalog so cannot create a state manager. + # Since the state manager is only required for incremental syncs, this is fine. + state_manager = None + try: parsed_config = self._get_parsed_config(config) self.stream_reader.config = parsed_config streams: List[Stream] = [] for stream_config in parsed_config.streams: + # Like state_manager, `catalog_stream` may be None during `check` + catalog_stream = self._get_stream_from_catalog(stream_config) + stream_state = ( + state_manager.get_stream_state(catalog_stream.name, catalog_stream.namespace) + if (state_manager and catalog_stream) + else None + ) self._validate_input_schema(stream_config) - streams.append( - DefaultFileBasedStream( - config=stream_config, - catalog_schema=self.stream_schemas.get(stream_config.name), - stream_reader=self.stream_reader, - availability_strategy=self.availability_strategy, - discovery_policy=self.discovery_policy, - parsers=self.parsers, - validation_policy=self._validate_and_get_validation_policy(stream_config), - cursor=self.cursor_cls(stream_config), + + sync_mode = self._get_sync_mode_from_catalog(stream_config.name) + + if sync_mode == SyncMode.full_refresh and hasattr(self, "_concurrency_level") and self._concurrency_level is not None: + cursor = FileBasedNoopCursor(stream_config) + stream = FileBasedStreamFacade.create_from_stream( + self._make_default_stream(stream_config, cursor), self, self.logger, stream_state, cursor ) - ) + + elif ( + sync_mode == SyncMode.incremental + and issubclass(self.cursor_cls, AbstractConcurrentFileBasedCursor) + and hasattr(self, "_concurrency_level") + and self._concurrency_level is not None + ): + assert ( + state_manager is not None + ), "No ConnectorStateManager was created, but it is required for incremental syncs. This is unexpected. Please contact Support." + + cursor = self.cursor_cls( + stream_config, + stream_config.name, + None, + stream_state, + self.message_repository, + state_manager, + CursorField(DefaultFileBasedStream.ab_last_mod_col), + ) + stream = FileBasedStreamFacade.create_from_stream( + self._make_default_stream(stream_config, cursor), self, self.logger, stream_state, cursor + ) + else: + cursor = self.cursor_cls(stream_config) + stream = self._make_default_stream(stream_config, cursor) + + streams.append(stream) return streams except ValidationError as exc: raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc + def _make_default_stream( + self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor] + ) -> AbstractFileBasedStream: + return DefaultFileBasedStream( + config=stream_config, + catalog_schema=self.stream_schemas.get(stream_config.name), + stream_reader=self.stream_reader, + availability_strategy=self.availability_strategy, + discovery_policy=self.discovery_policy, + parsers=self.parsers, + validation_policy=self._validate_and_get_validation_policy(stream_config), + errors_collector=self.errors_collector, + cursor=cursor, + ) + + def _get_stream_from_catalog(self, stream_config: FileBasedStreamConfig) -> Optional[AirbyteStream]: + if self.catalog: + for stream in self.catalog.streams or []: + if stream.stream.name == stream_config.name: + return stream.stream + return None + + def _get_sync_mode_from_catalog(self, stream_name: str) -> Optional[SyncMode]: + if self.catalog: + for catalog_stream in self.catalog.streams: + if stream_name == catalog_stream.stream.name: + return catalog_stream.sync_mode + self.logger.warning(f"No sync mode was found for {stream_name}.") + return None + def read( self, logger: logging.Logger, @@ -121,6 +245,8 @@ def read( state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, ) -> Iterator[AirbyteMessage]: yield from super().read(logger, config, catalog, state) + # emit all the errors collected + yield from self.errors_collector.yield_and_raise_collected() # count streams using a certain parser parsed_config = self._get_parsed_config(config) for parser, count in Counter(stream.format.filetype for stream in parsed_config.streams).items(): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py index 366e7429ba1d..25267b9a5c23 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py @@ -8,6 +8,7 @@ import fastavro from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -144,15 +145,20 @@ def parse_records( if not isinstance(avro_format, AvroFormat): raise ValueError(f"Expected ParquetFormat, got {avro_format}") - with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: - avro_reader = fastavro.reader(fp) - schema = avro_reader.writer_schema - schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]} - for record in avro_reader: - yield { - record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field]) - for record_field, record_value in schema_field_name_to_type.items() - } + line_no = 0 + try: + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + avro_reader = fastavro.reader(fp) + schema = avro_reader.writer_schema + schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]} + for record in avro_reader: + line_no += 1 + yield { + record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field]) + for record_field, record_value in schema_field_name_to_type.items() + } + except Exception as exc: + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from exc @property def file_read_mode(self) -> FileReadMode: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py index e687be07f092..627c3573b669 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py @@ -10,6 +10,7 @@ from functools import partial from io import IOBase from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set, Tuple +from uuid import uuid4 from airbyte_cdk.models import FailureType from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderUserProvided, InferenceType @@ -38,8 +39,10 @@ def read_data( # Formats are configured individually per-stream so a unique dialect should be registered for each stream. # We don't unregister the dialect because we are lazily parsing each csv file to generate records - # This will potentially be a problem if we ever process multiple streams concurrently - dialect_name = config.name + DIALECT_NAME + # Give each stream's dialect a unique name; otherwise, when we are doing a concurrent sync we can end up + # with a race condition where a thread attempts to use a dialect before a separate thread has finished + # registering it. + dialect_name = f"{config.name}_{str(uuid4())}_{DIALECT_NAME}" csv.register_dialect( dialect_name, delimiter=config_format.delimiter, @@ -178,17 +181,25 @@ def parse_records( logger: logging.Logger, discovered_schema: Optional[Mapping[str, SchemaType]], ) -> Iterable[Dict[str, Any]]: - config_format = _extract_format(config) - if discovered_schema: - property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()} # type: ignore # discovered_schema["properties"] is known to be a mapping - deduped_property_types = CsvParser._pre_propcess_property_types(property_types) - else: - deduped_property_types = {} - cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless) - data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) - for row in data_generator: - yield CsvParser._to_nullable(cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null) - data_generator.close() + line_no = 0 + try: + config_format = _extract_format(config) + if discovered_schema: + property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()} # type: ignore # discovered_schema["properties"] is known to be a mapping + deduped_property_types = CsvParser._pre_propcess_property_types(property_types) + else: + deduped_property_types = {} + cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless) + data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) + for row in data_generator: + line_no += 1 + yield CsvParser._to_nullable( + cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null + ) + except RecordParseError as parse_err: + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from parse_err + finally: + data_generator.close() @property def file_read_mode(self) -> FileReadMode: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py index e543e1a4f257..122103c5739d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py @@ -119,7 +119,7 @@ def _parse_jsonl_entries( break if had_json_parsing_error and not yielded_at_least_once: - raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD) + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line) @staticmethod def _instantiate_accumulator(line: Union[bytes, str]) -> Union[bytes, str]: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py index 06072a40cf10..00b78c489801 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py @@ -11,7 +11,7 @@ import pyarrow as pa import pyarrow.parquet as pq from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ParquetFormat -from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -64,19 +64,27 @@ def parse_records( if not isinstance(parquet_format, ParquetFormat): logger.info(f"Expected ParquetFormat, got {parquet_format}") raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) - with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: - reader = pq.ParquetFile(fp) - partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)} - for row_group in range(reader.num_row_groups): - batch = reader.read_row_group(row_group) - for row in range(batch.num_rows): - yield { - **{ - column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format) - for column in batch.column_names - }, - **partition_columns, - } + + line_no = 0 + try: + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + reader = pq.ParquetFile(fp) + partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)} + for row_group in range(reader.num_row_groups): + batch = reader.read_row_group(row_group) + for row in range(batch.num_rows): + line_no += 1 + yield { + **{ + column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format) + for column in batch.column_names + }, + **partition_columns, + } + except Exception as exc: + raise RecordParseError( + FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=f"{row_group=}, {line_no=}" + ) from exc @staticmethod def _extract_partitions(filepath: str) -> List[str]: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py index 64407d5ea04e..7c117b208672 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py @@ -3,6 +3,7 @@ # import logging import traceback +from datetime import datetime from io import BytesIO, IOBase from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union @@ -56,6 +57,8 @@ def user_error(e: Exception) -> bool: """ Return True if this exception is caused by user error, False otherwise. """ + if not isinstance(e, RecordParseError): + return False if not isinstance(e, requests.exceptions.RequestException): return False return bool(e.response and 400 <= e.response.status_code < 500) @@ -154,7 +157,7 @@ def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: Unstr filetype = self._get_filetype(file_handle, remote_file) - if filetype == FileType.MD: + if filetype == FileType.MD or filetype == FileType.TXT: file_content: bytes = file_handle.read() decoded_content: str = optional_decode(file_content) return decoded_content @@ -164,10 +167,14 @@ def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: Unstr return self._read_file_locally(file_handle, filetype, format.strategy, remote_file) elif format.processing.mode == "api": try: - result: str = self._read_file_remotely_with_retries(file_handle, format.processing, filetype, format.strategy) + result: str = self._read_file_remotely_with_retries(file_handle, format.processing, filetype, format.strategy, remote_file) except Exception as e: - # Re-throw as config error so the sync is stopped as problems with the external API need to be resolved by the user and are not considered part of the SLA. + # If a parser error happens during remotely processing the file, this means the file is corrupted. This case is handled by the parse_records method, so just rethrow. + # + # For other exceptions, re-throw as config error so the sync is stopped as problems with the external API need to be resolved by the user and are not considered part of the SLA. # Once this parser leaves experimental stage, we should consider making this a system error instead for issues that might be transient. + if isinstance(e, RecordParseError): + raise e raise AirbyteTracedException.from_exception(e, failure_type=FailureType.config_error) return result @@ -210,7 +217,13 @@ def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[st return False, "Base URL must start with https://" try: - self._read_file_remotely(BytesIO(b"# Airbyte source connection test"), format_config.processing, FileType.MD, "auto") + self._read_file_remotely( + BytesIO(b"# Airbyte source connection test"), + format_config.processing, + FileType.MD, + "auto", + RemoteFile(uri="test", last_modified=datetime.now()), + ) except Exception: return False, "".join(traceback.format_exc()) @@ -218,14 +231,16 @@ def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[st @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) def _read_file_remotely_with_retries( - self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str + self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile ) -> str: """ Read a file remotely, retrying up to 5 times if the error is not caused by user error. This is useful for transient network errors or the API server being overloaded temporarily. """ - return self._read_file_remotely(file_handle, format, filetype, strategy) + return self._read_file_remotely(file_handle, format, filetype, strategy, remote_file) - def _read_file_remotely(self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str) -> str: + def _read_file_remotely( + self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile + ) -> str: headers = {"accept": "application/json", "unstructured-api-key": format.api_key} data = self._params_to_dict(format.parameters, strategy) @@ -233,7 +248,13 @@ def _read_file_remotely(self, file_handle: IOBase, format: APIProcessingConfigMo file_data = {"files": ("filename", file_handle, FILETYPE_TO_MIMETYPE[filetype])} response = requests.post(f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data) - response.raise_for_status() + + if response.status_code == 422: + # 422 means the file couldn't be processed, but the API is working. Treat this as a parsing error (passing an error record to the destination). + raise self._create_parse_error(remote_file, response.json()) + else: + # Other error statuses are raised as requests exceptions (retry everything except user errors) + response.raise_for_status() json_response = response.json() @@ -304,7 +325,7 @@ def _get_filetype(self, file: IOBase, remote_file: RemoteFile) -> Optional[FileT return type_based_on_content def _supported_file_types(self) -> List[Any]: - return [FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX] + return [FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT] def _get_file_type_error_message(self, file_type: FileType) -> str: supported_file_types = ", ".join([str(type) for type in self._supported_file_types()]) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py index 474a271a48d7..69a44845dd85 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py @@ -10,11 +10,12 @@ from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, PrimaryKeyType from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy -from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError, UndefinedParserError +from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError, RecordParseError, UndefinedParserError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor from airbyte_cdk.sources.file_based.types import StreamSlice from airbyte_cdk.sources.streams import Stream @@ -44,6 +45,8 @@ def __init__( discovery_policy: AbstractDiscoveryPolicy, parsers: Dict[Type[Any], FileTypeParser], validation_policy: AbstractSchemaValidationPolicy, + errors_collector: FileBasedErrorsCollector, + cursor: AbstractFileBasedCursor, ): super().__init__() self.config = config @@ -53,6 +56,8 @@ def __init__( self._discovery_policy = discovery_policy self._availability_strategy = availability_strategy self._parsers = parsers + self.errors_collector = errors_collector + self._cursor = cursor @property @abstractmethod diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/__init__.py similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json rename to airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py new file mode 100644 index 000000000000..abaa8f7d044f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py @@ -0,0 +1,328 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +import copy +import logging +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.file_based.availability_strategy import ( + AbstractFileBasedAvailabilityStrategy, + AbstractFileBasedAvailabilityStrategyWrapper, +) +from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamSlice +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade +from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream +from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage +from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from deprecated.classic import deprecated + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor + +""" +This module contains adapters to help enabling concurrency on File-based Stream objects without needing to migrate to AbstractStream +""" + + +@deprecated("This class is experimental. Use at your own risk.") +class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBasedStream): + @classmethod + def create_from_stream( + cls, + stream: AbstractFileBasedStream, + source: AbstractSource, + logger: logging.Logger, + state: Optional[MutableMapping[str, Any]], + cursor: "AbstractConcurrentFileBasedCursor", + ) -> "FileBasedStreamFacade": + """ + Create a ConcurrentStream from a FileBasedStream object. + """ + pk = get_primary_key_from_stream(stream.primary_key) + cursor_field = get_cursor_field_from_stream(stream) + stream._cursor = cursor + + if not source.message_repository: + raise ValueError( + "A message repository is required to emit non-record messages. Please set the message repository on the source." + ) + + message_repository = source.message_repository + return FileBasedStreamFacade( + DefaultStream( + partition_generator=FileBasedStreamPartitionGenerator( + stream, + message_repository, + SyncMode.full_refresh if isinstance(cursor, FileBasedNoopCursor) else SyncMode.incremental, + [cursor_field] if cursor_field is not None else None, + state, + cursor, + ), + name=stream.name, + json_schema=stream.get_json_schema(), + availability_strategy=AbstractFileBasedAvailabilityStrategyWrapper(stream), + primary_key=pk, + cursor_field=cursor_field, + logger=logger, + namespace=stream.namespace, + cursor=cursor, + ), + stream, + cursor, + logger=logger, + slice_logger=source._slice_logger, + ) + + def __init__( + self, + stream: DefaultStream, + legacy_stream: AbstractFileBasedStream, + cursor: AbstractFileBasedCursor, + slice_logger: SliceLogger, + logger: logging.Logger, + ): + """ + :param stream: The underlying AbstractStream + """ + self._abstract_stream = stream + self._legacy_stream = legacy_stream + self._cursor = cursor + self._slice_logger = slice_logger + self._logger = logger + self.catalog_schema = legacy_stream.catalog_schema + self.config = legacy_stream.config + self.validation_policy = legacy_stream.validation_policy + + @property + def cursor_field(self) -> Union[str, List[str]]: + if self._abstract_stream.cursor_field is None: + return [] + else: + return self._abstract_stream.cursor_field + + @property + def name(self) -> str: + return self._abstract_stream.name + + @property + def supports_incremental(self) -> bool: + return self._legacy_stream.supports_incremental + + @property + def availability_strategy(self) -> AbstractFileBasedAvailabilityStrategy: + return self._legacy_stream.availability_strategy + + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + return self._abstract_stream.get_json_schema() + + @property + def primary_key(self) -> PrimaryKeyType: + return self._legacy_stream.config.primary_key or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config) + + def get_parser(self) -> FileTypeParser: + return self._legacy_stream.get_parser() + + def get_files(self) -> Iterable[RemoteFile]: + return self._legacy_stream.get_files() + + def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]: + yield from self._legacy_stream.read_records_from_slice(stream_slice) + + def compute_slices(self) -> Iterable[Optional[StreamSlice]]: + return self._legacy_stream.compute_slices() + + def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: + return self._legacy_stream.infer_schema(files) + + def get_underlying_stream(self) -> DefaultStream: + return self._abstract_stream + + def read_full_refresh( + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + ) -> Iterable[StreamData]: + """ + Read full refresh. Delegate to the underlying AbstractStream, ignoring all the parameters + :param cursor_field: (ignored) + :param logger: (ignored) + :param slice_logger: (ignored) + :return: Iterable of StreamData + """ + yield from self._read_records() + + def read_incremental( + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + stream_state: MutableMapping[str, Any], + state_manager: ConnectorStateManager, + per_stream_state_enabled: bool, + internal_config: InternalConfig, + ) -> Iterable[StreamData]: + yield from self._read_records() + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from self._read_records() + except Exception as exc: + if hasattr(self._cursor, "state"): + state = str(self._cursor.state) + else: + # This shouldn't happen if the ConcurrentCursor was used + state = "unknown; no state attribute was available on the cursor" + yield AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}") + ) + raise exc + + def _read_records(self) -> Iterable[StreamData]: + for partition in self._abstract_stream.generate_partitions(): + if self._slice_logger.should_log_slice_message(self._logger): + yield self._slice_logger.create_slice_log_message(partition.to_slice()) + for record in partition.read(): + yield record.data + + +class FileBasedStreamPartition(Partition): + def __init__( + self, + stream: AbstractFileBasedStream, + _slice: Optional[Mapping[str, Any]], + message_repository: MessageRepository, + sync_mode: SyncMode, + cursor_field: Optional[List[str]], + state: Optional[MutableMapping[str, Any]], + cursor: "AbstractConcurrentFileBasedCursor", + ): + self._stream = stream + self._slice = _slice + self._message_repository = message_repository + self._sync_mode = sync_mode + self._cursor_field = cursor_field + self._state = state + self._cursor = cursor + self._is_closed = False + + def read(self) -> Iterable[Record]: + try: + for record_data in self._stream.read_records( + cursor_field=self._cursor_field, + sync_mode=SyncMode.full_refresh, + stream_slice=copy.deepcopy(self._slice), + stream_state=self._state, + ): + if isinstance(record_data, Mapping): + data_to_return = dict(record_data) + self._stream.transformer.transform(data_to_return, self._stream.get_json_schema()) + yield Record(data_to_return, self.stream_name()) + elif isinstance(record_data, AirbyteMessage) and record_data.type == Type.RECORD: + # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued + yield Record(record_data.record.data, self.stream_name()) + else: + self._message_repository.emit_message(record_data) + except Exception as e: + display_message = self._stream.get_error_display_message(e) + if display_message: + raise ExceptionWithDisplayMessage(display_message) from e + else: + raise e + + def to_slice(self) -> Optional[Mapping[str, Any]]: + if self._slice is None: + return None + assert ( + len(self._slice["files"]) == 1 + ), f"Expected 1 file per partition but got {len(self._slice['files'])} for stream {self.stream_name()}" + file = self._slice["files"][0] + return {"files": [file]} + + def close(self) -> None: + self._cursor.close_partition(self) + self._is_closed = True + + def is_closed(self) -> bool: + return self._is_closed + + def __hash__(self) -> int: + if self._slice: + # Convert the slice to a string so that it can be hashed + if len(self._slice["files"]) != 1: + raise ValueError( + f"Slices for file-based streams should be of length 1, but got {len(self._slice['files'])}. This is unexpected. Please contact Support." + ) + else: + s = f"{self._slice['files'][0].last_modified.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}_{self._slice['files'][0].uri}" + return hash((self._stream.name, s)) + else: + return hash(self._stream.name) + + def stream_name(self) -> str: + return self._stream.name + + def __repr__(self) -> str: + return f"FileBasedStreamPartition({self._stream.name}, {self._slice})" + + +class FileBasedStreamPartitionGenerator(PartitionGenerator): + def __init__( + self, + stream: AbstractFileBasedStream, + message_repository: MessageRepository, + sync_mode: SyncMode, + cursor_field: Optional[List[str]], + state: Optional[MutableMapping[str, Any]], + cursor: "AbstractConcurrentFileBasedCursor", + ): + self._stream = stream + self._message_repository = message_repository + self._sync_mode = sync_mode + self._cursor_field = cursor_field + self._state = state + self._cursor = cursor + + def generate(self) -> Iterable[FileBasedStreamPartition]: + pending_partitions = [] + for _slice in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state): + if _slice is not None: + for file in _slice.get("files", []): + pending_partitions.append( + FileBasedStreamPartition( + self._stream, + {"files": [copy.deepcopy(file)]}, + self._message_repository, + self._sync_mode, + self._cursor_field, + self._state, + self._cursor, + ) + ) + self._cursor.set_pending_partitions(pending_partitions) + yield from pending_partitions diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py new file mode 100644 index 000000000000..6ab66bb39888 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py @@ -0,0 +1,5 @@ +from .abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor +from .file_based_concurrent_cursor import FileBasedConcurrentCursor +from .file_based_noop_cursor import FileBasedNoopCursor + +__all__ = ["AbstractConcurrentFileBasedCursor", "FileBasedConcurrentCursor", "FileBasedNoopCursor"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py new file mode 100644 index 000000000000..d21a6a01e70e --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC, abstractmethod +from datetime import datetime +from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping + +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamState +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition + + +class AbstractConcurrentFileBasedCursor(Cursor, AbstractFileBasedCursor, ABC): + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + @property + @abstractmethod + def state(self) -> MutableMapping[str, Any]: + ... + + @abstractmethod + def observe(self, record: Record) -> None: + ... + + @abstractmethod + def close_partition(self, partition: Partition) -> None: + ... + + @abstractmethod + def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None: + ... + + @abstractmethod + def add_file(self, file: RemoteFile) -> None: + ... + + @abstractmethod + def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]: + ... + + @abstractmethod + def get_state(self) -> MutableMapping[str, Any]: + ... + + @abstractmethod + def set_initial_state(self, value: StreamState) -> None: + ... + + @abstractmethod + def get_start_time(self) -> datetime: + ... + + @abstractmethod + def emit_state_message(self) -> None: + ... + + @abstractmethod + def ensure_at_least_one_state_emitted(self) -> None: + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py new file mode 100644 index 000000000000..4019dfd17b9e --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py @@ -0,0 +1,279 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime, timedelta +from threading import RLock +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, MutableMapping, Optional, Tuple + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamState +from airbyte_cdk.sources.message.repository import MessageRepository +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition + +_NULL_FILE = "" + + +class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor): + CURSOR_FIELD = "_ab_source_file_last_modified" + DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL + DEFAULT_MAX_HISTORY_SIZE = 10_000 + DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT + zero_value = datetime.min + zero_cursor_value = f"0001-01-01T00:00:00.000000Z_{_NULL_FILE}" + + def __init__( + self, + stream_config: FileBasedStreamConfig, + stream_name: str, + stream_namespace: Optional[str], + stream_state: MutableMapping[str, Any], + message_repository: MessageRepository, + connector_state_manager: ConnectorStateManager, + cursor_field: CursorField, + ) -> None: + super().__init__() + self._stream_name = stream_name + self._stream_namespace = stream_namespace + self._state = stream_state + self._message_repository = message_repository + self._connector_state_manager = connector_state_manager + self._cursor_field = cursor_field + self._time_window_if_history_is_full = timedelta( + days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL + ) + self._state_lock = RLock() + self._pending_files_lock = RLock() + self._pending_files: Optional[Dict[str, RemoteFile]] = None + self._file_to_datetime_history = stream_state.get("history", {}) if stream_state else {} + self._prev_cursor_value = self._compute_prev_sync_cursor(stream_state) + self._sync_start = self._compute_start_time() + + @property + def state(self) -> MutableMapping[str, Any]: + return self._state + + def observe(self, record: Record) -> None: + pass + + def close_partition(self, partition: Partition) -> None: + with self._pending_files_lock: + if self._pending_files is None: + raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.") + + def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None: + with self._pending_files_lock: + self._pending_files = {} + for partition in partitions: + _slice = partition.to_slice() + if _slice is None: + continue + for file in _slice["files"]: + if file.uri in self._pending_files.keys(): + raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.") + self._pending_files.update({file.uri: file}) + + def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]: + if not value: + return self.zero_value, "" + prev_cursor_str = value.get(self._cursor_field.cursor_field_key) or self.zero_cursor_value + # So if we see a cursor greater than the earliest file, it means that we have likely synced all files. + # However, we take the earliest file as the cursor value for the purpose of checking which files to + # sync, in case new files have been uploaded in the meantime. + # This should be very rare, as it would indicate a race condition where a file with an earlier + # last_modified time was uploaded after a file with a later last_modified time. Since last_modified + # represents the start time that the file was uploaded, we can usually expect that all previous + # files have already been uploaded. If that's the case, they'll be in history and we'll skip + # re-uploading them. + earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history()) + cursor_str = min(prev_cursor_str, earliest_file_cursor_value) + cursor_dt, cursor_uri = cursor_str.split("_", 1) + return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri + + def _get_cursor_key_from_file(self, file: Optional[RemoteFile]) -> str: + if file: + return f"{datetime.strftime(file.last_modified, self.DATE_TIME_FORMAT)}_{file.uri}" + return self.zero_cursor_value + + def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]: + with self._state_lock: + if self._file_to_datetime_history: + filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])) + return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT)) + else: + return None + + def add_file(self, file: RemoteFile) -> None: + """ + Add a file to the cursor. This method is called when a file is processed by the stream. + :param file: The file to add + """ + if self._pending_files is None: + raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.") + with self._pending_files_lock: + with self._state_lock: + if file.uri not in self._pending_files: + self._message_repository.emit_message( + AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage( + level=Level.WARN, + message=f"The file {file.uri} was not found in the list of pending files. This is unexpected. Please contact Support", + ), + ) + ) + else: + self._pending_files.pop(file.uri) + self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT) + if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE: + # Get the earliest file based on its last modified date and its uri + oldest_file = self._compute_earliest_file_in_history() + if oldest_file: + del self._file_to_datetime_history[oldest_file.uri] + else: + raise Exception( + "The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK." + ) + self.emit_state_message() + + def emit_state_message(self) -> None: + with self._state_lock: + new_state = self.get_state() + self._connector_state_manager.update_state_for_stream( + self._stream_name, + self._stream_namespace, + new_state, + ) + state_message = self._connector_state_manager.create_state_message( + self._stream_name, self._stream_namespace, send_per_stream_state=True + ) + self._message_repository.emit_message(state_message) + + def _get_new_cursor_value(self) -> str: + with self._pending_files_lock: + with self._state_lock: + if self._pending_files: + # If there are partitions that haven't been synced, we don't know whether the files that have been synced + # represent a contiguous region. + # To avoid missing files, we only increment the cursor up to the oldest pending file, because we know + # that all older files have been synced. + return self._get_cursor_key_from_file(self._compute_earliest_pending_file()) + elif self._file_to_datetime_history: + # If all partitions have been synced, we know that the sync is up-to-date and so can advance + # the cursor to the newest file in history. + return self._get_cursor_key_from_file(self._compute_latest_file_in_history()) + else: + return f"{self.zero_value.strftime(self.DATE_TIME_FORMAT)}_" + + def _compute_earliest_pending_file(self) -> Optional[RemoteFile]: + if self._pending_files: + return min(self._pending_files.values(), key=lambda x: x.last_modified) + else: + return None + + def _compute_latest_file_in_history(self) -> Optional[RemoteFile]: + with self._state_lock: + if self._file_to_datetime_history: + filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])) + return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT)) + else: + return None + + def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Given the list of files in the source, return the files that should be synced. + :param all_files: All files in the source + :param logger: + :return: The files that should be synced + """ + with self._state_lock: + if self._is_history_full(): + logger.warning( + f"The state history is full. " + f"This sync and future syncs won't be able to use the history to filter out duplicate files. " + f"It will instead use the time window of {self._time_window_if_history_is_full} to filter out files." + ) + for f in all_files: + if self._should_sync_file(f, logger): + yield f + + def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool: + with self._state_lock: + if file.uri in self._file_to_datetime_history: + # If the file's uri is in the history, we should sync the file if it has been modified since it was synced + updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT) + if file.last_modified < updated_at_from_history: + self._message_repository.emit_message( + AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage( + level=Level.WARN, + message=f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file.", + ), + ) + ) + return False + else: + return file.last_modified > updated_at_from_history + + prev_cursor_timestamp, prev_cursor_uri = self._prev_cursor_value + if self._is_history_full(): + if file.last_modified > prev_cursor_timestamp: + # If the history is partial and the file's datetime is strictly greater than the cursor, we should sync it + return True + elif file.last_modified == prev_cursor_timestamp: + # If the history is partial and the file's datetime is equal to the earliest file in the history, + # we should sync it if its uri is greater than or equal to the cursor value. + return file.uri > prev_cursor_uri + else: + return file.last_modified >= self._sync_start + else: + # The file is not in the history and the history is complete. We know we need to sync the file + return True + + def _is_history_full(self) -> bool: + """ + Returns true if the state's history is full, meaning new entries will start to replace old entries. + """ + with self._state_lock: + if self._file_to_datetime_history is None: + raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.") + return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE + + def _compute_start_time(self) -> datetime: + if not self._file_to_datetime_history: + return datetime.min + else: + earliest = min(self._file_to_datetime_history.values()) + earliest_dt = datetime.strptime(earliest, self.DATE_TIME_FORMAT) + if self._is_history_full(): + time_window = datetime.now() - self._time_window_if_history_is_full + earliest_dt = min(earliest_dt, time_window) + return earliest_dt + + def get_start_time(self) -> datetime: + return self._sync_start + + def get_state(self) -> MutableMapping[str, Any]: + """ + Get the state of the cursor. + """ + with self._state_lock: + return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()} + + def set_initial_state(self, value: StreamState) -> None: + pass + + def ensure_at_least_one_state_emitted(self) -> None: + self.emit_state_message() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py new file mode 100644 index 000000000000..2aa5a204d503 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime +from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamState +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition + + +class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor): + def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any): + pass + + @property + def state(self) -> MutableMapping[str, Any]: + return {} + + def observe(self, record: Record) -> None: + pass + + def close_partition(self, partition: Partition) -> None: + pass + + def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None: + pass + + def add_file(self, file: RemoteFile) -> None: + pass + + def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]: + return all_files + + def get_state(self) -> MutableMapping[str, Any]: + return {} + + def set_initial_state(self, value: StreamState) -> None: + return None + + def get_start_time(self) -> datetime: + return datetime.min + + def emit_state_message(self) -> None: + pass + + def ensure_at_least_one_state_emitted(self) -> None: + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py index 86888236b466..92fdef8996ae 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py @@ -43,9 +43,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin): ab_file_name_col = "_ab_source_file_url" airbyte_columns = [ab_last_mod_col, ab_file_name_col] - def __init__(self, cursor: AbstractFileBasedCursor, **kwargs: Any): + def __init__(self, **kwargs: Any): super().__init__(**kwargs) - self._cursor = cursor @property def state(self) -> MutableMapping[str, Any]: @@ -56,6 +55,16 @@ def state(self, value: MutableMapping[str, Any]) -> None: """State setter, accept state serialized by state getter.""" self._cursor.set_initial_state(value) + @property + def cursor(self) -> Optional[AbstractFileBasedCursor]: + return self._cursor + + @cursor.setter + def cursor(self, value: AbstractFileBasedCursor) -> None: + if self._cursor is not None: + raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.") + self._cursor = value + @property def primary_key(self) -> PrimaryKeyType: return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config) @@ -112,12 +121,14 @@ def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Airbyte except RecordParseError: # Increment line_no because the exception was raised before we could increment it line_no += 1 - yield AirbyteMessage( - type=MessageType.LOG, - log=AirbyteLogMessage( - level=Level.ERROR, - message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", - stack_trace=traceback.format_exc(), + self.errors_collector.collect( + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), ), ) @@ -162,9 +173,13 @@ def get_json_schema(self) -> JsonSchema: try: schema = self._get_raw_json_schema() except (InvalidSchemaError, NoFilesMatchingError) as config_exception: + self.logger.exception(FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, exc_info=config_exception) raise AirbyteTracedException( - message=FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, exception=config_exception, failure_type=FailureType.config_error - ) from config_exception + internal_message="Please check the logged errors for more information.", + message=FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, + exception=AirbyteTracedException(exception=config_exception), + failure_type=FailureType.config_error, + ) except Exception as exc: raise SchemaInferenceError(FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name) from exc else: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py index d98e7a7b5498..d48c32c48dbe 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py @@ -7,6 +7,7 @@ from airbyte_cdk.models import AirbyteStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from deprecated.classic import deprecated @@ -81,3 +82,10 @@ def log_stream_sync_configuration(self) -> None: """ Logs the stream's configuration for debugging purposes. """ + + @property + @abstractmethod + def cursor(self) -> Cursor: + """ + :return: The cursor associated with this stream. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py new file mode 100644 index 000000000000..18cacbc500d5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from abc import ABC, abstractmethod +from typing import Generic, Optional, TypeVar + +from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage + +StreamType = TypeVar("StreamType") + + +class AbstractStreamFacade(Generic[StreamType], ABC): + @abstractmethod + def get_underlying_stream(self) -> StreamType: + """ + Return the underlying stream facade object. + """ + ... + + @property + def source_defined_cursor(self) -> bool: + # Streams must be aware of their cursor at instantiation time + return True + + def get_error_display_message(self, exception: BaseException) -> Optional[str]: + """ + Retrieves the user-friendly display message that corresponds to an exception. + This will be called when encountering an exception while reading records from the stream, and used to build the AirbyteTraceMessage. + + A display message will be returned if the exception is an instance of ExceptionWithDisplayMessage. + + :param exception: The exception that was raised + :return: A user-friendly message that indicates the cause of the error + """ + if isinstance(exception, ExceptionWithDisplayMessage): + return exception.display_message + else: + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py index c8589f2e3bac..86542618354f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py @@ -8,13 +8,13 @@ from functools import lru_cache from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from airbyte_cdk.models import AirbyteStream, SyncMode +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, Level, SyncMode, Type from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade from airbyte_cdk.sources.streams.concurrent.availability_strategy import ( AbstractAvailabilityStrategy, StreamAvailability, @@ -24,6 +24,7 @@ from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage +from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator from airbyte_cdk.sources.streams.concurrent.partitions.record import Record @@ -38,7 +39,7 @@ @deprecated("This class is experimental. Use at your own risk.") -class StreamFacade(Stream): +class StreamFacade(AbstractStreamFacade[DefaultStream], Stream): """ The StreamFacade is a Stream that wraps an AbstractStream and exposes it as a Stream. @@ -62,8 +63,8 @@ def create_from_stream( :param max_workers: The maximum number of worker thread to use :return: """ - pk = cls._get_primary_key_from_stream(stream.primary_key) - cursor_field = cls._get_cursor_field_from_stream(stream) + pk = get_primary_key_from_stream(stream.primary_key) + cursor_field = get_cursor_field_from_stream(stream) if not source.message_repository: raise ValueError( @@ -88,6 +89,7 @@ def create_from_stream( primary_key=pk, cursor_field=cursor_field, logger=logger, + cursor=cursor, ), stream, cursor, @@ -104,33 +106,7 @@ def state(self, value: Mapping[str, Any]) -> None: if "state" in dir(self._legacy_stream): self._legacy_stream.state = value # type: ignore # validating `state` is attribute of stream using `if` above - @classmethod - def _get_primary_key_from_stream(cls, stream_primary_key: Optional[Union[str, List[str], List[List[str]]]]) -> List[str]: - if stream_primary_key is None: - return [] - elif isinstance(stream_primary_key, str): - return [stream_primary_key] - elif isinstance(stream_primary_key, list): - if len(stream_primary_key) > 0 and all(isinstance(k, str) for k in stream_primary_key): - return stream_primary_key # type: ignore # We verified all items in the list are strings - else: - raise ValueError(f"Nested primary keys are not supported. Found {stream_primary_key}") - else: - raise ValueError(f"Invalid type for primary key: {stream_primary_key}") - - @classmethod - def _get_cursor_field_from_stream(cls, stream: Stream) -> Optional[str]: - if isinstance(stream.cursor_field, list): - if len(stream.cursor_field) > 1: - raise ValueError(f"Nested cursor fields are not supported. Got {stream.cursor_field} for {stream.name}") - elif len(stream.cursor_field) == 0: - return None - else: - return stream.cursor_field[0] - else: - return stream.cursor_field - - def __init__(self, stream: AbstractStream, legacy_stream: Stream, cursor: Cursor, slice_logger: SliceLogger, logger: logging.Logger): + def __init__(self, stream: DefaultStream, legacy_stream: Stream, cursor: Cursor, slice_logger: SliceLogger, logger: logging.Logger): """ :param stream: The underlying AbstractStream """ @@ -174,7 +150,18 @@ def read_records( stream_slice: Optional[Mapping[str, Any]] = None, stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: - yield from self._read_records() + try: + yield from self._read_records() + except Exception as exc: + if hasattr(self._cursor, "state"): + state = str(self._cursor.state) + else: + # This shouldn't happen if the ConcurrentCursor was used + state = "unknown; no state attribute was available on the cursor" + yield AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}") + ) + raise exc def _read_records(self) -> Iterable[StreamData]: for partition in self._abstract_stream.generate_partitions(): @@ -199,11 +186,6 @@ def cursor_field(self) -> Union[str, List[str]]: else: return self._abstract_stream.cursor_field - @property - def source_defined_cursor(self) -> bool: - # Streams must be aware of their cursor at instantiation time - return True - @lru_cache(maxsize=None) def get_json_schema(self) -> Mapping[str, Any]: return self._abstract_stream.get_json_schema() @@ -222,27 +204,15 @@ def check_availability(self, logger: logging.Logger, source: Optional["Source"] availability = self._abstract_stream.check_availability() return availability.is_available(), availability.message() - def get_error_display_message(self, exception: BaseException) -> Optional[str]: - """ - Retrieves the user-friendly display message that corresponds to an exception. - This will be called when encountering an exception while reading records from the stream, and used to build the AirbyteTraceMessage. - - A display message will be returned if the exception is an instance of ExceptionWithDisplayMessage. - - :param exception: The exception that was raised - :return: A user-friendly message that indicates the cause of the error - """ - if isinstance(exception, ExceptionWithDisplayMessage): - return exception.display_message - else: - return None - def as_airbyte_stream(self) -> AirbyteStream: return self._abstract_stream.as_airbyte_stream() def log_stream_sync_configuration(self) -> None: self._abstract_stream.log_stream_sync_configuration() + def get_underlying_stream(self) -> DefaultStream: + return self._abstract_stream + class StreamPartition(Partition): """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py index 7da5ff5fa8b3..82d11318f5ea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -3,13 +3,14 @@ # import functools from abc import ABC, abstractmethod -from typing import Any, List, Mapping, Optional, Protocol, Tuple +from datetime import datetime +from typing import Any, List, Mapping, MutableMapping, Optional, Protocol, Tuple from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record -from airbyte_cdk.sources.streams.concurrent.state_converter import ConcurrentStreamStateConverter +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import AbstractStreamStateConverter def _extract_value(mapping: Mapping[str, Any], path: List[str]) -> Any: @@ -26,16 +27,21 @@ def __lt__(self: "Comparable", other: "Comparable") -> bool: class CursorField: def __init__(self, cursor_field_key: str) -> None: - self._cursor_field_key = cursor_field_key + self.cursor_field_key = cursor_field_key def extract_value(self, record: Record) -> Comparable: - cursor_value = record.data.get(self._cursor_field_key) + cursor_value = record.data.get(self.cursor_field_key) if cursor_value is None: - raise ValueError(f"Could not find cursor field {self._cursor_field_key} in record") + raise ValueError(f"Could not find cursor field {self.cursor_field_key} in record") return cursor_value # type: ignore # we assume that the value the path points at is a comparable class Cursor(ABC): + @property + @abstractmethod + def state(self) -> MutableMapping[str, Any]: + ... + @abstractmethod def observe(self, record: Record) -> None: """ @@ -50,14 +56,29 @@ def close_partition(self, partition: Partition) -> None: """ raise NotImplementedError() + @abstractmethod + def ensure_at_least_one_state_emitted(self) -> None: + """ + State messages are emitted when a partition is closed. However, the platform expects at least one state to be emitted per sync per + stream. Hence, if no partitions are generated, this method needs to be called. + """ + raise NotImplementedError() + class NoopCursor(Cursor): + @property + def state(self) -> MutableMapping[str, Any]: + return {} + def observe(self, record: Record) -> None: pass def close_partition(self, partition: Partition) -> None: pass + def ensure_at_least_one_state_emitted(self) -> None: + pass + class ConcurrentCursor(Cursor): _START_BOUNDARY = 0 @@ -70,9 +91,10 @@ def __init__( stream_state: Any, message_repository: MessageRepository, connector_state_manager: ConnectorStateManager, - connector_state_converter: ConcurrentStreamStateConverter, + connector_state_converter: AbstractStreamStateConverter, cursor_field: CursorField, slice_boundary_fields: Optional[Tuple[str, str]], + start: Optional[Any], ) -> None: self._stream_name = stream_name self._stream_namespace = stream_namespace @@ -82,9 +104,19 @@ def __init__( self._cursor_field = cursor_field # To see some example where the slice boundaries might not be defined, check https://github.com/airbytehq/airbyte/blob/1ce84d6396e446e1ac2377362446e3fb94509461/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L363-L379 self._slice_boundary_fields = slice_boundary_fields if slice_boundary_fields else tuple() + self._start = start self._most_recent_record: Optional[Record] = None self._has_closed_at_least_one_slice = False - self._state = connector_state_converter.get_concurrent_stream_state(stream_state) + self.start, self._concurrent_state = self._get_concurrent_state(stream_state) + + @property + def state(self) -> MutableMapping[str, Any]: + return self._concurrent_state + + def _get_concurrent_state(self, state: MutableMapping[str, Any]) -> Tuple[datetime, MutableMapping[str, Any]]: + if self._connector_state_converter.is_state_message_compatible(state): + return self._start or self._connector_state_converter.zero_value, self._connector_state_converter.deserialize(state) + return self._connector_state_converter.convert_from_sequential_state(self._cursor_field, state, self._start) def observe(self, record: Record) -> None: if self._slice_boundary_fields: @@ -96,22 +128,24 @@ def observe(self, record: Record) -> None: if not self._most_recent_record or self._extract_cursor_value(self._most_recent_record) < self._extract_cursor_value(record): self._most_recent_record = record - def _extract_cursor_value(self, record: Record) -> Comparable: - return self._cursor_field.extract_value(record) + def _extract_cursor_value(self, record: Record) -> Any: + return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record)) def close_partition(self, partition: Partition) -> None: - slice_count_before = len(self._state["slices"]) + slice_count_before = len(self.state.get("slices", [])) self._add_slice_to_state(partition) - if slice_count_before < len(self._state["slices"]): + if slice_count_before < len(self.state["slices"]): # only emit if at least one slice has been processed self._merge_partitions() self._emit_state_message() self._has_closed_at_least_one_slice = True def _add_slice_to_state(self, partition: Partition) -> None: if self._slice_boundary_fields: - if "slices" not in self._state: - self._state["slices"] = [] - self._state["slices"].append( + if "slices" not in self.state: + raise RuntimeError( + f"The state for stream {self._stream_name} should have at least one slice to delineate the sync start time, but no slices are present. This is unexpected. Please contact Support." + ) + self.state["slices"].append( { "start": self._extract_from_slice(partition, self._slice_boundary_fields[self._START_BOUNDARY]), "end": self._extract_from_slice(partition, self._slice_boundary_fields[self._END_BOUNDARY]), @@ -119,33 +153,57 @@ def _add_slice_to_state(self, partition: Partition) -> None: ) elif self._most_recent_record: if self._has_closed_at_least_one_slice: + # If we track state value using records cursor field, we can only do that if there is one partition. This is because we save + # the state every time we close a partition. We assume that if there are multiple slices, they need to be providing + # boundaries. There are cases where partitions could not have boundaries: + # * The cursor should be per-partition + # * The stream state is actually the parent stream state + # There might be other cases not listed above. Those are not supported today hence the stream should not use this cursor for + # state management. For the specific user that was affected with this issue, we need to: + # * Fix state tracking (which is currently broken) + # * Make the new version available + # * (Probably) ask the user to reset the stream to avoid data loss raise ValueError( "Given that slice_boundary_fields is not defined and that per-partition state is not supported, only one slice is " - "expected." + "expected. Please contact the Airbyte team." ) - self._state["slices"].append( + self.state["slices"].append( { - "start": 0, # FIXME this only works with int datetime - "end": self._extract_cursor_value(self._most_recent_record), + self._connector_state_converter.START_KEY: self.start, + self._connector_state_converter.END_KEY: self._extract_cursor_value(self._most_recent_record), } ) def _emit_state_message(self) -> None: - self._connector_state_manager.update_state_for_stream(self._stream_name, self._stream_namespace, self._state) + self._connector_state_manager.update_state_for_stream( + self._stream_name, + self._stream_namespace, + self._connector_state_converter.convert_to_sequential_state(self._cursor_field, self.state), + ) + # TODO: if we migrate stored state to the concurrent state format + # (aka stop calling self._connector_state_converter.convert_to_sequential_state`), we'll need to cast datetimes to string or + # int before emitting state state_message = self._connector_state_manager.create_state_message( self._stream_name, self._stream_namespace, send_per_stream_state=True ) self._message_repository.emit_message(state_message) def _merge_partitions(self) -> None: - self._state["slices"] = self._connector_state_converter.merge_intervals(self._state["slices"]) + self.state["slices"] = self._connector_state_converter.merge_intervals(self.state["slices"]) def _extract_from_slice(self, partition: Partition, key: str) -> Comparable: try: _slice = partition.to_slice() if not _slice: raise KeyError(f"Could not find key `{key}` in empty slice") - return _slice[key] # type: ignore # we expect the devs to specify a key that would return a Comparable + return self._connector_state_converter.parse_value(_slice[key]) # type: ignore # we expect the devs to specify a key that would return a Comparable except KeyError as exception: raise KeyError(f"Partition is expected to have key `{key}` but could not be found") from exception + + def ensure_at_least_one_state_emitted(self) -> None: + """ + The platform expect to have at least one state message on successful syncs. Hence, whatever happens, we expect this method to be + called. + """ + self._emit_state_message() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py index 8606d273bb4f..3e839cb3959e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py @@ -9,6 +9,7 @@ from airbyte_cdk.models import AirbyteStream, SyncMode from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator @@ -23,6 +24,7 @@ def __init__( primary_key: List[str], cursor_field: Optional[str], logger: Logger, + cursor: Optional[Cursor], namespace: Optional[str] = None, ) -> None: self._stream_partition_generator = partition_generator @@ -32,6 +34,7 @@ def __init__( self._primary_key = primary_key self._cursor_field = cursor_field self._logger = logger + self._cursor = cursor or NoopCursor() self._namespace = namespace def generate_partitions(self) -> Iterable[Partition]: @@ -77,3 +80,7 @@ def log_stream_sync_configuration(self) -> None: "cursor_field": self.cursor_field, }, ) + + @property + def cursor(self) -> Cursor: + return self._cursor diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/helpers.py new file mode 100644 index 000000000000..ad7722726498 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/helpers.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import List, Optional, Union + +from airbyte_cdk.sources.streams import Stream + + +def get_primary_key_from_stream(stream_primary_key: Optional[Union[str, List[str], List[List[str]]]]) -> List[str]: + if stream_primary_key is None: + return [] + elif isinstance(stream_primary_key, str): + return [stream_primary_key] + elif isinstance(stream_primary_key, list): + if len(stream_primary_key) > 0 and all(isinstance(k, str) for k in stream_primary_key): + return stream_primary_key # type: ignore # We verified all items in the list are strings + else: + raise ValueError(f"Nested primary keys are not supported. Found {stream_primary_key}") + else: + raise ValueError(f"Invalid type for primary key: {stream_primary_key}") + + +def get_cursor_field_from_stream(stream: Stream) -> Optional[str]: + if isinstance(stream.cursor_field, list): + if len(stream.cursor_field) > 1: + raise ValueError(f"Nested cursor fields are not supported. Got {stream.cursor_field} for {stream.name}") + elif len(stream.cursor_field) == 0: + return None + else: + return stream.cursor_field[0] + else: + return stream.cursor_field diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py index 138bb9cf86b7..3869c6cf9e73 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py @@ -1,10 +1,11 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import time from queue import Queue from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem @@ -14,26 +15,40 @@ class PartitionEnqueuer: Generates partitions from a partition generator and puts them in a queue. """ - def __init__(self, queue: Queue[QueueItem]) -> None: + def __init__(self, queue: Queue[QueueItem], thread_pool_manager: ThreadPoolManager, sleep_time_in_seconds: float = 0.1) -> None: """ :param queue: The queue to put the partitions in. - :param sentinel: The sentinel to put in the queue when all the partitions have been generated. + :param throttler: The throttler to use to throttle the partition generation. """ self._queue = queue + self._thread_pool_manager = thread_pool_manager + self._sleep_time_in_seconds = sleep_time_in_seconds def generate_partitions(self, stream: AbstractStream) -> None: """ Generate partitions from a partition generator and put them in a queue. When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. - If an exception is encountered, the exception will be caught and put in the queue. + If an exception is encountered, the exception will be caught and put in the queue. This is very important because if we don't, the + main thread will have no way to know that something when wrong and will wait until the timeout is reached This method is meant to be called in a separate thread. - :param partition_generator: The partition Generator - :return: """ try: for partition in stream.generate_partitions(): + # Adding partitions to the queue generates futures. To avoid having too many futures, we throttle here. We understand that + # we might add more futures than the limit by throttling in the threads while it is the main thread that actual adds the + # future but we expect the delta between the max futures length and the actual to be small enough that it would not be an + # issue. We do this in the threads because we want the main thread to always be processing QueueItems as if it does not, the + # queue size could grow and generating OOM issues. + # + # Also note that we do not expect this to create deadlocks where all worker threads wait because we have less + # PartitionEnqueuer threads than worker threads. + # + # Also note that prune_to_validate_has_reached_futures_limit has a lock while pruning which might create a bottleneck in + # terms of performance. + while self._thread_pool_manager.prune_to_validate_has_reached_futures_limit(): + time.sleep(self._sleep_time_in_seconds) self._queue.put(partition) self._queue.put(PartitionGenerationCompletedSentinel(stream)) except Exception as e: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py index 0bc9c35117a6..3df19ca29f92 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py @@ -1,7 +1,6 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from queue import Queue from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition @@ -24,7 +23,8 @@ def process_partition(self, partition: Partition) -> None: Process a partition and put the records in the output queue. When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. - If an exception is encountered, the exception will be caught and put in the queue. + If an exception is encountered, the exception will be caught and put in the queue. This is very important because if we don't, the + main thread will have no way to know that something when wrong and will wait until the timeout is reached This method is meant to be called from a thread. :param partition: The partition to read data from diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converter.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converter.py deleted file mode 100644 index a384e030af60..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converter.py +++ /dev/null @@ -1,143 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, List, MutableMapping, Optional - - -class ConcurrencyCompatibleStateType(Enum): - date_range = "date-range" - - -class ConcurrentStreamStateConverter(ABC): - START_KEY = "start" - END_KEY = "end" - - def get_concurrent_stream_state(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - if self.is_state_message_compatible(state): - return state - return self.convert_from_sequential_state(state) - - @staticmethod - def is_state_message_compatible(state: MutableMapping[str, Any]) -> bool: - return state.get("state_type") in [t.value for t in ConcurrencyCompatibleStateType] - - @abstractmethod - def convert_from_sequential_state(self, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """ - Convert the state message to the format required by the ThreadBasedConcurrentStream. - - e.g. - { - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "metadata": { … }, - "slices": [ - {starts: 0, end: 1617030403, finished_processing: true}] - } - """ - ... - - @abstractmethod - def convert_to_sequential_state(self, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """ - Convert the state message from the concurrency-compatible format to the stream's original format. - - e.g. - { "created": 1617030403 } - """ - ... - - def _get_latest_complete_time(self, slices: List[MutableMapping[str, Any]]) -> Optional[Any]: - """ - Get the latest time before which all records have been processed. - """ - if slices: - first_interval = self.merge_intervals(slices)[0][self.END_KEY] - return first_interval - else: - return None - - @staticmethod - @abstractmethod - def increment(timestamp: Any) -> Any: - """ - Increment a timestamp by a single unit. - """ - ... - - @classmethod - def merge_intervals(cls, intervals: List[MutableMapping[str, Any]]) -> List[MutableMapping[str, Any]]: - sorted_intervals = sorted(intervals, key=lambda x: (x[cls.START_KEY], x[cls.END_KEY])) - if len(sorted_intervals) > 0: - merged_intervals = [sorted_intervals[0]] - else: - return [] - for interval in sorted_intervals[1:]: - if interval[cls.START_KEY] <= cls.increment(merged_intervals[-1][cls.END_KEY]): - merged_intervals[-1][cls.END_KEY] = interval[cls.END_KEY] - else: - merged_intervals.append(interval) - - return merged_intervals - - -class EpochValueConcurrentStreamStateConverter(ConcurrentStreamStateConverter): - def __init__(self, cursor_field: str): - self._cursor_field = cursor_field - - def convert_from_sequential_state(self, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """ - e.g. - { "created": 1617030403 } - => - { - "state_type": "date-range", - "metadata": { … }, - "slices": [ - {starts: 0, end: 1617030403, finished_processing: true} - ] - } - """ - if self.is_state_message_compatible(stream_state): - return stream_state - if self._cursor_field in stream_state: - slices = [ - { - self.START_KEY: 0, - self.END_KEY: stream_state[self._cursor_field], - }, - ] - else: - slices = [] - return { - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "slices": slices, - "legacy": stream_state, - } - - def convert_to_sequential_state(self, stream_state: MutableMapping[str, Any]) -> Any: - """ - e.g. - { - "state_type": "date-range", - "metadata": { … }, - "slices": [ - {starts: 0, end: 1617030403, finished_processing: true} - ] - } - => - { "created": 1617030403 } - """ - if self.is_state_message_compatible(stream_state): - legacy_state = stream_state.get("legacy", {}) - if slices := stream_state.pop("slices", None): - legacy_state.update({self._cursor_field: self._get_latest_complete_time(slices)}) - return legacy_state - else: - return stream_state - - @staticmethod - def increment(timestamp: Any) -> Any: - return timestamp + 1 diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json rename to airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py new file mode 100644 index 000000000000..843f477ddb16 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Any, List, MutableMapping, Tuple + +if TYPE_CHECKING: + from airbyte_cdk.sources.streams.concurrent.cursor import CursorField + + +class ConcurrencyCompatibleStateType(Enum): + date_range = "date-range" + + +class AbstractStreamStateConverter(ABC): + START_KEY = "start" + END_KEY = "end" + + @abstractmethod + def deserialize(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Perform any transformations needed for compatibility with the converter. + """ + ... + + @staticmethod + def is_state_message_compatible(state: MutableMapping[str, Any]) -> bool: + return bool(state) and state.get("state_type") in [t.value for t in ConcurrencyCompatibleStateType] + + @abstractmethod + def convert_from_sequential_state( + self, + cursor_field: "CursorField", + stream_state: MutableMapping[str, Any], + start: Any, + ) -> Tuple[Any, MutableMapping[str, Any]]: + """ + Convert the state message to the format required by the ConcurrentCursor. + + e.g. + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "metadata": { … }, + "slices": [ + {starts: 0, end: 1617030403, finished_processing: true}] + } + """ + ... + + @abstractmethod + def convert_to_sequential_state(self, cursor_field: "CursorField", stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Convert the state message from the concurrency-compatible format to the stream's original format. + + e.g. + { "created": 1617030403 } + """ + ... + + @abstractmethod + def increment(self, timestamp: Any) -> Any: + """ + Increment a timestamp by a single unit. + """ + ... + + @abstractmethod + def merge_intervals(self, intervals: List[MutableMapping[str, Any]]) -> List[MutableMapping[str, Any]]: + """ + Compute and return a list of merged intervals. + + Intervals may be merged if the start time of the second interval is 1 unit or less (as defined by the + `increment` method) than the end time of the first interval. + """ + ... + + @abstractmethod + def parse_value(self, value: Any) -> Any: + """ + Parse the value of the cursor field into a comparable value. + """ + ... + + @property + @abstractmethod + def zero_value(self) -> Any: + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py new file mode 100644 index 000000000000..83f8a44b23db --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py @@ -0,0 +1,196 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import abstractmethod +from datetime import datetime, timedelta +from typing import Any, List, MutableMapping, Optional, Tuple + +import pendulum +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ( + AbstractStreamStateConverter, + ConcurrencyCompatibleStateType, +) +from pendulum.datetime import DateTime + + +class DateTimeStreamStateConverter(AbstractStreamStateConverter): + @property + @abstractmethod + def _zero_value(self) -> Any: + ... + + @property + def zero_value(self) -> datetime: + return self.parse_timestamp(self._zero_value) + + @abstractmethod + def increment(self, timestamp: datetime) -> datetime: + ... + + @abstractmethod + def parse_timestamp(self, timestamp: Any) -> datetime: + ... + + @abstractmethod + def output_format(self, timestamp: datetime) -> Any: + ... + + def deserialize(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + for stream_slice in state.get("slices", []): + stream_slice[self.START_KEY] = self.parse_timestamp(stream_slice[self.START_KEY]) + stream_slice[self.END_KEY] = self.parse_timestamp(stream_slice[self.END_KEY]) + return state + + def parse_value(self, value: Any) -> Any: + """ + Parse the value of the cursor field into a comparable value. + """ + return self.parse_timestamp(value) + + def merge_intervals(self, intervals: List[MutableMapping[str, datetime]]) -> List[MutableMapping[str, datetime]]: + if not intervals: + return [] + + sorted_intervals = sorted(intervals, key=lambda x: (x[self.START_KEY], x[self.END_KEY])) + merged_intervals = [sorted_intervals[0]] + + for interval in sorted_intervals[1:]: + last_end_time = merged_intervals[-1][self.END_KEY] + current_start_time = interval[self.START_KEY] + if self._compare_intervals(last_end_time, current_start_time): + merged_end_time = max(last_end_time, interval[self.END_KEY]) + merged_intervals[-1][self.END_KEY] = merged_end_time + else: + merged_intervals.append(interval) + + return merged_intervals + + def _compare_intervals(self, end_time: Any, start_time: Any) -> bool: + return bool(self.increment(end_time) >= start_time) + + def convert_from_sequential_state( + self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: datetime + ) -> Tuple[datetime, MutableMapping[str, Any]]: + """ + Convert the state message to the format required by the ConcurrentCursor. + + e.g. + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "metadata": { … }, + "slices": [ + {"start": "2021-01-18T21:18:20.000+00:00", "end": "2021-01-18T21:18:20.000+00:00"}, + ] + } + """ + sync_start = self._get_sync_start(cursor_field, stream_state, start) + if self.is_state_message_compatible(stream_state): + return sync_start, stream_state + + # Create a slice to represent the records synced during prior syncs. + # The start and end are the same to avoid confusion as to whether the records for this slice + # were actually synced + slices = [{self.START_KEY: sync_start, self.END_KEY: sync_start}] + + return sync_start, { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "slices": slices, + "legacy": stream_state, + } + + def _get_sync_start(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: Optional[Any]) -> datetime: + sync_start = self.parse_timestamp(start) if start is not None else self.zero_value + prev_sync_low_water_mark = ( + self.parse_timestamp(stream_state[cursor_field.cursor_field_key]) if cursor_field.cursor_field_key in stream_state else None + ) + if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start: + return prev_sync_low_water_mark + else: + return sync_start + + def convert_to_sequential_state(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Convert the state message from the concurrency-compatible format to the stream's original format. + + e.g. + { "created": "2021-01-18T21:18:20.000Z" } + """ + if self.is_state_message_compatible(stream_state): + legacy_state = stream_state.get("legacy", {}) + latest_complete_time = self._get_latest_complete_time(stream_state.get("slices", [])) + if latest_complete_time is not None: + legacy_state.update({cursor_field.cursor_field_key: self.output_format(latest_complete_time)}) + return legacy_state or {} + else: + return stream_state + + def _get_latest_complete_time(self, slices: List[MutableMapping[str, Any]]) -> Optional[datetime]: + """ + Get the latest time before which all records have been processed. + """ + if not slices: + raise RuntimeError("Expected at least one slice but there were none. This is unexpected; please contact Support.") + + merged_intervals = self.merge_intervals(slices) + first_interval = merged_intervals[0] + return first_interval[self.END_KEY] + + +class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter): + """ + e.g. + { "created": 1617030403 } + => + { + "state_type": "date-range", + "metadata": { … }, + "slices": [ + {starts: 0, end: 1617030403, finished_processing: true} + ] + } + """ + + _zero_value = 0 + + def increment(self, timestamp: datetime) -> datetime: + return timestamp + timedelta(seconds=1) + + def output_format(self, timestamp: datetime) -> int: + return int(timestamp.timestamp()) + + def parse_timestamp(self, timestamp: int) -> datetime: + dt_object = pendulum.from_timestamp(timestamp) + if not isinstance(dt_object, DateTime): + raise ValueError(f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})") + return dt_object # type: ignore # we are manually type checking because pendulum.parse may return different types + + +class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter): + """ + e.g. + { "created": "2021-01-18T21:18:20.000Z" } + => + { + "state_type": "date-range", + "metadata": { … }, + "slices": [ + {starts: "2020-01-18T21:18:20.000Z", end: "2021-01-18T21:18:20.000Z", finished_processing: true} + ] + } + """ + + _zero_value = "0001-01-01T00:00:00.000Z" + + def increment(self, timestamp: datetime) -> datetime: + return timestamp + timedelta(milliseconds=1) + + def output_format(self, timestamp: datetime) -> Any: + return timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + def parse_timestamp(self, timestamp: str) -> datetime: + dt_object = pendulum.parse(timestamp) + if not isinstance(dt_object, DateTime): + raise ValueError(f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})") + return dt_object # type: ignore # we are manually type checking because pendulum.parse may return different types diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 22e2caa6a2e8..63915f71d651 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.http_logger import format_http_message from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from airbyte_cdk.utils import AirbyteTracedException +from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets from requests.auth import AuthBase from ..exceptions import DefaultBackoffException @@ -45,7 +46,7 @@ def __init__( self._refresh_token_error_key = refresh_token_error_key self._refresh_token_error_values = refresh_token_error_values - def __call__(self, request: requests.Request) -> requests.Request: + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: """Attach the HTTP headers required to authenticate on the HTTP request""" request.headers.update(self.get_auth_header()) return request @@ -65,7 +66,7 @@ def get_access_token(self) -> str: def token_has_expired(self) -> bool: """Returns True if the token is expired""" - return pendulum.now() > self.get_token_expiry_date() + return pendulum.now() > self.get_token_expiry_date() # type: ignore # this is always a bool despite what mypy thinks def build_refresh_request_body(self) -> Mapping[str, Any]: """ @@ -80,7 +81,7 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: "refresh_token": self.get_refresh_token(), } - if self.get_scopes: + if self.get_scopes(): payload["scopes"] = self.get_scopes() if self.get_refresh_request_body(): @@ -93,7 +94,10 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: def _wrap_refresh_token_exception(self, exception: requests.exceptions.RequestException) -> bool: try: - exception_content = exception.response.json() + if exception.response is not None: + exception_content = exception.response.json() + else: + return False except JSONDecodeError: return False return ( @@ -109,15 +113,27 @@ def _wrap_refresh_token_exception(self, exception: requests.exceptions.RequestEx ), max_time=300, ) - def _get_refresh_access_token_response(self): + def _get_refresh_access_token_response(self) -> Any: try: response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body()) - self._log_response(response) - response.raise_for_status() - return response.json() + if response.ok: + response_json = response.json() + # Add the access token to the list of secrets so it is replaced before logging the response + # An argument could be made to remove the prevous access key from the list of secrets, but unmasking values seems like a security incident waiting to happen... + access_key = response_json.get(self.get_access_token_name()) + if not access_key: + raise Exception("Token refresh API response was missing access token {self.get_access_token_name()}") + add_to_secrets(access_key) + self._log_response(response) + return response_json + else: + # log the response even if the request failed for troubleshooting purposes + self._log_response(response) + response.raise_for_status() except requests.exceptions.RequestException as e: - if e.response.status_code == 429 or e.response.status_code >= 500: - raise DefaultBackoffException(request=e.response.request, response=e.response) + if e.response is not None: + if e.response.status_code == 429 or e.response.status_code >= 500: + raise DefaultBackoffException(request=e.response.request, response=e.response) if self._wrap_refresh_token_exception(e): message = "Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." raise AirbyteTracedException(internal_message=message, message=message, failure_type=FailureType.config_error) @@ -147,7 +163,7 @@ def _parse_token_expiration_date(self, value: Union[str, int]) -> pendulum.DateT raise ValueError( f"Invalid token expiry date format {self.token_expiry_date_format}; a string representing the format is required." ) - return pendulum.from_format(value, self.token_expiry_date_format) + return pendulum.from_format(str(value), self.token_expiry_date_format) else: return pendulum.now().add(seconds=int(float(value))) @@ -192,7 +208,7 @@ def get_token_expiry_date(self) -> pendulum.DateTime: """Expiration date of the access token""" @abstractmethod - def set_token_expiry_date(self, value: Union[str, int]): + def set_token_expiry_date(self, value: Union[str, int]) -> None: """Setter for access token expiration date""" @abstractmethod @@ -228,14 +244,15 @@ def _message_repository(self) -> Optional[MessageRepository]: """ return _NOOP_MESSAGE_REPOSITORY - def _log_response(self, response: requests.Response): - self._message_repository.log_message( - Level.DEBUG, - lambda: format_http_message( - response, - "Refresh token", - "Obtains access token", - self._NO_STREAM_NAME, - is_auxiliary=True, - ), - ) + def _log_response(self, response: requests.Response) -> None: + if self._message_repository: + self._message_repository.log_message( + Level.DEBUG, + lambda: format_http_message( + response, + "Refresh token", + "Obtains access token", + self._NO_STREAM_NAME, + is_auxiliary=True, + ), + ) diff --git a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py index 6d7def3915ea..612b2742ea1e 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py +++ b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py @@ -26,7 +26,16 @@ from airbyte_cdk.exception_handler import assemble_uncaught_exception from airbyte_cdk.logger import AirbyteLogFormatter from airbyte_cdk.sources import Source -from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, AirbyteStreamStatus, ConfiguredAirbyteCatalog, Level, TraceType, Type +from airbyte_protocol.models import ( + AirbyteLogMessage, + AirbyteMessage, + AirbyteStateMessage, + AirbyteStreamStatus, + ConfiguredAirbyteCatalog, + Level, + TraceType, + Type, +) from pydantic.error_wrappers import ValidationError @@ -60,6 +69,13 @@ def records(self) -> List[AirbyteMessage]: def state_messages(self) -> List[AirbyteMessage]: return self._get_message_by_types([Type.STATE]) + @property + def most_recent_state(self) -> Any: + state_messages = self._get_message_by_types([Type.STATE]) + if not state_messages: + raise ValueError("Can't provide most recent state as there are no state messages") + return state_messages[-1].state.data + @property def logs(self) -> List[AirbyteMessage]: return self._get_message_by_types([Type.LOG]) @@ -97,7 +113,7 @@ def read( source: Source, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, - state: Optional[Any] = None, + state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False, ) -> EntrypointOutput: """ @@ -122,13 +138,14 @@ def read( "--catalog", make_file(tmp_directory_path / "catalog.json", catalog.json()), ] - if state: + if state is not None: args.extend( [ "--state", - make_file(tmp_directory_path / "state.json", state), + make_file(tmp_directory_path / "state.json", f"[{','.join([stream_state.json() for stream_state in state])}]"), ] ) + args.append("--debug") source_entrypoint = AirbyteEntrypoint(source) parsed_args = source_entrypoint.parse_args(args) diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py index d6af705369b6..441a765b7321 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py @@ -7,16 +7,20 @@ class HttpRequestMatcher: def __init__(self, request: HttpRequest, minimum_number_of_expected_match: int): self._request_to_match = request self._minimum_number_of_expected_match = minimum_number_of_expected_match - self._actual_number_of_match = 0 + self._actual_number_of_matches = 0 def matches(self, request: HttpRequest) -> bool: hit = request.matches(self._request_to_match) if hit: - self._actual_number_of_match += 1 + self._actual_number_of_matches += 1 return hit def has_expected_match_count(self) -> bool: - return self._actual_number_of_match >= self._minimum_number_of_expected_match + return self._actual_number_of_matches >= self._minimum_number_of_expected_match + + @property + def actual_number_of_matches(self) -> int: + return self._actual_number_of_matches @property def request(self) -> HttpRequest: @@ -27,5 +31,5 @@ def __str__(self) -> str: f"HttpRequestMatcher(" f"request_to_match={self._request_to_match}, " f"minimum_number_of_expected_match={self._minimum_number_of_expected_match}, " - f"actual_number_of_match={self._actual_number_of_match})" + f"actual_number_of_matches={self._actual_number_of_matches})" ) diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py index 8ed58119c1f6..5fcf6692056d 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py @@ -2,6 +2,7 @@ import contextlib import functools +from enum import Enum from types import TracebackType from typing import Callable, List, Optional, Union @@ -9,9 +10,26 @@ from airbyte_cdk.test.mock_http import HttpRequest, HttpRequestMatcher, HttpResponse +class SupportedHttpMethods(str, Enum): + GET = "get" + POST = "post" + + class HttpMocker(contextlib.ContextDecorator): """ - WARNING: This implementation only works if the lib used to perform HTTP requests is `requests` + WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`. + + WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios + where the same request is added twice (in which case there will always be an exception because we will never match the second + request) or in a case like this: + ``` + http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>) + http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>) + requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"}) + ``` + In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see + https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even + though the request sent is a better match for the first `http_mocker.get`. """ def __init__(self) -> None: @@ -30,40 +48,49 @@ def _validate_all_matchers_called(self) -> None: if not matcher.has_expected_match_count(): raise ValueError(f"Invalid number of matches for `{matcher}`") - def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None: - """ - WARNING: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios - where the same request is added twice (in which case there will always be an exception because we will never match the second - request) or in a case like this: - ``` - http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>) - http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>) - requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"}) - ``` - In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see - https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even - though the request sent is a better match for the first `http_mocker.get`. - """ + def _mock_request_method( + self, method: SupportedHttpMethods, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]] + ) -> None: if isinstance(responses, HttpResponse): responses = [responses] matcher = HttpRequestMatcher(request, len(responses)) self._matchers.append(matcher) - self._mocker.get( + + getattr(self._mocker, method)( requests_mock.ANY, additional_matcher=self._matches_wrapper(matcher), - response_list=[{"text": response.body, "status_code": response.status_code} for response in responses], + response_list=[ + {"text": response.body, "status_code": response.status_code, "headers": response.headers} for response in responses + ], ) - def _matches_wrapper(self, matcher: HttpRequestMatcher) -> Callable[[requests_mock.request._RequestObjectProxy], bool]: + def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None: + self._mock_request_method(SupportedHttpMethods.GET, request, responses) + + def post(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None: + self._mock_request_method(SupportedHttpMethods.POST, request, responses) + + @staticmethod + def _matches_wrapper(matcher: HttpRequestMatcher) -> Callable[[requests_mock.request._RequestObjectProxy], bool]: def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool: # query_params are provided as part of `requests_mock_request.url` - http_request = HttpRequest(requests_mock_request.url, query_params={}, headers=requests_mock_request.headers) + http_request = HttpRequest( + requests_mock_request.url, query_params={}, headers=requests_mock_request.headers, body=requests_mock_request.body + ) return matcher.matches(http_request) return matches - def __call__(self, f): # type: ignore # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` + def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None: + corresponding_matchers = list(filter(lambda matcher: matcher.request == request, self._matchers)) + if len(corresponding_matchers) != 1: + raise ValueError(f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`") + + assert corresponding_matchers[0].actual_number_of_matches == number_of_calls + + # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` + def __call__(self, f): # type: ignore @functools.wraps(f) def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed with self: @@ -75,14 +102,22 @@ def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper except requests_mock.NoMockAddress as no_mock_exception: matchers_as_string = "\n\t".join(map(lambda matcher: str(matcher.request), self._matchers)) raise ValueError( - f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}`. Matchers currently configured are:\n\t{matchers_as_string}" + f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` " + f"and body `{no_mock_exception.request.body}`. " + f"Matchers currently configured are:\n\t{matchers_as_string}." ) from no_mock_exception except AssertionError as test_assertion: assertion_error = test_assertion - # We validate the matchers before raising the assertion error because we want to show the tester if a HTTP request wasn't + # We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't # mocked correctly - self._validate_all_matchers_called() + try: + self._validate_all_matchers_called() + except ValueError as http_mocker_exception: + # This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in + # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)` + # like we do here provides additional context for the exception. + raise ValueError(http_mocker_exception) from None if assertion_error: raise assertion_error return result diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py index 243701280de0..a2b6bdb9430a 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py @@ -1,5 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import json from typing import Any, List, Mapping, Optional, Union from urllib.parse import parse_qs, urlencode, urlparse @@ -16,6 +17,7 @@ def __init__( url: str, query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None, headers: Optional[Mapping[str, str]] = None, + body: Optional[Union[str, bytes, Mapping[str, Any]]] = None, ) -> None: self._parsed_url = urlparse(url) self._query_params = query_params @@ -25,31 +27,61 @@ def __init__( raise ValueError("If query params are provided as part of the url, `query_params` should be empty") self._headers = headers or {} + self._body = body - def _encode_qs(self, query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str: + @staticmethod + def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str: if isinstance(query_params, str): return query_params return urlencode(query_params, doseq=True) def matches(self, other: Any) -> bool: """ - Note that headers only need to be a subset of `other` in order to match + If the body of any request is a Mapping, we compare as Mappings which means that the order is not important. + If the body is a string, encoding ISO-8859-1 will be assumed + Headers only need to be a subset of `other` in order to match """ if isinstance(other, HttpRequest): + # if `other` is a mapping, we match as an object and formatting is not considers + if isinstance(self._body, Mapping) or isinstance(other._body, Mapping): + body_match = self._to_mapping(self._body) == self._to_mapping(other._body) + else: + body_match = self._to_bytes(self._body) == self._to_bytes(other._body) + return ( self._parsed_url.scheme == other._parsed_url.scheme and self._parsed_url.hostname == other._parsed_url.hostname and self._parsed_url.path == other._parsed_url.path and ( - ANY_QUERY_PARAMS in [self._query_params, other._query_params] + ANY_QUERY_PARAMS in (self._query_params, other._query_params) or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query) ) and _is_subdict(other._headers, self._headers) + and body_match ) return False + @staticmethod + def _to_mapping(body: Optional[Union[str, bytes, Mapping[str, Any]]]) -> Optional[Mapping[str, Any]]: + if isinstance(body, Mapping): + return body + elif isinstance(body, bytes): + return json.loads(body.decode()) # type: ignore # assumes return type of Mapping[str, Any] + elif isinstance(body, str): + return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any] + return None + + @staticmethod + def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes: + if isinstance(body, bytes): + return body + elif isinstance(body, str): + # `ISO-8859-1` is the default encoding used by requests + return body.encode("ISO-8859-1") + return b"" + def __str__(self) -> str: - return f"{self._parsed_url} with headers {self._headers})" + return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})" def __repr__(self) -> str: - return f"HttpRequest(request={self._parsed_url}, headers={self._headers})" + return f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})" diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py index 3aea355cd4e0..8d5dc4c308da 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py @@ -1,10 +1,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from types import MappingProxyType +from typing import Mapping + class HttpResponse: - def __init__(self, body: str, status_code: int = 200): + def __init__(self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})): self._body = body self._status_code = status_code + self._headers = headers @property def body(self) -> str: @@ -13,3 +17,7 @@ def body(self) -> str: @property def status_code(self) -> int: return self._status_code + + @property + def headers(self) -> Mapping[str, str]: + return self._headers diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py index 1fcefb7793bb..02dd3d285107 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py @@ -4,7 +4,7 @@ import json from abc import ABC, abstractmethod from pathlib import Path as FilePath -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union from airbyte_cdk.test.mock_http import HttpResponse @@ -20,7 +20,18 @@ def _replace_value(dictionary: Dict[str, Any], path: List[str], value: Any) -> N current[path[-1]] = value +def _write(dictionary: Dict[str, Any], path: List[str], value: Any) -> None: + current = dictionary + for key in path[:-1]: + current = current.setdefault(key, {}) + current[path[-1]] = value + + class Path(ABC): + @abstractmethod + def write(self, template: Dict[str, Any], value: Any) -> None: + pass + @abstractmethod def update(self, template: Dict[str, Any], value: Any) -> None: pass @@ -33,6 +44,9 @@ class FieldPath(Path): def __init__(self, field: str): self._path = [field] + def write(self, template: Dict[str, Any], value: Any) -> None: + _write(template, self._path, value) + def update(self, template: Dict[str, Any], value: Any) -> None: _replace_value(template, self._path, value) @@ -47,6 +61,9 @@ class NestedPath(Path): def __init__(self, path: List[str]): self._path = path + def write(self, template: Dict[str, Any], value: Any) -> None: + _write(template, self._path, value) + def update(self, template: Dict[str, Any], value: Any) -> None: _replace_value(template, self._path, value) @@ -103,6 +120,10 @@ def with_cursor(self, cursor_value: Any) -> "RecordBuilder": self._set_field("cursor", self._cursor_path, cursor_value) return self + def with_field(self, path: Path, value: Any) -> "RecordBuilder": + path.write(self._record, value) + return self + def _set_field(self, field_name: str, path: Optional[Path], value: Any) -> None: if not path: raise ValueError( @@ -165,27 +186,28 @@ def find_template(resource: str, execution_folder: str) -> Dict[str, Any]: return json.load(template_file) # type: ignore # we assume the dev correctly set up the resource file -def create_builders_from_resource( +def create_record_builder( response_template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], record_id_path: Optional[Path] = None, record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None, - pagination_strategy: Optional[PaginationStrategy] = None -) -> Tuple[RecordBuilder, HttpResponseBuilder]: +) -> RecordBuilder: """ This will use the first record define at `records_path` as a template for the records. If more records are defined, they will be ignored """ - if not isinstance(records_path, (FieldPath, NestedPath)): - raise ValueError(f"records_path only supports FieldPath and NestedPath but {type(records_path)} was provided") - try: record_template = records_path.extract(response_template)[0] if not record_template: raise ValueError(f"Could not extract any record from template at path `{records_path}`. " f"Please fix the template to provide a record sample or fix `records_path`.") - return ( - RecordBuilder(record_template, record_id_path, record_cursor_path), - HttpResponseBuilder(response_template, records_path, pagination_strategy) - ) + return RecordBuilder(record_template, record_id_path, record_cursor_path) except (IndexError, KeyError): raise ValueError(f"Error while extracting records at path `{records_path}` from response template `{response_template}`") + + +def create_response_builder( + response_template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + pagination_strategy: Optional[PaginationStrategy] = None +) -> HttpResponseBuilder: + return HttpResponseBuilder(response_template, records_path, pagination_strategy) diff --git a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py new file mode 100644 index 000000000000..1c356afef889 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, List + +from airbyte_protocol.models import AirbyteStateMessage + + +class StateBuilder: + def __init__(self) -> None: + self._state: List[AirbyteStateMessage] = [] + + def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": + self._state.append(AirbyteStateMessage.parse_obj({ + "type": "STREAM", + "stream": { + "stream_state": state, + "stream_descriptor": { + "name": stream_name + } + } + })) + return self + + def build(self) -> List[AirbyteStateMessage]: + return self._state diff --git a/airbyte-cdk/python/airbyte_cdk/utils/airbyte_secrets_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/airbyte_secrets_utils.py index eb04a6cf891f..e690a556606b 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/airbyte_secrets_utils.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/airbyte_secrets_utils.py @@ -10,7 +10,7 @@ def get_secret_paths(spec: Mapping[str, Any]) -> List[List[str]]: paths = [] - def traverse_schema(schema_item: Any, path: List[str]): + def traverse_schema(schema_item: Any, path: List[str]) -> None: """ schema_item can be any property or value in the originally input jsonschema, depending on how far down the recursion stack we go path is the path to that schema item in the original input @@ -56,12 +56,18 @@ def get_secrets(connection_specification: Mapping[str, Any], config: Mapping[str __SECRETS_FROM_CONFIG: List[str] = [] -def update_secrets(secrets: List[str]): +def update_secrets(secrets: List[str]) -> None: """Update the list of secrets to be replaced""" global __SECRETS_FROM_CONFIG __SECRETS_FROM_CONFIG = secrets +def add_to_secrets(secret: str) -> None: + """Add to the list of secrets to be replaced""" + global __SECRETS_FROM_CONFIG + __SECRETS_FROM_CONFIG.append(secret) + + def filter_secrets(string: str) -> str: """Filter secrets from a string by replacing them with ****""" # TODO this should perform a maximal match for each secret. if "x" and "xk" are both secret values, and this method is called twice on diff --git a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py index dec09fcf1929..753296a5dd74 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py @@ -13,6 +13,7 @@ AirbyteTraceMessage, FailureType, Status, + StreamDescriptor, TraceType, ) from airbyte_cdk.models import Type as MessageType @@ -43,7 +44,7 @@ def __init__( self._exception = exception super().__init__(internal_message) - def as_airbyte_message(self) -> AirbyteMessage: + def as_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage: """ Builds an AirbyteTraceMessage from the exception """ @@ -60,6 +61,7 @@ def as_airbyte_message(self) -> AirbyteMessage: internal_message=self.internal_message, failure_type=self.failure_type, stack_trace=stack_trace_str, + stream_descriptor=stream_descriptor, ), ) @@ -88,3 +90,16 @@ def from_exception(cls, exc: BaseException, *args, **kwargs) -> "AirbyteTracedEx :param exc: the exception that caused the error """ return cls(internal_message=str(exc), exception=exc, *args, **kwargs) # type: ignore # ignoring because of args and kwargs + + def as_sanitized_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage: + """ + Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body + """ + error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor) + if error_message.trace.error.message: + error_message.trace.error.message = filter_secrets(error_message.trace.error.message) + if error_message.trace.error.internal_message: + error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message) + if error_message.trace.error.stack_trace: + error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace) + return error_message diff --git a/airbyte-cdk/python/bin/build_code_generator_image.sh b/airbyte-cdk/python/bin/build_code_generator_image.sh new file mode 100755 index 000000000000..f73c318317c5 --- /dev/null +++ b/airbyte-cdk/python/bin/build_code_generator_image.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +DOCKER_BUILD_ARCH="${DOCKER_BUILD_ARCH:-amd64}" +# https://docs.docker.com/develop/develop-images/build_enhancements/ +export DOCKER_BUILDKIT=1 + +CODE_GENERATOR_DOCKERFILE="$(dirname $0)/../code-generator/Dockerfile" +test -f $CODE_GENERATOR_DOCKERFILE +docker build --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" -t "airbyte/code-generator:dev" - < $CODE_GENERATOR_DOCKERFILE diff --git a/airbyte-cdk/python/bin/generate-component-manifest-files.sh b/airbyte-cdk/python/bin/generate-component-manifest-files.sh index a1939345e649..d366d3ca9cde 100755 --- a/airbyte-cdk/python/bin/generate-component-manifest-files.sh +++ b/airbyte-cdk/python/bin/generate-component-manifest-files.sh @@ -19,7 +19,8 @@ function main() { --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ --disable-timestamp \ - --enum-field-as-literal one + --enum-field-as-literal one \ + --set-default-enum-member # There is a limitation of Pydantic where a model's private fields starting with an underscore are inaccessible. # The Pydantic model generator replaces special characters like $ with the underscore which results in all diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index 63cc9992a73b..61f355742382 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -1,25 +1,134 @@ +import ru.vyarus.gradle.plugin.python.task.PythonTask + plugins { - id 'airbyte-python' - id 'airbyte-docker-legacy' + id 'base' + id 'ru.vyarus.use-python' version '2.3.0' } +def generateCodeGeneratorImage = tasks.register('generateCodeGeneratorImage', Exec) { + commandLine 'bin/build_code_generator_image.sh' +} def generateComponentManifestClassFiles = tasks.register('generateComponentManifestClassFiles', Exec) { - environment 'ROOT_DIR', rootDir.absolutePath + environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath commandLine 'bin/generate-component-manifest-files.sh' } generateComponentManifestClassFiles.configure { - dependsOn project(':tools:code-generator').tasks.named('assemble') + dependsOn generateCodeGeneratorImage } tasks.register('generate').configure { dependsOn generateComponentManifestClassFiles } tasks.register('validateSourceYamlManifest', Exec) { - environment 'ROOT_DIR', rootDir.absolutePath + environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath commandLine 'bin/validate-yaml-schema.sh' } tasks.register('runLowCodeConnectorUnitTests', Exec) { - environment 'ROOT_DIR', rootDir.absolutePath + environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath commandLine 'bin/low-code-unit-tests.sh' } + +def venvDirectoryName = '.venv' + +// Add a task that allows cleaning up venvs to every python project +def cleanPythonVenv = tasks.register('cleanPythonVenv', Exec) { + commandLine 'rm' + args '-rf', "${projectDir.absolutePath}/${venvDirectoryName}" +} + +tasks.named('clean').configure { + dependsOn cleanPythonVenv +} + +// Configure gradle python plugin. +python { + envPath = venvDirectoryName + minPythonVersion '3.10' + + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } + // Pyenv support. + try { + def pyenvRoot = "pyenv root".execute() + def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() + // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. + pythonBinary "python" + if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { + pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" + } + } catch (IOException _) { + // Swallow exception if pyenv is not installed. + } + + scope 'VIRTUALENV' + installVirtualenv = true + pip 'pip:23.2.1' + pip 'mccabe:0.6.1' + // https://github.com/csachs/pyproject-flake8/issues/13 + pip 'flake8:4.0.1' + // flake8 doesn't support pyproject.toml files + // and thus there is the wrapper "pyproject-flake8" for this + pip 'pyproject-flake8:0.0.1a2' + pip 'pytest:6.2.5' + pip 'coverage[toml]:6.3.1' +} + +def installLocalReqs = tasks.register('installLocalReqs', PythonTask) { + module = "pip" + command = "install .[dev,tests]" + inputs.file('setup.py') + outputs.file('build/installedlocalreqs.txt') +} + +def flakeCheck = tasks.register('flakeCheck', PythonTask) { + module = "pflake8" + command = "--config pyproject.toml ./" +} + +def installReqs = tasks.register('installReqs', PythonTask) { + module = "pip" + command = "install .[main]" + inputs.file('setup.py') + outputs.file('build/installedreqs.txt') +} +installReqs.configure { + dependsOn installLocalReqs +} + +tasks.named('check').configure { + dependsOn installReqs + dependsOn flakeCheck +} + +def installTestReqs = tasks.register('installTestReqs', PythonTask) { + module = "pip" + command = "install .[tests]" + inputs.file('setup.py') + outputs.file('build/installedtestreqs.txt') +} +installTestReqs.configure { + dependsOn installReqs +} + +def testTask = tasks.register('testPython', PythonTask) { + module = "coverage" + command = "run --data-file=unit_tests/.coverage.testPython --rcfile=pyproject.toml -m pytest -s unit_tests -c pytest.ini" +} +testTask.configure { + dependsOn installTestReqs +} + +tasks.named('check').configure { + dependsOn testTask +} diff --git a/tools/code-generator/Dockerfile b/airbyte-cdk/python/code-generator/Dockerfile similarity index 100% rename from tools/code-generator/Dockerfile rename to airbyte-cdk/python/code-generator/Dockerfile diff --git a/airbyte-cdk/python/gradle.properties b/airbyte-cdk/python/gradle.properties new file mode 100644 index 000000000000..a458cfe27eb9 --- /dev/null +++ b/airbyte-cdk/python/gradle.properties @@ -0,0 +1,11 @@ +# NOTE: some of these values are overwritten in CI! +# NOTE: if you want to override this for your local machine, set overrides in ~/.gradle/gradle.properties + +org.gradle.parallel=true +org.gradle.caching=true + +# Note, this might have issues on the normal Github runner. +org.gradle.vfs.watch=true + +# Tune # of cores Gradle uses. +# org.gradle.workers.max=3 diff --git a/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.jar b/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000000..7f93135c49b7 Binary files /dev/null and b/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.jar differ diff --git a/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.properties b/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000000..a80b22ce5cff --- /dev/null +++ b/airbyte-cdk/python/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/airbyte-cdk/python/gradlew b/airbyte-cdk/python/gradlew new file mode 100755 index 000000000000..1aa94a426907 --- /dev/null +++ b/airbyte-cdk/python/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/airbyte-cdk/python/gradlew.bat b/airbyte-cdk/python/gradlew.bat new file mode 100644 index 000000000000..6689b85beecd --- /dev/null +++ b/airbyte-cdk/python/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/airbyte-cdk/python/pyproject.toml b/airbyte-cdk/python/pyproject.toml index b5e8c83ca37b..f03d6cbcbe01 100644 --- a/airbyte-cdk/python/pyproject.toml +++ b/airbyte-cdk/python/pyproject.toml @@ -6,3 +6,39 @@ requires = [ ] build-backend = "setuptools.build_meta" + +[tool.coverage.report] +fail_under = 0 +skip_empty = true +sort = "-cover" +omit = [ + ".venv/*", + "main.py", + "setup.py", + "unit_tests/*", + "integration_tests/*", + "**/generated/*", +] + +[tool.flake8] +extend-exclude = [ + "*/lib/*/site-packages", + ".venv", + "build", + "models", + ".eggs", + "airbyte-cdk/python/airbyte_cdk/models/__init__.py", + "airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py", + ".tox", + "airbyte_api_client", + "**/generated/*", +] +max-complexity = 20 +max-line-length = 140 + +extend-ignore = [ + "E203", # whitespace before ':' (conflicts with Black) + "E231", # Bad trailing comma (conflicts with Black) + "E501", # line too long (conflicts with Black) + "W503", # line break before binary operator (conflicts with Black) +] \ No newline at end of file diff --git a/airbyte-cdk/python/settings.gradle b/airbyte-cdk/python/settings.gradle new file mode 100644 index 000000000000..02e3dd9a6724 --- /dev/null +++ b/airbyte-cdk/python/settings.gradle @@ -0,0 +1,29 @@ +import com.gradle.scan.plugin.PublishedBuildScan + +pluginManagement { + repositories { + // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. + gradlePluginPortal() + } +} + +// Configure the gradle enterprise plugin to enable build scans. Enabling the plugin at the top of the settings file allows the build scan to record +// as much information as possible. +plugins { + id "com.gradle.enterprise" version "3.15.1" +} + +ext.isCiServer = System.getenv().containsKey("CI") + +gradleEnterprise { + buildScan { + termsOfServiceUrl = "https://gradle.com/terms-of-service" + termsOfServiceAgree = "yes" + uploadInBackground = !isCiServer // Disable in CI or scan URLs may not work. + buildScanPublished { PublishedBuildScan scan -> + file("scan-journal.log") << "${new Date()} - ${scan.buildScanId} - ${scan.buildScanUri}\n" + } + } +} + +rootProject.name = 'airbyte-cdk-python' diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 0bb2a57ebe3a..b441974dc1bb 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.57.2", + version="0.67.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -71,7 +71,7 @@ "isodate~=0.6.1", "jsonschema~=3.2.0", "jsonref~=0.2", - "pendulum", + "pendulum<3.0.0", "genson==1.2.2", "pydantic>=1.10.8,<2.0.0", "pyrate-limiter~=3.1.0", diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index a314a928b904..190f8d4bcb56 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -769,8 +769,8 @@ def test_read_source_single_page_single_slice(mock_http_stream): "deployment_mode, url_base, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/api/v1/characters", None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "https://10.0.27.27", "ValueError", id="test_cloud_read_with_private_endpoint"), - pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "ValueError", id="test_cloud_read_with_localhost"), + pytest.param("CLOUD", "https://10.0.27.27", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "AirbyteTracedException", id="test_cloud_read_with_localhost"), pytest.param("CLOUD", "http://unsecured.protocol/api/v1", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"), pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"), pytest.param("OSS", "https://airbyte.com/api/v1/", None, id="test_oss_read_with_public_endpoint"), @@ -820,7 +820,7 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error "deployment_mode, token_url, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/tokens/bearer", None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "ValueError", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"), pytest.param("CLOUD", "http://unsecured.protocol/tokens/bearer", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"), pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"), pytest.param("OSS", "https://airbyte.com/tokens/bearer", None, id="test_oss_read_with_public_endpoint"), diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index ae98f6ad70ab..437a775dd8de 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -218,14 +218,14 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: @pytest.mark.parametrize( - "request_record_limit, max_record_limit", + "request_record_limit, max_record_limit, should_fail", [ - pytest.param(1, 3, id="test_create_request_with_record_limit"), - pytest.param(3, 1, id="test_create_request_record_limit_exceeds_max"), + pytest.param(1, 3, False, id="test_create_request_with_record_limit"), + pytest.param(3, 1, True, id="test_create_request_record_limit_exceeds_max"), ], ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") -def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int) -> None: +def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int, should_fail: bool) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { "headers": {"Content-Type": "application/json"}, @@ -249,16 +249,23 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r record_limit = min(request_record_limit, max_record_limit) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit) - actual_response: StreamRead = api.get_message_groups( - mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), record_limit=request_record_limit - ) - single_slice = actual_response.slices[0] - total_records = 0 - for i, actual_page in enumerate(single_slice.pages): - total_records += len(actual_page.records) - assert total_records == min([record_limit, n_records]) - - assert (total_records >= max_record_limit) == actual_response.test_read_limit_reached + # this is the call we expect to raise an exception + if should_fail: + with pytest.raises(ValueError): + api.get_message_groups( + mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), record_limit=request_record_limit + ) + else: + actual_response: StreamRead = api.get_message_groups( + mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), record_limit=request_record_limit + ) + single_slice = actual_response.slices[0] + total_records = 0 + for i, actual_page in enumerate(single_slice.pages): + total_records += len(actual_page.records) + assert total_records == min([record_limit, n_records]) + + assert (total_records >= max_record_limit) == actual_response.test_read_limit_reached @pytest.mark.parametrize( diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py index 0e8760b73cc7..41da64916368 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py @@ -285,7 +285,7 @@ def test_process_multiple_chunks_with_relevant_fields(): "text: Special tokens like", "<|endoftext|> are treated like regular", "text", - ] + ], ), ( "Custom separator", diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py index 088b4f85fb8e..a5f22b752ed2 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py @@ -13,12 +13,12 @@ OpenAICompatibleEmbeddingConfigModel, OpenAIEmbeddingConfigModel, ) -from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk from airbyte_cdk.destinations.vector_db_based.embedder import ( COHERE_VECTOR_SIZE, OPEN_AI_VECTOR_SIZE, AzureOpenAIEmbedder, CohereEmbedder, + Document, FakeEmbedder, FromFieldEmbedder, OpenAICompatibleEmbedder, @@ -82,10 +82,10 @@ def test_embedder(embedder_class, args, dimensions): mock_embedding_instance.embed_documents.return_value = [[0] * dimensions] * 2 chunks = [ - Chunk(page_content="a", metadata={}, record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), - Chunk(page_content="b", metadata={}, record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), + Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), + Document(page_content="b", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), ] - assert embedder.embed_chunks(chunks) == mock_embedding_instance.embed_documents.return_value + assert embedder.embed_documents(chunks) == mock_embedding_instance.embed_documents.return_value mock_embedding_instance.embed_documents.assert_called_with(["a", "b"]) @@ -102,12 +102,12 @@ def test_embedder(embedder_class, args, dimensions): ) def test_from_field_embedder(field_name, dimensions, metadata, expected_embedding, expected_error): embedder = FromFieldEmbedder(FromFieldEmbeddingConfigModel(mode="from_field", dimensions=dimensions, field_name=field_name)) - chunks = [Chunk(page_content="a", metadata=metadata, record=AirbyteRecordMessage(stream="mystream", data=metadata, emitted_at=0))] + chunks = [Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data=metadata, emitted_at=0))] if expected_error: with pytest.raises(AirbyteTracedException): - embedder.embed_chunks(chunks) + embedder.embed_documents(chunks) else: - assert embedder.embed_chunks(chunks) == [expected_embedding] + assert embedder.embed_documents(chunks) == [expected_embedding] def test_openai_chunking(): @@ -118,8 +118,6 @@ def test_openai_chunking(): mock_embedding_instance.embed_documents.side_effect = lambda texts: [[0] * OPEN_AI_VECTOR_SIZE] * len(texts) - chunks = [ - Chunk(page_content="a", metadata={}, record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)) for _ in range(1005) - ] - assert embedder.embed_chunks(chunks) == [[0] * OPEN_AI_VECTOR_SIZE] * 1005 + chunks = [Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)) for _ in range(1005)] + assert embedder.embed_documents(chunks) == [[0] * OPEN_AI_VECTOR_SIZE] * 1005 mock_embedding_instance.embed_documents.assert_has_calls([call(["a"] * 1000), call(["a"] * 5)]) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py index dff570d6e698..c906d0f3e9b5 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py @@ -48,8 +48,8 @@ def generate_stream(name: str = "example_stream", namespace: Optional[str] = Non def generate_mock_embedder(): mock_embedder = MagicMock() - mock_embedder.embed_chunks.return_value = [[0] * 1536] * (BATCH_SIZE + 5 + 5) - mock_embedder.embed_chunks.side_effect = lambda chunks: [[0] * 1536] * len(chunks) + mock_embedder.embed_documents.return_value = [[0] * 1536] * (BATCH_SIZE + 5 + 5) + mock_embedder.embed_documents.side_effect = lambda chunks: [[0] * 1536] * len(chunks) return mock_embedder @@ -88,7 +88,7 @@ def test_write(omit_raw_text: bool): # 1 batches due to max batch size reached and 1 batch due to state message assert mock_indexer.index.call_count == 2 assert mock_indexer.delete.call_count == 2 - assert mock_embedder.embed_chunks.call_count == 2 + assert mock_embedder.embed_documents.call_count == 2 if omit_raw_text: for call_args in mock_indexer.index.call_args_list: @@ -110,7 +110,7 @@ def test_write(omit_raw_text: bool): # 1 batch due to end of message stream assert mock_indexer.index.call_count == 3 assert mock_indexer.delete.call_count == 3 - assert mock_embedder.embed_chunks.call_count == 3 + assert mock_embedder.embed_documents.call_count == 3 mock_indexer.post_sync.assert_called() @@ -169,4 +169,4 @@ def test_write_stream_namespace_split(): call(ANY, None, "example_stream2"), ] ) - assert mock_embedder.embed_chunks.call_count == 4 + assert mock_embedder.embed_documents.call_count == 4 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py index 0992d0e331bc..78dd0b591ec3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py @@ -10,6 +10,7 @@ import pytest import requests from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets from requests import Response LOGGER = logging.getLogger(__name__) @@ -81,7 +82,6 @@ def test_refresh_with_encode_config_params(self): "client_id": base64.b64encode(config["client_id"].encode("utf-8")).decode(), "client_secret": base64.b64encode(config["client_secret"].encode("utf-8")).decode(), "refresh_token": None, - "scopes": None, } assert body == expected @@ -104,7 +104,6 @@ def test_refresh_with_decode_config_params(self): "client_id": "some_client_id", "client_secret": "some_client_secret", "refresh_token": None, - "scopes": None, } assert body == expected @@ -126,7 +125,6 @@ def test_refresh_without_refresh_token(self): "client_id": "some_client_id", "client_secret": "some_client_secret", "refresh_token": None, - "scopes": None, } assert body == expected @@ -168,6 +166,32 @@ def test_refresh_access_token(self, mocker): assert ("access_token", 1000) == token + filtered = filter_secrets("access_token") + assert filtered == "****" + + def test_refresh_access_token_missing_access_token(self, mocker): + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] }}", + client_secret="{{ config['client_secret'] }}", + refresh_token="{{ config['refresh_token'] }}", + config=config, + scopes=["scope1", "scope2"], + token_expiry_date="{{ config['token_expiry_date'] }}", + refresh_request_body={ + "custom_field": "{{ config['custom_field'] }}", + "another_field": "{{ config['another_field'] }}", + "scopes": ["no_override"], + }, + parameters={}, + ) + + resp.status_code = 200 + mocker.patch.object(resp, "json", return_value={"expires_in": 1000}) + mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) + with pytest.raises(Exception): + oauth.refresh_access_token() + @pytest.mark.parametrize( "timestamp, expected_date", [ @@ -278,6 +302,28 @@ def test_set_token_expiry_date_no_format(self, mocker, expires_in_response, next assert "access_token" == token assert oauth.get_token_expiry_date() == pendulum.parse(next_day) + def test_error_handling(self, mocker): + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] }}", + client_secret="{{ config['client_secret'] }}", + refresh_token="{{ config['refresh_token'] }}", + config=config, + scopes=["scope1", "scope2"], + refresh_request_body={ + "custom_field": "{{ config['custom_field'] }}", + "another_field": "{{ config['another_field'] }}", + "scopes": ["no_override"], + }, + parameters={}, + ) + resp.status_code = 400 + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": 123}) + mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) + with pytest.raises(requests.exceptions.HTTPError) as e: + oauth.refresh_access_token() + assert e.value.errno == 400 + def mock_request(method, url, data): if url == "refresh_end": diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py new file mode 100644 index 000000000000..346b284c3786 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator + + +def test_authenticator_selected(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + auth = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) + + assert auth is authenticators["one"] + + +def test_selection_path_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The path from `authenticator_selection_path` is not found in the config"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth_type"], + ) + + +def test_selected_auth_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The authenticator `unknown` is not found"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "unknown"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py index 4db4a1ea0b0a..599667c42f9b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -83,7 +83,7 @@ def test_api_key_authenticator(test_name, header, token, expected_header, expect """ token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) token_auth = ApiKeyAuthenticator( - request_option=RequestOption(inject_into=RequestOptionType.header, field_name=header, parameters={}), + request_option=RequestOption(inject_into=RequestOptionType.header, field_name=header, parameters=parameters), token_provider=token_provider, config=config, parameters=parameters, @@ -192,7 +192,7 @@ def test_api_key_authenticator_inject(test_name, field_name, token, expected_fie """ token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) token_auth = ApiKeyAuthenticator( - request_option=RequestOption(inject_into=inject_type, field_name=field_name, parameters={}), + request_option=RequestOption(inject_into=inject_type, field_name=field_name, parameters=parameters), token_provider=token_provider, config=config, parameters=parameters, diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index fbcad74101e4..5fa6af43d831 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -13,6 +13,7 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @pytest.mark.parametrize( @@ -68,6 +69,7 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da stream_state = {"created_at": "06-06-21"} stream_slice = {"last_seen": "06-10-21"} next_page_token = {"last_seen_id": 14} + schema = create_schema() first_transformation = Mock(spec=RecordTransformation) second_transformation = Mock(spec=RecordTransformation) transformations = [first_transformation, second_transformation] @@ -80,13 +82,19 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da else: record_filter = RecordFilter(config=config, condition=filter_template, parameters=parameters) record_selector = RecordSelector( - extractor=extractor, record_filter=record_filter, transformations=transformations, config=config, parameters=parameters + extractor=extractor, + record_filter=record_filter, + transformations=transformations, + config=config, + parameters=parameters, + schema_normalization=TypeTransformer(TransformConfig.NoTransform), ) actual_records = record_selector.select_records( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + response=response, records_schema=schema, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) assert actual_records == [Record(data, stream_slice) for data in expected_data] + calls = [] for record in expected_data: calls.append(call(record, config=config, stream_state=stream_state, stream_slice=stream_slice)) @@ -95,7 +103,77 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da transformation.transform.assert_has_calls(calls) +@pytest.mark.parametrize( + "test_name, schema, schema_transformation, body, expected_data", + [ + ( + "test_with_empty_schema", + {}, + TransformConfig.NoTransform, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}], + ), + ( + "test_with_schema_none_normalizer", + {}, + TransformConfig.NoTransform, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}], + ), + ( + "test_with_schema_and_default_normalizer", + {}, + TransformConfig.DefaultSchemaNormalization, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": "1", "created_at": "06-06-21", "field_int": 100, "field_float": 123.3}], + ), + ], +) +def test_schema_normalization(test_name, schema, schema_transformation, body, expected_data): + config = {"response_override": "stop_if_you_see_me"} + parameters = {"parameters_field": "data", "created_at": "06-07-21"} + stream_state = {"created_at": "06-06-21"} + stream_slice = {"last_seen": "06-10-21"} + next_page_token = {"last_seen_id": 14} + + response = create_response(body) + schema = create_schema() + decoder = JsonDecoder(parameters={}) + extractor = DpathExtractor(field_path=["data"], decoder=decoder, config=config, parameters=parameters) + record_selector = RecordSelector( + extractor=extractor, + record_filter=None, + transformations=[], + config=config, + parameters=parameters, + schema_normalization=TypeTransformer(schema_transformation), + ) + + actual_records = record_selector.select_records( + response=response, + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + records_schema=schema, + ) + + assert actual_records == [Record(data, stream_slice) for data in expected_data] + + def create_response(body): response = requests.Response() response._content = json.dumps(body).encode("utf-8") return response + + +def create_schema(): + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": {"type": "string"}, + "created_at": {"type": "string"}, + "field_int": {"type": "integer"}, + "field_float": {"type": "number"}, + }, + } diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index 18cf013d6127..63efac6688d5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -347,3 +347,60 @@ def test_only_propagate_parameters_to_components(): actual_component = transformer.propagate_types_and_parameters("", component, {}) assert actual_component == expected_component + + +def test_do_not_propagate_parameters_on_json_schema_object(): + component = { + "type": "DeclarativeStream", + "streams": [ + { + "type": "DeclarativeStream", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/schema#", + "properties": {"id": {"type": "string"}}, + }, + }, + "$parameters": { + "name": "roasters", + "primary_key": "id", + }, + } + ], + } + + expected_component = { + "type": "DeclarativeStream", + "streams": [ + { + "type": "DeclarativeStream", + "name": "roasters", + "primary_key": "id", + "schema_loader": { + "type": "InlineSchemaLoader", + "name": "roasters", + "primary_key": "id", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/schema#", + "properties": {"id": {"type": "string"}}, + }, + "$parameters": { + "name": "roasters", + "primary_key": "id", + }, + }, + "$parameters": { + "name": "roasters", + "primary_key": "id", + }, + } + ], + } + + transformer = ManifestComponentTransformer() + actual_component = transformer.propagate_types_and_parameters("", component, {}) + + assert actual_component == expected_component diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index efac06b3d08b..c96c19850578 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -39,6 +39,7 @@ from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory @@ -231,7 +232,7 @@ def test_full_config_stream(): assert isinstance(stream.retriever.paginator, DefaultPaginator) assert isinstance(stream.retriever.paginator.decoder, JsonDecoder) - assert stream.retriever.paginator.page_size_option.field_name == "page_size" + assert stream.retriever.paginator.page_size_option.field_name.eval(input_config) == "page_size" assert stream.retriever.paginator.page_size_option.inject_into == RequestOptionType.request_parameter assert isinstance(stream.retriever.paginator.page_token_option, RequestPath) assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com/v3/" @@ -244,7 +245,7 @@ def test_full_config_stream(): assert stream.retriever.paginator.pagination_strategy.page_size == 10 assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester._http_method == HttpMethod.GET + assert stream.retriever.requester.http_method == HttpMethod.GET assert stream.retriever.requester.name == stream.name assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}" assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}" @@ -302,10 +303,10 @@ def test_interpolate_config(): ) assert isinstance(authenticator, DeclarativeOauth2Authenticator) - assert authenticator.client_id.eval(input_config) == "some_client_id" - assert authenticator.client_secret.string == "some_client_secret" - assert authenticator.token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" - assert authenticator.refresh_token.eval(input_config) == "verysecrettoken" + assert authenticator._client_id.eval(input_config) == "some_client_id" + assert authenticator._client_secret.string == "some_client_secret" + assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" + assert authenticator._refresh_token.eval(input_config) == "verysecrettoken" assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} @@ -331,9 +332,9 @@ def test_interpolate_config_with_token_expiry_date_format(): assert isinstance(authenticator, DeclarativeOauth2Authenticator) assert authenticator.token_expiry_date_format == "%Y-%m-%d %H:%M:%S.%f+00:00" assert authenticator.token_expiry_is_time_of_expiration - assert authenticator.client_id.eval(input_config) == "some_client_id" - assert authenticator.client_secret.string == "some_client_secret" - assert authenticator.token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" + assert authenticator._client_id.eval(input_config) == "some_client_id" + assert authenticator._client_secret.string == "some_client_secret" + assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" def test_single_use_oauth_branch(): @@ -421,7 +422,7 @@ def test_list_based_stream_slicer_with_values_defined_in_config(): assert isinstance(partition_router, ListPartitionRouter) assert partition_router.values == ["airbyte", "airbyte-cloud"] assert partition_router.request_option.inject_into == RequestOptionType.header - assert partition_router.request_option.field_name == "repository" + assert partition_router.request_option.field_name.eval(config=input_config) == "repository" def test_create_substream_partition_router(): @@ -483,7 +484,7 @@ def test_create_substream_partition_router(): assert partition_router.parent_stream_configs[0].parent_key.eval({}) == "id" assert partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id" assert partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter - assert partition_router.parent_stream_configs[0].request_option.field_name == "repository_id" + assert partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id" assert partition_router.parent_stream_configs[1].parent_key.eval({}) == "someid" assert partition_router.parent_stream_configs[1].partition_field.eval({}) == "word_id" @@ -508,17 +509,17 @@ def test_datetime_based_cursor(): start_time_option: type: RequestOption inject_into: request_parameter - field_name: created[gte] + field_name: "since_{{ config['cursor_field'] }}" end_time_option: type: RequestOption inject_into: body_json - field_name: end_time + field_name: "before_{{ parameters['cursor_field'] }}" partition_field_start: star partition_field_end: en """ parsed_manifest = YamlDeclarativeSource._parse(content) resolved_manifest = resolver.preprocess_manifest(parsed_manifest) - slicer_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["incremental"], {}) + slicer_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["incremental"], {"cursor_field": "created_at"}) stream_slicer = factory.create_component(model_type=DatetimeBasedCursorModel, component_definition=slicer_manifest, config=input_config) @@ -528,9 +529,9 @@ def test_datetime_based_cursor(): assert stream_slicer.cursor_granularity == "PT0.000001S" assert stream_slicer.lookback_window.string == "P5D" assert stream_slicer.start_time_option.inject_into == RequestOptionType.request_parameter - assert stream_slicer.start_time_option.field_name == "created[gte]" + assert stream_slicer.start_time_option.field_name.eval(config=input_config | {"cursor_field": "updated_at"}) == "since_updated_at" assert stream_slicer.end_time_option.inject_into == RequestOptionType.body_json - assert stream_slicer.end_time_option.field_name == "end_time" + assert stream_slicer.end_time_option.field_name.eval({}) == "before_created_at" assert stream_slicer.partition_field_start.eval({}) == "star" assert stream_slicer.partition_field_end.eval({}) == "en" @@ -828,7 +829,7 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty ) assert isinstance(selector, HttpRequester) - assert selector._http_method == HttpMethod.GET + assert selector.http_method == HttpMethod.GET assert selector.name == "name" assert selector._path.string == "/v3/marketing/lists" assert selector._url_base.string == "https://api.sendgrid.com" @@ -936,6 +937,52 @@ def test_create_request_with_session_authenticator(): } +@pytest.mark.parametrize( + "input_config, expected_authenticator_class", + [ + pytest.param( + {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}}, + ApiKeyAuthenticator, + id="test_create_requester_with_selective_authenticator_and_token_selected", + ), + pytest.param( + {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}}, + DeclarativeOauth2Authenticator, + id="test_create_requester_with_selective_authenticator_and_oauth_selected", + ), + ], +) +def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class): + content = """ +authenticator: + type: SelectiveAuthenticator + authenticator_selection_path: + - auth + - type + authenticators: + token: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "api_key={{ config['credentials']['api_key'] }}" + oauth: + type: OAuthAuthenticator + token_refresh_endpoint: https://api.url.com + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: some_secret + refresh_token: some_token + """ + name = "name" + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator = factory.create_component( + model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name + ) + + assert isinstance(authenticator, expected_authenticator_class) + + def test_create_composite_error_handler(): content = """ error_handler: @@ -1030,7 +1077,7 @@ def test_config_with_defaults(): assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml" assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester._http_method == HttpMethod.GET + assert stream.retriever.requester.http_method == HttpMethod.GET assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator) assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken" @@ -1076,7 +1123,7 @@ def test_create_default_paginator(): assert isinstance(paginator.page_size_option, RequestOption) assert paginator.page_size_option.inject_into == RequestOptionType.request_parameter - assert paginator.page_size_option.field_name == "page_size" + assert paginator.page_size_option.field_name.eval(config=input_config) == "page_size" assert isinstance(paginator.page_token_option, RequestPath) @@ -1249,7 +1296,7 @@ def test_custom_components_do_not_contain_extra_fields(): assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id" assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id" assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter - assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name == "repository_id" + assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id" assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement) assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100 @@ -1298,7 +1345,7 @@ def test_parse_custom_component_fields_if_subcomponent(): assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id" assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id" assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter - assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name == "repository_id" + assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id" assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement) assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py index ce1b93b9d75b..3a83af1eb714 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py @@ -12,39 +12,40 @@ @pytest.mark.parametrize( - "test_name, partition_values, cursor_field, expected_slices", + "partition_values, cursor_field, expected_slices", [ ( - "test_single_element", ["customer", "store", "subscription"], "owner_resource", [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], ), ( - "test_input_list_is_string", '["customer", "store", "subscription"]', "owner_resource", [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], ), ( - "test_using_cursor_from_parameters", '["customer", "store", "subscription"]', "{{ parameters['cursor_field'] }}", [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], ), ], + ids=[ + "test_single_element", + "test_input_list_is_string", + "test_using_cursor_from_parameters", + ], ) -def test_list_partition_router(test_name, partition_values, cursor_field, expected_slices): +def test_list_partition_router(partition_values, cursor_field, expected_slices): slicer = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, parameters=parameters) slices = [s for s in slicer.stream_slices()] assert slices == expected_slices @pytest.mark.parametrize( - "test_name, request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data", + "request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data", [ ( - "test_inject_into_req_param", RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner_resource"), {"owner_resource": "customer"}, {}, @@ -52,7 +53,6 @@ def test_list_partition_router(test_name, partition_values, cursor_field, expect {}, ), ( - "test_pass_by_header", RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="owner_resource"), {}, {"owner_resource": "customer"}, @@ -60,7 +60,6 @@ def test_list_partition_router(test_name, partition_values, cursor_field, expect {}, ), ( - "test_inject_into_body_json", RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="owner_resource"), {}, {}, @@ -68,7 +67,6 @@ def test_list_partition_router(test_name, partition_values, cursor_field, expect {}, ), ( - "test_inject_into_body_data", RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner_resource"), {}, {}, @@ -76,8 +74,14 @@ def test_list_partition_router(test_name, partition_values, cursor_field, expect {"owner_resource": "customer"}, ), ], + ids=[ + "test_inject_into_req_param", + "test_pass_by_header", + "test_inject_into_body_json", + "test_inject_into_body_data", + ], ) -def test_request_option(test_name, request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data): +def test_request_option(request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data): partition_router = ListPartitionRouter( values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={} ) @@ -89,6 +93,31 @@ def test_request_option(test_name, request_option, expected_req_params, expected assert expected_body_data == partition_router.get_request_body_data(stream_slice=stream_slice) +@pytest.mark.parametrize( + "field_name_interpolation, expected_request_params", + [ + ("{{parameters['partition_name']}}", {"parameters_partition": "customer"}), + ("{{config['partition_name']}}", {"config_partition": "customer"}), + ], + ids=[ + "parameters_interpolation", + "config_interpolation", + ], +) +def test_request_options_interpolation(field_name_interpolation: str, expected_request_params: dict): + config = {"partition_name": "config_partition"} + parameters = {"partition_name": "parameters_partition"} + request_option = RequestOption( + inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_interpolation + ) + partition_router = ListPartitionRouter( + values=partition_values, cursor_field=cursor_field, config=config, request_option=request_option, parameters=parameters + ) + stream_slice = {cursor_field: "customer"} + + assert expected_request_params == partition_router.get_request_params(stream_slice=stream_slice) + + def test_request_option_before_updating_cursor(): request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner_resource") partition_router = ListPartitionRouter( diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index e677666f46eb..618a0fdb23e9 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -57,11 +57,10 @@ def read_records( @pytest.mark.parametrize( - "test_name, parent_stream_configs, expected_slices", + "parent_stream_configs, expected_slices", [ - ("test_no_parents", [], None), + ([], None), ( - "test_single_parent_slices_no_records", [ ParentStreamConfig( stream=MockStream([{}], [], "first_stream"), @@ -74,7 +73,6 @@ def read_records( [], ), ( - "test_single_parent_slices_with_records", [ ParentStreamConfig( stream=MockStream([{}], parent_records, "first_stream"), @@ -87,7 +85,6 @@ def read_records( [{"first_stream_id": 1, "parent_slice": {}}, {"first_stream_id": 2, "parent_slice": {}}], ), ( - "test_with_parent_slices_and_records", [ ParentStreamConfig( stream=MockStream(parent_slices, all_parent_data, "first_stream"), @@ -104,7 +101,6 @@ def read_records( ], ), ( - "test_multiple_parent_streams", [ ParentStreamConfig( stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), @@ -130,7 +126,6 @@ def read_records( ], ), ( - "test_missed_parent_key", [ ParentStreamConfig( stream=MockStream([{}], [{"id": 0}, {"id": 1}, {"_id": 2}, {"id": 3}], "first_stream"), @@ -147,7 +142,6 @@ def read_records( ], ), ( - "test_dpath_extraction", [ ParentStreamConfig( stream=MockStream([{}], [{"a": {"b": 0}}, {"a": {"b": 1}}, {"a": {"c": 2}}, {"a": {"b": 3}}], "first_stream"), @@ -164,8 +158,17 @@ def read_records( ], ), ], + ids=[ + "test_no_parents", + "test_single_parent_slices_no_records", + "test_single_parent_slices_with_records", + "test_with_parent_slices_and_records", + "test_multiple_parent_streams", + "test_missed_parent_key", + "test_dpath_extraction", + ], ) -def test_substream_slicer(test_name, parent_stream_configs, expected_slices): +def test_substream_slicer(parent_stream_configs, expected_slices): if expected_slices is None: try: SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters={}, config={}) @@ -178,10 +181,9 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): @pytest.mark.parametrize( - "test_name, parent_stream_request_parameters, expected_req_params, expected_headers, expected_body_json, expected_body_data", + "parent_stream_request_parameters, expected_req_params, expected_headers, expected_body_json, expected_body_data", [ ( - "test_request_option_in_request_param", [ RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="first_stream"), RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="second_stream"), @@ -192,7 +194,6 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): {}, ), ( - "test_request_option_in_header", [ RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="first_stream"), RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="second_stream"), @@ -203,7 +204,6 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): {}, ), ( - "test_request_option_in_param_and_header", [ RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="first_stream"), RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="second_stream"), @@ -214,7 +214,6 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): {}, ), ( - "test_request_option_in_body_json", [ RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="first_stream"), RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="second_stream"), @@ -225,7 +224,6 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): {}, ), ( - "test_request_option_in_body_data", [ RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="first_stream"), RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="second_stream"), @@ -236,9 +234,15 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): {"first_stream": "1234", "second_stream": "4567"}, ), ], + ids=[ + "test_request_option_in_request_param", + "test_request_option_in_header", + "test_request_option_in_param_and_header", + "test_request_option_in_body_json", + "test_request_option_in_body_data", + ], ) def test_request_option( - test_name, parent_stream_request_parameters, expected_req_params, expected_headers, @@ -275,6 +279,61 @@ def test_request_option( assert expected_body_data == partition_router.get_request_body_data(stream_slice=stream_slice) +@pytest.mark.parametrize( + "field_name_first_stream, field_name_second_stream, expected_request_params", + [ + ( + "{{parameters['field_name_first_stream']}}", + "{{parameters['field_name_second_stream']}}", + {"parameter_first_stream_id": "1234", "parameter_second_stream_id": "4567"}, + ), + ( + "{{config['field_name_first_stream']}}", + "{{config['field_name_second_stream']}}", + {"config_first_stream_id": "1234", "config_second_stream_id": "4567"}, + ), + ], + ids=[ + "parameters_interpolation", + "config_interpolation", + ], +) +def test_request_params_interpolation_for_parent_stream( + field_name_first_stream: str, field_name_second_stream: str, expected_request_params: dict +): + config = {"field_name_first_stream": "config_first_stream_id", "field_name_second_stream": "config_second_stream_id"} + parameters = {"field_name_first_stream": "parameter_first_stream_id", "field_name_second_stream": "parameter_second_stream_id"} + partition_router = SubstreamPartitionRouter( + parent_stream_configs=[ + ParentStreamConfig( + stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), + parent_key="id", + partition_field="first_stream_id", + parameters=parameters, + config=config, + request_option=RequestOption( + inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_first_stream + ), + ), + ParentStreamConfig( + stream=MockStream(second_parent_stream_slice, more_records, "second_stream"), + parent_key="id", + partition_field="second_stream_id", + parameters=parameters, + config=config, + request_option=RequestOption( + inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_second_stream + ), + ), + ], + parameters=parameters, + config=config, + ) + stream_slice = {"first_stream_id": "1234", "second_stream_id": "4567"} + + assert expected_request_params == partition_router.get_request_params(stream_slice=stream_slice) + + def test_given_record_is_airbyte_message_when_stream_slices_then_use_record_data(): parent_slice = {} partition_router = SubstreamPartitionRouter( diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 17ef223171c3..d8326d5227ec 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -21,10 +21,9 @@ @pytest.mark.parametrize( - "test_name, page_token_request_option, stop_condition, expected_updated_path, expected_request_params, expected_headers, expected_body_data, expected_body_json, last_records, expected_next_page_token, limit", + "page_token_request_option, stop_condition, expected_updated_path, expected_request_params, expected_headers, expected_body_data, expected_body_json, last_records, expected_next_page_token, limit", [ ( - "test_default_paginator_path", RequestPath(parameters={}), None, "/next_url", @@ -37,7 +36,6 @@ 2, ), ( - "test_default_paginator_request_param", RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", parameters={}), None, None, @@ -50,7 +48,6 @@ 2, ), ( - "test_default_paginator_no_token", RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", parameters={}), InterpolatedBoolean(condition="{{True}}", parameters={}), None, @@ -63,7 +60,6 @@ 2, ), ( - "test_default_paginator_cursor_header", RequestOption(inject_into=RequestOptionType.header, field_name="from", parameters={}), None, None, @@ -76,7 +72,6 @@ 2, ), ( - "test_default_paginator_cursor_body_data", RequestOption(inject_into=RequestOptionType.body_data, field_name="from", parameters={}), None, None, @@ -89,7 +84,6 @@ 2, ), ( - "test_default_paginator_cursor_body_json", RequestOption(inject_into=RequestOptionType.body_json, field_name="from", parameters={}), None, None, @@ -102,9 +96,16 @@ 2, ), ], + ids=[ + "test_default_paginator_path", + "test_default_paginator_request_param", + "test_default_paginator_no_token", + "test_default_paginator_cursor_header", + "test_default_paginator_cursor_body_data", + "test_default_paginator_cursor_body_json", + ], ) def test_default_paginator_with_cursor( - test_name, page_token_request_option, stop_condition, expected_updated_path, @@ -116,7 +117,9 @@ def test_default_paginator_with_cursor( expected_next_page_token, limit, ): - page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={}) + page_size_request_option = RequestOption( + inject_into=RequestOptionType.request_parameter, field_name="{{parameters['page_limit']}}", parameters={"page_limit": "limit"} + ) cursor_value = "{{ response.next }}" url_base = "https://airbyte.io" config = {} @@ -157,6 +160,62 @@ def test_default_paginator_with_cursor( assert actual_body_json == expected_body_json +@pytest.mark.parametrize( + "field_name_page_size_interpolation, field_name_page_token_interpolation, expected_request_params", + [ + ( + "{{parameters['page_limit']}}", + "{{parameters['page_token']}}", + {"parameters_limit": 50, "parameters_token": "https://airbyte.io/next_url"}, + ), + ("{{config['page_limit']}}", "{{config['page_token']}}", {"config_limit": 50, "config_token": "https://airbyte.io/next_url"}), + ], + ids=[ + "parameters_interpolation", + "config_interpolation", + ], +) +def test_paginator_request_param_interpolation( + field_name_page_size_interpolation: str, field_name_page_token_interpolation: str, expected_request_params: dict +): + config = {"page_limit": "config_limit", "page_token": "config_token"} + parameters = {"page_limit": "parameters_limit", "page_token": "parameters_token"} + page_size_request_option = RequestOption( + inject_into=RequestOptionType.request_parameter, + field_name=field_name_page_size_interpolation, + parameters=parameters, + ) + cursor_value = "{{ response.next }}" + url_base = "https://airbyte.io" + limit = 50 + strategy = CursorPaginationStrategy( + page_size=limit, + cursor_value=cursor_value, + stop_condition=None, + decoder=JsonDecoder(parameters={}), + config=config, + parameters=parameters, + ) + paginator = DefaultPaginator( + page_size_option=page_size_request_option, + page_token_option=RequestOption( + inject_into=RequestOptionType.request_parameter, field_name=field_name_page_token_interpolation, parameters=parameters + ), + pagination_strategy=strategy, + config=config, + url_base=url_base, + parameters=parameters, + ) + response = requests.Response() + response.headers = {"A_HEADER": "HEADER_VALUE"} + response_body = {"next": "https://airbyte.io/next_url"} + response._content = json.dumps(response_body).encode("utf-8") + last_records = [{"id": 0}, {"id": 1}] + paginator.next_page_token(response, last_records) + actual_request_params = paginator.get_request_params() + assert actual_request_params == expected_request_params + + def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="page_size", parameters={}) page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py index 1dc2b070fa65..5caa11f57f16 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py @@ -7,15 +7,37 @@ @pytest.mark.parametrize( - "test_name, option_type, field_name", + "option_type, field_name, expected_field_name", [ - ("test_limit_param_with_field_name", RequestOptionType.request_parameter, "field"), - ("test_limit_header_with_field_name", RequestOptionType.header, "field"), - ("test_limit_data_with_field_name", RequestOptionType.body_data, "field"), - ("test_limit_json_with_field_name", RequestOptionType.body_json, "field"), + (RequestOptionType.request_parameter, "field", "field"), + (RequestOptionType.header, "field", "field"), + (RequestOptionType.body_data, "field", "field"), + (RequestOptionType.body_json, "field", "field"), + (RequestOptionType.request_parameter, "since_{{ parameters['cursor_field'] }}", "since_updated_at"), + (RequestOptionType.header, "since_{{ parameters['cursor_field'] }}", "since_updated_at"), + (RequestOptionType.body_data, "since_{{ parameters['cursor_field'] }}", "since_updated_at"), + (RequestOptionType.body_json, "since_{{ parameters['cursor_field'] }}", "since_updated_at"), + (RequestOptionType.request_parameter, "since_{{ config['cursor_field'] }}", "since_created_at"), + (RequestOptionType.header, "since_{{ config['cursor_field'] }}", "since_created_at"), + (RequestOptionType.body_data, "since_{{ config['cursor_field'] }}", "since_created_at"), + (RequestOptionType.body_json, "since_{{ config['cursor_field'] }}", "since_created_at"), + ], + ids=[ + "test_limit_param_with_field_name", + "test_limit_header_with_field_name", + "test_limit_data_with_field_name", + "test_limit_json_with_field_name", + "test_limit_param_with_parameters_interpolation", + "test_limit_header_with_parameters_interpolation", + "test_limit_data_with_parameters_interpolation", + "test_limit_json_with_parameters_interpolation", + "test_limit_param_with_config_interpolation", + "test_limit_header_with_config_interpolation", + "test_limit_data_with_config_interpolation", + "test_limit_json_with_config_interpolation", ], ) -def test_request_option(test_name, option_type, field_name): - request_option = RequestOption(inject_into=option_type, field_name=field_name, parameters={}) - assert request_option.field_name == field_name +def test_request_option(option_type: RequestOptionType, field_name: str, expected_field_name: str): + request_option = RequestOption(inject_into=option_type, field_name=field_name, parameters={"cursor_field": "updated_at"}) + assert request_option.field_name.eval({"cursor_field": "created_at"}) == expected_field_name assert request_option.inject_into == option_type diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index bcd395b51ec8..a861e76f2a0c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -61,7 +61,7 @@ def factory( def test_http_requester(): - http_method = "GET" + http_method = HttpMethod.GET request_options_provider = MagicMock() request_params = {"param": "value"} @@ -106,7 +106,7 @@ def test_http_requester(): assert requester.get_url_base() == "https://airbyte.io/" assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" assert requester.get_authenticator() == authenticator - assert requester.get_method() == HttpMethod.GET + assert requester.get_method() == http_method assert requester.get_request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params assert requester.get_request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json @@ -379,52 +379,84 @@ def test_send_request_params(provider_params, param_params, authenticator_params "k=a%2Cb", # k=a,b id="test-request-parameter-comma-separated-strings", ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, + # {'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'} + "k=%7B%27updatedDateFrom%27%3A+%272023-08-20T00%3A00%3A00Z%27%2C+%27updatedDateTo%27%3A+%272023-08-20T23%3A59%3A59Z%27%7D", + id="test-request-parameter-from-config-object", + ), + ], +) +def test_request_param_interpolation(request_parameters, config, expected_query_params): + options_provider = InterpolatedRequestOptionsProvider( + config=config, + request_parameters=request_parameters, + request_body_data={}, + request_headers={}, + parameters={}, + ) + requester = create_requester() + requester._request_options_provider = options_provider + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.url.split("?", 1)[-1] == expected_query_params + + +@pytest.mark.parametrize( + "request_parameters, config, invalid_value_for_key", + [ pytest.param( {"k": "[1,2]"}, {}, - "k=1&k=2", + "k", id="test-request-parameter-list-of-numbers", ), pytest.param( - {"k": '["a", "b"]'}, + {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, {}, - "k=a&k=b", - id="test-request-parameter-list-of-strings", + "k", + id="test-request-parameter-object-of-the-updated-info", ), pytest.param( - {"k": '{{ config["k"] }}'}, - {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, - # {'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'} - "k=%7B%27updatedDateFrom%27%3A+%272023-08-20T00%3A00%3A00Z%27%2C+%27updatedDateTo%27%3A+%272023-08-20T23%3A59%3A59Z%27%7D", - id="test-request-parameter-from-config-object", + {"k": '["a", "b"]'}, + {}, + "k", + id="test-request-parameter-list-of-strings", ), pytest.param( {"k": '{{ config["k"] }}'}, {"k": [1, 2]}, - "k=1&k=2", + "k", id="test-request-parameter-from-config-list-of-numbers", ), pytest.param( {"k": '{{ config["k"] }}'}, {"k": ["a", "b"]}, - "k=a&k=b", + "k", id="test-request-parameter-from-config-list-of-strings", ), pytest.param( {"k": '{{ config["k"] }}'}, {"k": ["a,b"]}, - "k=a%2Cb", # k=a,b + "k", id="test-request-parameter-from-config-comma-separated-strings", ), pytest.param( {'["a", "b"]': '{{ config["k"] }}'}, {"k": [1, 2]}, - "%5B%22a%22%2C+%22b%22%5D=1&%5B%22a%22%2C+%22b%22%5D=2", + '["a", "b"]', id="test-key-with-list-is-not-interpolated", ), + pytest.param( + {"a": '{{ config["k"] }}', "b": {"end_timestamp": 1699109113}}, + {"k": 1699108113}, + "b", + id="test-key-with-multiple-keys", + ), ], ) -def test_request_param_interpolation(request_parameters, config, expected_query_params): +def test_request_param_interpolation_with_incorrect_values(request_parameters, config, invalid_value_for_key): options_provider = InterpolatedRequestOptionsProvider( config=config, request_parameters=request_parameters, @@ -434,9 +466,13 @@ def test_request_param_interpolation(request_parameters, config, expected_query_ ) requester = create_requester() requester._request_options_provider = options_provider - requester.send_request() - sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] - assert sent_request.url.split("?", 1)[-1] == expected_query_params + with pytest.raises(ValueError) as error: + requester.send_request() + + assert ( + error.value.args[0] + == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an array or object." + ) @pytest.mark.parametrize( diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 7139fff1ef41..438a1497df84 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -1,7 +1,6 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from unittest.mock import MagicMock, Mock, patch import pytest @@ -94,7 +93,7 @@ def test_simple_retriever_full(mock_http_stream): assert retriever._last_response is None assert retriever._records_from_last_response == [] - assert retriever._parse_response(response, stream_state={}) == records + assert retriever._parse_response(response, stream_state={}, records_schema={}) == records assert retriever._last_response == response assert retriever._records_from_last_response == records @@ -170,7 +169,7 @@ def test_simple_retriever_with_request_response_log_last_records(mock_http_strea assert retriever._last_response is None assert retriever._records_from_last_response == [] - assert retriever._parse_response(response, stream_state={}) == request_response_logs + assert retriever._parse_response(response, stream_state={}, records_schema={}) == request_response_logs assert retriever._last_response == response assert retriever._records_from_last_response == request_response_logs @@ -274,6 +273,46 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): pass +@pytest.mark.parametrize( + "test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping", + [ + ("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}), + ("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}), + ("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}), + ("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}), + ], +) +def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping): + # This test is separate from the other request options because request headers must be strings + paginator = MagicMock() + paginator.get_request_headers.return_value = paginator_mapping + requester = MagicMock(use_cache=False) + + stream_slicer = MagicMock() + stream_slicer.get_request_headers.return_value = {"key_from_slicer": "value"} + + record_selector = MagicMock() + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + stream_slicer=stream_slicer, + paginator=paginator, + ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests, + parameters={}, + config={}, + ) + + request_option_type_to_method = { + RequestOptionType.header: retriever._request_headers, + } + + for _, method in request_option_type_to_method.items(): + actual_mapping = method(None, None, next_page_token={"next_page_token": "1000"}) + assert expected_mapping == actual_mapping + + @pytest.mark.parametrize( "test_name, slicer_body_data, paginator_body_data, expected_body_data", [ @@ -396,13 +435,16 @@ def test_when_read_records_then_cursor_close_slice_with_greater_record(test_name ) stream_slice = {"repository": "airbyte"} + def retriever_read_pages(_, __, ___): + return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={}) + with patch.object( SimpleRetriever, "_read_pages", return_value=iter([first_record, second_record]), - side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice), + side_effect=retriever_read_pages, ): - list(retriever.read_records(stream_slice=stream_slice)) + list(retriever.read_records(stream_slice=stream_slice, records_schema={})) cursor.close_slice.assert_called_once_with(stream_slice, first_record if first_greater_than_second else second_record) @@ -425,13 +467,16 @@ def test_given_stream_data_is_not_record_when_read_records_then_update_slice_wit ) stream_slice = {"repository": "airbyte"} + def retriever_read_pages(_, __, ___): + return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={}) + with patch.object( SimpleRetriever, "_read_pages", return_value=iter(stream_data), - side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice), + side_effect=retriever_read_pages, ): - list(retriever.read_records(stream_slice=stream_slice)) + list(retriever.read_records(stream_slice=stream_slice, records_schema={})) cursor.close_slice.assert_called_once_with(stream_slice, None) @@ -440,7 +485,7 @@ def _generate_slices(number_of_slices): @patch.object(SimpleRetriever, "_read_pages", return_value=iter([])) -def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages): +def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages, mocker): requester = MagicMock() paginator = MagicMock() record_selector = MagicMock() @@ -459,9 +504,10 @@ def test_given_state_selector_when_read_records_use_stream_state(http_stream_rea parameters={}, config={}, ) - list(retriever.read_records(stream_slice=A_STREAM_SLICE)) - http_stream_read_pages.assert_called_once_with(retriever._parse_records, A_STREAM_STATE, A_STREAM_SLICE) + list(retriever.read_records(stream_slice=A_STREAM_SLICE, records_schema={})) + + http_stream_read_pages.assert_called_once_with(mocker.ANY, A_STREAM_STATE, A_STREAM_SLICE) def test_emit_log_request_response_messages(mocker): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py index 30cd88787935..abcecdd884f8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py @@ -10,39 +10,79 @@ @pytest.mark.parametrize( - ["input_record", "field_pointers", "expected"], + ["input_record", "field_pointers", "condition", "expected"], [ - pytest.param({"k1": "v", "k2": "v"}, [["k1"]], {"k2": "v"}, id="remove a field that exists (flat dict)"), - pytest.param({"k1": "v", "k2": "v"}, [["k3"]], {"k1": "v", "k2": "v"}, id="remove a field that doesn't exist (flat dict)"), - pytest.param({"k1": "v", "k2": "v"}, [["k1"], ["k2"]], {}, id="remove multiple fields that exist (flat dict)"), + pytest.param({"k1": "v", "k2": "v"}, [["k1"]], None, {"k2": "v"}, id="remove a field that exists (flat dict), condition = None"), + pytest.param({"k1": "v", "k2": "v"}, [["k1"]], "", {"k2": "v"}, id="remove a field that exists (flat dict)"), + pytest.param({"k1": "v", "k2": "v"}, [["k3"]], "", {"k1": "v", "k2": "v"}, id="remove a field that doesn't exist (flat dict)"), + pytest.param({"k1": "v", "k2": "v"}, [["k1"], ["k2"]], "", {}, id="remove multiple fields that exist (flat dict)"), # TODO: should we instead splice the element out of the array? I think that's the more intuitive solution # Otherwise one could just set the field's value to null. - pytest.param({"k1": [1, 2]}, [["k1", 0]], {"k1": [None, 2]}, id="remove field inside array (int index)"), - pytest.param({"k1": [1, 2]}, [["k1", "0"]], {"k1": [None, 2]}, id="remove field inside array (string index)"), + pytest.param({"k1": [1, 2]}, [["k1", 0]], "", {"k1": [None, 2]}, id="remove field inside array (int index)"), + pytest.param({"k1": [1, 2]}, [["k1", "0"]], "", {"k1": [None, 2]}, id="remove field inside array (string index)"), pytest.param( {"k1": "v", "k2": "v", "k3": [0, 1], "k4": "v"}, [["k1"], ["k2"], ["k3", 0]], + "", {"k3": [None, 1], "k4": "v"}, id="test all cases (flat)", ), - pytest.param({"k1": [0, 1]}, [[".", "k1", 10]], {"k1": [0, 1]}, id="remove array index that doesn't exist (flat)"), - pytest.param({".": {"k1": [0, 1]}}, [[".", "k1", 10]], {".": {"k1": [0, 1]}}, id="remove array index that doesn't exist (nested)"), - pytest.param({".": {"k2": "v", "k1": "v"}}, [[".", "k1"]], {".": {"k2": "v"}}, id="remove nested field that exists"), + pytest.param({"k1": [0, 1]}, [[".", "k1", 10]], "", {"k1": [0, 1]}, id="remove array index that doesn't exist (flat)"), pytest.param( - {".": {"k2": "v", "k1": "v"}}, [[".", "k3"]], {".": {"k2": "v", "k1": "v"}}, id="remove field that doesn't exist (nested)" + {".": {"k1": [0, 1]}}, [[".", "k1", 10]], "", {".": {"k1": [0, 1]}}, id="remove array index that doesn't exist (nested)" ), - pytest.param({".": {"k2": "v", "k1": "v"}}, [[".", "k1"], [".", "k2"]], {".": {}}, id="remove multiple fields that exist (nested)"), + pytest.param({".": {"k2": "v", "k1": "v"}}, [[".", "k1"]], "", {".": {"k2": "v"}}, id="remove nested field that exists"), pytest.param( - {".": {"k1": [0, 1]}}, [[".", "k1", 0]], {".": {"k1": [None, 1]}}, id="remove multiple fields that exist in arrays (nested)" + {".": {"k2": "v", "k1": "v"}}, [[".", "k3"]], "", {".": {"k2": "v", "k1": "v"}}, id="remove field that doesn't exist (nested)" + ), + pytest.param( + {".": {"k2": "v", "k1": "v"}}, [[".", "k1"], [".", "k2"]], "", {".": {}}, id="remove multiple fields that exist (nested)" + ), + pytest.param( + {".": {"k1": [0, 1]}}, + [[".", "k1", 0]], + "", + {".": {"k1": [None, 1]}}, + id="remove multiple fields that exist in arrays (nested)", ), pytest.param( {".": {"k1": [{"k2": "v", "k3": "v"}, {"k4": "v"}]}}, [[".", "k1", 0, "k2"], [".", "k1", 1, "k4"]], + "", {".": {"k1": [{"k3": "v"}, {}]}}, id="remove fields that exist in arrays (deeply nested)", ), + pytest.param( + {"k1": "v", "k2": "v"}, + [["**"]], + "{{ False }}", + {"k1": "v", "k2": "v"}, + id="do not remove any field if condition is boolean False", + ), + pytest.param({"k1": "v", "k2": "v"}, [["**"]], "{{ True }}", {}, id="remove all field if condition is boolean True"), + pytest.param( + {"k1": "v", "k2": "v1", "k3": "v1", "k4": {"k_nested": "v1", "k_nested2": "v2"}}, + [["**"]], + "{{ property == 'v1' }}", + {"k1": "v", "k4": {"k_nested2": "v2"}}, + id="recursively remove any field that matches property condition and leave that does not", + ), + pytest.param( + {"k1": "v", "k2": "some_long_string", "k3": "some_long_string", "k4": {"k_nested": "v1", "k_nested2": "v2"}}, + [["**"]], + "{{ property|length > 5 }}", + {"k1": "v", "k4": {"k_nested": "v1", "k_nested2": "v2"}}, + id="remove any field that have length > 5 and leave that does not", + ), + pytest.param( + {"k1": 255, "k2": "some_string", "k3": "some_long_string", "k4": {"k_nested": 123123, "k_nested2": "v2"}}, + [["**"]], + "{{ property is integer }}", + {"k2": "some_string", "k3": "some_long_string", "k4": {"k_nested2": "v2"}}, + id="recursively remove any field that of type integer and leave that does not", + ), ], ) -def test_remove_fields(input_record: Mapping[str, Any], field_pointers: List[FieldPointer], expected: Mapping[str, Any]): - transformation = RemoveFields(field_pointers=field_pointers, parameters={}) +def test_remove_fields(input_record: Mapping[str, Any], field_pointers: List[FieldPointer], condition: str, expected: Mapping[str, Any]): + transformation = RemoveFields(field_pointers=field_pointers, condition=condition, parameters={}) assert transformation.transform(input_record) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py index 4b1b2bb9fcad..1b2d23b810cc 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py @@ -5,7 +5,7 @@ import unittest import pytest -from airbyte_cdk.sources.file_based.config.csv_format import CsvHeaderAutogenerated, CsvHeaderFromCsv, CsvHeaderUserProvided +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderFromCsv, CsvHeaderUserProvided from pydantic import ValidationError @@ -26,3 +26,8 @@ def test_given_autogenerated_then_csv_does_not_have_header_row(self) -> None: def test_given_from_csv_then_csv_has_header_row(self) -> None: assert CsvHeaderFromCsv().has_header_row() + + +class CsvDelimiterTest(unittest.TestCase): + def test_tab_delimter(self): + assert CsvFormat(delimiter=r"\t").delimiter == '\\t' diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py index 9596cd84c598..3dfea9bb17df 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py @@ -447,11 +447,13 @@ def test_given_generator_closed_when_read_data_then_unregister_dialect(self) -> .build() ) + dialects_before = set(csv.list_dialects()) data_generator = self._read_data() next(data_generator) - assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + [new_dialect] = set(csv.list_dialects()) - dialects_before + assert self._CONFIG_NAME in new_dialect data_generator.close() - assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + assert new_dialect not in csv.list_dialects() def test_given_too_many_values_for_columns_when_read_data_then_raise_exception_and_unregister_dialect(self) -> None: self._stream_reader.open_file.return_value = ( @@ -466,13 +468,15 @@ def test_given_too_many_values_for_columns_when_read_data_then_raise_exception_a .build() ) + dialects_before = set(csv.list_dialects()) data_generator = self._read_data() next(data_generator) - assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + [new_dialect] = set(csv.list_dialects()) - dialects_before + assert self._CONFIG_NAME in new_dialect with pytest.raises(RecordParseError): next(data_generator) - assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + assert new_dialect not in csv.list_dialects() def test_given_too_few_values_for_columns_when_read_data_then_raise_exception_and_unregister_dialect(self) -> None: self._stream_reader.open_file.return_value = ( @@ -487,13 +491,15 @@ def test_given_too_few_values_for_columns_when_read_data_then_raise_exception_an .build() ) + dialects_before = set(csv.list_dialects()) data_generator = self._read_data() next(data_generator) - assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + [new_dialect] = set(csv.list_dialects()) - dialects_before + assert self._CONFIG_NAME in new_dialect with pytest.raises(RecordParseError): next(data_generator) - assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + assert new_dialect not in csv.list_dialects() def _read_data(self) -> Generator[Dict[str, str], None, None]: data_generator = self._csv_reader.read_data( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py index 7085ca1ca6e6..9bc096c5136e 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py @@ -86,7 +86,10 @@ def test_infer_schema(mock_detect_filetype, filetype, format_config, raises): assert schema == { "content": {"type": "string", "description": "Content of the file as markdown. Might be null if the file could not be parsed"}, "document_key": {"type": "string", "description": "Unique identifier of the document, e.g. the file path"}, - "_ab_source_file_parse_error": {"type": "string", "description": "Error message if the file could not be parsed even though the file is supported"}, + "_ab_source_file_parse_error": { + "type": "string", + "description": "Error message if the file could not be parsed even though the file is supported", + }, } loop.close() asyncio.set_event_loop(main_loop) @@ -128,7 +131,7 @@ def test_infer_schema(mock_detect_filetype, filetype, format_config, raises): { "content": None, "document_key": FILE_URI, - "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX", + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", } ], False, @@ -201,7 +204,7 @@ def test_infer_schema(mock_detect_filetype, filetype, format_config, raises): { "content": None, "document_key": FILE_URI, - "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=weird parsing error" + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=weird parsing error", } ], True, @@ -315,7 +318,7 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp @pytest.mark.parametrize( - "filetype, format_config, raises_for_status, file_content, json_response, expected_requests, raises, expected_records", + "filetype, format_config, raises_for_status, file_content, json_response, expected_requests, raises, expected_records, http_status_code", [ pytest.param( FileType.PDF, @@ -323,32 +326,49 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp None, "test", [{"type": "Text", "text": "test"}], - [call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")})], - False, [ - { - "content": "test", - "document_key": FILE_URI, - "_ab_source_file_parse_error": None - } + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ) ], + False, + [{"content": "test", "document_key": FILE_URI, "_ab_source_file_parse_error": None}], + 200, id="basic_request", ), pytest.param( FileType.PDF, - UnstructuredFormat(skip_unprocessable_file_types=False, strategy="hi_res", processing=APIProcessingConfigModel(mode="api", api_key="test", api_url="http://localhost:8000", parameters=[APIParameterConfigModel(name="include_page_breaks", value="true"), APIParameterConfigModel(name="ocr_languages", value="eng"), APIParameterConfigModel(name="ocr_languages", value="kor")])), + UnstructuredFormat( + skip_unprocessable_file_types=False, + strategy="hi_res", + processing=APIProcessingConfigModel( + mode="api", + api_key="test", + api_url="http://localhost:8000", + parameters=[ + APIParameterConfigModel(name="include_page_breaks", value="true"), + APIParameterConfigModel(name="ocr_languages", value="eng"), + APIParameterConfigModel(name="ocr_languages", value="kor"), + ], + ), + ), None, "test", [{"type": "Text", "text": "test"}], - [call("http://localhost:8000/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "hi_res", "include_page_breaks": "true", "ocr_languages": ["eng", "kor"]}, files={"files": ("filename", mock.ANY, "application/pdf")})], - False, [ - { - "content": "test", - "document_key": FILE_URI, - "_ab_source_file_parse_error": None - } + call( + "http://localhost:8000/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "hi_res", "include_page_breaks": "true", "ocr_languages": ["eng", "kor"]}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ) ], + False, + [{"content": "test", "document_key": FILE_URI, "_ab_source_file_parse_error": None}], + 200, id="request_with_params", ), pytest.param( @@ -359,13 +379,8 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp None, None, False, - [ - { - "content": "# Mymarkdown", - "document_key": FILE_URI, - "_ab_source_file_parse_error": None - } - ], + [{"content": "# Mymarkdown", "document_key": FILE_URI, "_ab_source_file_parse_error": None}], + 200, id="handle_markdown_locally", ), pytest.param( @@ -381,19 +396,45 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp "test", None, [ - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), ], True, None, + 200, id="retry_and_raise_on_api_error", ), pytest.param( @@ -407,21 +448,31 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp "test", [{"type": "Text", "text": "test"}], [ - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), ], False, - [ - { - "content": "test", - "document_key": FILE_URI, - "_ab_source_file_parse_error": None - } - ], + [{"content": "test", "document_key": FILE_URI, "_ab_source_file_parse_error": None}], + 200, id="retry_and_recover", ), pytest.param( @@ -433,11 +484,17 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp "test", [{"type": "Text", "text": "test"}], [ - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), ], True, None, + 200, id="no_retry_on_unexpected_error", ), pytest.param( @@ -449,18 +506,49 @@ def test_check_config(requests_mock, format_config, raises_for_status, json_resp "test", [{"type": "Text", "text": "test"}], [ - call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), call().raise_for_status(), ], True, None, + 400, id="no_retry_on_400_error", ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + None, + "test", + [{"detail": "Something went wrong"}], + [ + call( + "https://api.unstructured.io/general/v0/general", + headers={"accept": "application/json", "unstructured-api-key": "test"}, + data={"strategy": "auto"}, + files={"files": ("filename", mock.ANY, "application/pdf")}, + ), + ], + False, + [ + { + "content": None, + "document_key": FILE_URI, + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=[{'detail': 'Something went wrong'}]", + } + ], + 422, + id="error_record_on_422_error", + ), ], ) @patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.requests") @patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.detect_filetype") -@patch('time.sleep', side_effect=lambda _: None) +@patch("time.sleep", side_effect=lambda _: None) def test_parse_records_remotely( time_mock, mock_detect_filetype, @@ -473,6 +561,7 @@ def test_parse_records_remotely( expected_requests, raises, expected_records, + http_status_code, ): stream_reader = MagicMock() mock_open(stream_reader.open_file, read_data=bytes(str(file_content), "utf-8")) @@ -484,6 +573,7 @@ def test_parse_records_remotely( mock_detect_filetype.return_value = filetype mock_response = MagicMock() mock_response.json.return_value = json_response + mock_response.status_code = http_status_code if raises_for_status: mock_response.raise_for_status.side_effect = raises_for_status requests_mock.post.return_value = mock_response diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py index d3e528e32105..6d4966e2c2c9 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py @@ -15,6 +15,7 @@ from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesStreamReader @@ -61,5 +62,9 @@ class LowHistoryLimitCursor(DefaultFileBasedCursor): DEFAULT_MAX_HISTORY_SIZE = 3 +class LowHistoryLimitConcurrentCursor(FileBasedConcurrentCursor): + DEFAULT_MAX_HISTORY_SIZE = 3 + + def make_remote_files(files: List[str]) -> List[RemoteFile]: return [RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) for f in files] diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py index 643461471fd5..5db12fc5679c 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py @@ -26,11 +26,14 @@ from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor, DefaultFileBasedCursor +from airbyte_cdk.sources.source import TState from avro import datafile from pydantic import AnyUrl class InMemoryFilesSource(FileBasedSource): + _concurrency_level = 10 + def __init__( self, files: Mapping[str, Any], @@ -41,6 +44,8 @@ def __init__( parsers: Mapping[str, FileTypeParser], stream_reader: Optional[AbstractFileBasedStreamReader], catalog: Optional[Mapping[str, Any]], + config: Optional[Mapping[str, Any]], + state: Optional[TState], file_write_options: Mapping[str, Any], cursor_cls: Optional[AbstractFileBasedCursor], ): @@ -48,6 +53,9 @@ def __init__( self.files = files self.file_type = file_type self.catalog = catalog + self.configured_catalog = ConfiguredAirbyteCatalog(streams=self.catalog["streams"]) if self.catalog else None + self.config = config + self.state = state # Source setup stream_reader = stream_reader or InMemoryFilesStreamReader(files=files, file_type=file_type, file_write_options=file_write_options) @@ -55,7 +63,9 @@ def __init__( super().__init__( stream_reader, spec_class=InMemorySpec, - catalog_path="fake_path" if catalog else None, + catalog=self.configured_catalog, + config=self.config, + state=self.state, availability_strategy=availability_strategy, discovery_policy=discovery_policy or DefaultDiscoveryPolicy(), parsers=parsers, @@ -64,7 +74,7 @@ def __init__( ) def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog(streams=self.catalog["streams"]) if self.catalog else None + return self.configured_catalog class InMemoryFilesStreamReader(AbstractFileBasedStreamReader): diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py new file mode 100644 index 000000000000..ccbcc1c7116a --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py @@ -0,0 +1,2904 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor +from airbyte_cdk.test.state_builder import StateBuilder +from unit_tests.sources.file_based.helpers import LowHistoryLimitConcurrentCursor +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder +from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder + +single_csv_input_state_is_earlier_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_earlier_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-01T03:54:07.000000Z_some_old_file.csv", + }, + ) + .build(), + ) + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent = ( + TestScenarioBuilder() + .set_name("single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + }, + ) + .build(), + ) + ) + .set_expected_records( + [ + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + } + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurrent = ( + TestScenarioBuilder() + .set_name("single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-01T03:54:07.000000Z_a.csv", + }, + ) + .build(), + ) + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_no_input_state_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_earlier_again_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +multi_csv_same_timestamp_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("multi_csv_same_timestamp_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +single_csv_input_state_is_later_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_later_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "recent_file.csv": "2023-07-15T23:59:59.000000Z", + "a.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}, + "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_different_timestamps_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("multi_csv_stream_different_timestamps_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-04T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "a.csv": "2023-06-04T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv", + } + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +multi_csv_per_timestamp_scenario_concurrent = ( + TestScenarioBuilder() + .set_name("multi_csv_per_timestamp_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +multi_csv_skip_file_if_already_in_history_concurrent = ( + TestScenarioBuilder() + .set_name("skip_files_already_in_history_concurrent") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + {"history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv"}, + ) + .build(), + ) + ) +).build() + +multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name("multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name("multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(FileBasedConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, + "_ab_source_file_last_modified": "2023-06-03T03:54:07.000000Z_x.csv", + }, + ) + .build() + ) + ) +).build() + +multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name("multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newer") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-10T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv", + } + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv", + } + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + "c.csv": "2023-06-10T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name("multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_older") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-10T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv", + } + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv", + } + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + "c.csv": "2023-06-10T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-05-01T03:54:07.000000Z_very_very_very_old_file.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name("multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_newer") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11d", + "col2": "val12d", + "col3": "val13d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21d", + "col2": "val22d", + "col3": "val23d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name("multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_older") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11d", + "col2": "val12d", + "col3": "val13d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21d", + "col2": "val22d", + "col3": "val23d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() + +multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_older") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + .set_file_type("csv") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + } + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_newer") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + .set_file_type("csv") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + } + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + }, + ) + .build(), + ) + ) +).build() + + +multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_older") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # This file is skipped because it is older than the time_window + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_newer") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # This file is skipped because it is older than the time_window + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_newer = ( + TestScenarioBuilder() + .set_name( + "multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_newer" + ) + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "b.csv": "2023-06-06T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + }, + ) + .build(), + ) + ) +).build() + +multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_older = ( + TestScenarioBuilder() + .set_name( + "multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_older" + ) + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitConcurrentCursor) + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "stream1": { + "history": { + "b.csv": "2023-06-06T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + ] + ) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=StateBuilder() + .with_stream_state( + "stream1", + { + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-04T00:00:00.000000Z_very_old_file.csv", + }, + ) + .build(), + ) + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index e6c5824b4e19..bba3977db2fd 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -324,9 +324,7 @@ "processing": { "title": "Processing", "description": "Processing configuration", - "default": { - "mode": "local" - }, + "default": {"mode": "local"}, "type": "object", "oneOf": [ { @@ -337,16 +335,12 @@ "title": "Mode", "default": "local", "const": "local", - "enum": [ - "local" - ], - "type": "string" + "enum": ["local"], + "type": "string", } }, "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", - "required": [ - "mode" - ] + "required": ["mode"], }, { "title": "via API", @@ -356,10 +350,8 @@ "title": "Mode", "default": "api", "const": "api", - "enum": [ - "api" - ], - "type": "string" + "enum": ["api"], + "type": "string", }, "api_key": { "title": "API Key", @@ -367,17 +359,15 @@ "default": "", "always_show": True, "airbyte_secret": True, - "type": "string" + "type": "string", }, "api_url": { "title": "API URL", "description": "The URL of the unstructured API to use", "default": "https://api.unstructured.io", "always_show": True, - "examples": [ - "https://api.unstructured.com" - ], - "type": "string" + "examples": ["https://api.unstructured.com"], + "type": "string", }, "parameters": { "title": "Additional URL Parameters", @@ -392,35 +382,24 @@ "name": { "title": "Parameter name", "description": "The name of the unstructured API parameter to use", - "examples": [ - "combine_under_n_chars", - "languages" - ], - "type": "string" + "examples": ["combine_under_n_chars", "languages"], + "type": "string", }, "value": { "title": "Value", "description": "The value of the parameter", - "examples": [ - "true", - "hi_res" - ], - "type": "string" - } + "examples": ["true", "hi_res"], + "type": "string", + }, }, - "required": [ - "name", - "value" - ] - } - } + "required": ["name", "value"], + }, + }, }, "description": "Process files via an API, using the `hi_res` mode. This option is useful for increased performance and accuracy, but requires an API key and a hosted instance of unstructured.", - "required": [ - "mode" - ] - } - ] + "required": ["mode"], + }, + ], }, }, "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", @@ -842,6 +821,104 @@ ) .set_expected_records([]) .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_logs( + { + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.INVALID_SCHEMA_ERROR.value} stream=stream1 file=a.csv line_no=1 n_skipped=0", + }, + ] + } + ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) +).build() + +invalid_csv_multi_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() + .set_name("invalid_csv_multi_scenario") # too many values for the number of headers + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "format": {"filetype": "csv"}, + "globs": ["b.csv"], + "validation_policy": "Emit Record", + }, + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1",), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b", "val14b"), + ("val23b", "val24b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records([]) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) .set_expected_logs( { "read": [ @@ -849,9 +926,14 @@ "level": "ERROR", "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.csv line_no=1 n_skipped=0", }, + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream2 file=b.csv line_no=1 n_skipped=0", + }, ] } ) + .set_expected_read_error(AirbyteTracedException, "Please check the logged errors for more information.") ).build() csv_single_stream_scenario: TestScenario[InMemoryFilesSource] = ( @@ -1368,28 +1450,7 @@ } ) .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) - .set_expected_records( - [ - { - "data": { - "col1": "val11", - "col2": "val12", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", - }, - "stream": "stream1", - }, - { - "data": { - "col1": "val21", - "col2": "val22", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", - }, - "stream": "stream1", - }, - ] - ) + .set_expected_records([]) ).build() schemaless_csv_scenario: TestScenario[InMemoryFilesSource] = ( @@ -1663,7 +1724,7 @@ } ) .set_expected_check_status("FAILED") - .set_expected_check_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_check_error(AirbyteTracedException, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) ).build() @@ -1751,7 +1812,7 @@ } ) .set_expected_check_status("FAILED") - .set_expected_check_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_check_error(AirbyteTracedException, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) ).build() @@ -2172,17 +2233,15 @@ }, ] ) - .set_expected_logs( - { - "read": [ - { - "level": "ERROR", - "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a.csv line_no=2 n_skipped=0", - } - ] - } + .set_expected_read_error( + AirbyteTracedException, + f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.csv line_no=2 n_skipped=0", ) .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() csv_escape_char_is_set_scenario: TestScenario[InMemoryFilesSource] = ( @@ -2928,6 +2987,7 @@ .set_file_type("csv") ) .set_expected_check_status("FAILED") + .set_expected_check_error(AirbyteTracedException, FileBasedSourceError.EMPTY_STREAM.value) .set_expected_catalog( { "streams": [ diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py index 90deb31fe41b..f3d72ab67e7a 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.source import TState from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder @@ -29,8 +30,10 @@ def __init__(self) -> None: self._stream_reader: Optional[AbstractFileBasedStreamReader] = None self._file_write_options: Mapping[str, Any] = {} self._cursor_cls: Optional[Type[AbstractFileBasedCursor]] = None + self._config: Optional[Mapping[str, Any]] = None + self._state: Optional[TState] = None - def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> InMemoryFilesSource: + def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> InMemoryFilesSource: if self._file_type is None: raise ValueError("file_type is not set") return InMemoryFilesSource( @@ -42,6 +45,8 @@ def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> InMemoryFile self._parsers, self._stream_reader, configured_catalog, + config, + state, self._file_write_options, self._cursor_cls, ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py index 3c3195fbac61..3b9785e11bfe 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.sources.file_based.helpers import LowHistoryLimitCursor from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder @@ -36,20 +38,18 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, + }, + ) + .build(), ) ) .set_expected_records( @@ -137,20 +137,18 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + }, + ) + .build(), ) ) .set_expected_records( @@ -220,20 +218,18 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, + }, + ) + .build(), ) ) .set_expected_records( @@ -321,6 +317,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -377,7 +374,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -420,6 +417,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -499,7 +497,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -534,6 +532,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -593,15 +592,14 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": {"history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}}, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}, + }, + ) + .build(), ) ) ).build() @@ -644,6 +642,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -731,7 +730,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -782,6 +781,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -891,7 +891,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -942,6 +942,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -1035,15 +1036,14 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": {"history": {"a.csv": "2023-06-05T03:54:07.000000Z"}}, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + }, + ) + .build(), ) ) ).build() @@ -1094,6 +1094,7 @@ } ) .set_file_type("csv") + .set_cursor_cls(DefaultFileBasedCursor) ) .set_expected_catalog( { @@ -1163,17 +1164,14 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, + }, + ) + .build(), ) ) ).build() @@ -1348,21 +1346,18 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", - "very_old_file.csv": "2023-06-02T03:54:07.000000Z", - "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", }, - } - ], + }, + ) + .build(), ) ) ).build() @@ -1546,7 +1541,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -1652,21 +1647,18 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-05T03:54:07.000000Z", - "d.csv": "2023-06-05T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", }, - } - ], + }, + ) + .build(), ) ) ).build() @@ -1794,21 +1786,18 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", - "e.csv": "2023-06-08T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", }, - } - ], + }, + ) + .build(), ) ) ).build() @@ -1962,21 +1951,18 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ + input_state=StateBuilder() + .with_stream_state( + "stream1", { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "old_file.csv": "2023-06-05T00:00:00.000000Z", - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", }, - } - ], + }, + ) + .build(), ) ) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py index b4a447c4f0c0..cee4c5b9a1e2 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py @@ -484,15 +484,9 @@ ] } ) - .set_expected_records( - [ - { - "data": {"col1": "val1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.jsonl"}, - "stream": "stream1", - }, - ] - ) + .set_expected_records([]) .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_read_error(AirbyteTracedException, "Please check the logged errors for more information.") .set_expected_logs( { "read": [ diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py index 0852de4a361a..30ffa263f88e 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py @@ -729,6 +729,7 @@ .set_expected_records([]) .set_expected_logs({"read": [{"level": "ERROR", "message": "Error parsing record"}]}) .set_expected_discover_error(AirbyteTracedException, "Error inferring schema from files") + .set_expected_read_error(AirbyteTracedException, "Please check the logged errors for more information.") .set_expected_catalog( { "streams": [ diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py index 75feaf360595..25811a9e60ad 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -6,8 +6,9 @@ from dataclasses import dataclass, field from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteStateMessage, SyncMode from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.source import TState from airbyte_protocol.models import ConfiguredAirbyteCatalog @@ -26,7 +27,7 @@ class SourceBuilder(ABC, Generic[SourceType]): """ @abstractmethod - def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> SourceType: + def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> SourceType: raise NotImplementedError() @@ -80,11 +81,11 @@ def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]] return self.catalog.dict() # type: ignore # dict() is not typed catalog: Mapping[str, Any] = {"streams": []} - for stream in self.source.streams(self.config): + for stream in catalog["streams"]: catalog["streams"].append( { "stream": { - "name": stream.name, + "name": stream["name"], "json_schema": {}, "supported_sync_modes": [sync_mode.value], }, @@ -152,7 +153,7 @@ def set_expected_logs(self, expected_logs: Mapping[str, List[Mapping[str, Any]]] self._expected_logs = expected_logs return self - def set_expected_records(self, expected_records: List[Mapping[str, Any]]) -> "TestScenarioBuilder[SourceType]": + def set_expected_records(self, expected_records: Optional[List[Mapping[str, Any]]]) -> "TestScenarioBuilder[SourceType]": self._expected_records = expected_records return self @@ -190,8 +191,14 @@ def copy(self) -> "TestScenarioBuilder[SourceType]": def build(self) -> "TestScenario[SourceType]": if self.source_builder is None: raise ValueError("source_builder is not set") + if self._incremental_scenario_config and self._incremental_scenario_config.input_state: + state = [AirbyteStateMessage.parse_obj(s) for s in self._incremental_scenario_config.input_state] + else: + state = None source = self.source_builder.build( - self._configured_catalog(SyncMode.incremental if self._incremental_scenario_config else SyncMode.full_refresh) + self._configured_catalog(SyncMode.incremental if self._incremental_scenario_config else SyncMode.full_refresh), + self._config, + state, ) return TestScenario( self._name, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py index dcea12ef9aa3..da1e468c9df5 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py @@ -15,12 +15,18 @@ json_schema = { "type": "object", "properties": { - "content": {"type": ["null", "string"], "description": "Content of the file as markdown. Might be null if the file could not be parsed"}, + "content": { + "type": ["null", "string"], + "description": "Content of the file as markdown. Might be null if the file could not be parsed", + }, "document_key": {"type": ["null", "string"], "description": "Unique identifier of the document, e.g. the file path"}, - "_ab_source_file_parse_error": {"type": ["null", "string"], "description": "Error message if the file could not be parsed even though the file is supported"}, + "_ab_source_file_parse_error": { + "type": ["null", "string"], + "description": "Error message if the file could not be parsed even though the file is supported", + }, "_ab_source_file_last_modified": {"type": "string"}, "_ab_source_file_url": {"type": "string"}, - } + }, } simple_markdown_scenario = ( @@ -69,7 +75,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -104,7 +110,78 @@ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c", "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + ] + ) +).build() +simple_txt_scenario = ( + TestScenarioBuilder() + .set_name("simple_txt_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.txt": { + "contents": bytes("Just some raw text", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b": { + "contents": bytes("Detected via mime type", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + "mime_type": "text/plain", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + "source_defined_primary_key": [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "a.txt", + "content": "Just some raw text", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.txt", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "b", + "content": "Detected via mime type", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b", + "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -132,7 +209,7 @@ FileBasedSourceBuilder() .set_files( { - "a.txt": { + "a.csv": { "contents": bytes("Just a humble text file", "UTF-8"), "last_modified": "2023-06-05T03:54:07.000Z", }, @@ -148,7 +225,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -156,6 +233,10 @@ ) .set_expected_records([]) .set_expected_discover_error(AirbyteTracedException, "Error inferring schema from files") + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() # If skip unprocessable file types is set to true, then discover will succeed even if there are non-matching file types @@ -178,7 +259,7 @@ FileBasedSourceBuilder() .set_files( { - "a.txt": { + "a.csv": { "contents": bytes("Just a humble text file", "UTF-8"), "last_modified": "2023-06-05T03:54:07.000Z", }, @@ -194,7 +275,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -204,11 +285,11 @@ [ { "data": { - "document_key": "a.txt", + "document_key": "a.csv", "content": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.txt", - "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=a.txt message=File type FileType.TXT is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX", + "_ab_source_file_url": "a.csv", + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=a.csv message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", }, "stream": "stream1", } @@ -242,7 +323,7 @@ "contents": bytes("A harmless markdown file", "UTF-8"), "last_modified": "2023-06-05T03:54:07.000Z", }, - "b.txt": { + "b.csv": { "contents": bytes("An evil text file", "UTF-8"), "last_modified": "2023-06-05T03:54:07.000Z", }, @@ -258,7 +339,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -338,7 +419,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -416,7 +497,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] @@ -484,7 +565,7 @@ "json_schema": json_schema, "name": "stream1", "source_defined_cursor": True, - 'source_defined_primary_key': [["document_key"]], + "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], } ] diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py index 58d528cb7caf..974bbf558974 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -442,7 +442,13 @@ ] } ) - .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).with_stream("stream2", SyncMode.full_refresh).with_stream("stream3", SyncMode.full_refresh).build()) + .set_catalog( + CatalogBuilder() + .with_stream("stream1", SyncMode.full_refresh) + .with_stream("stream2", SyncMode.full_refresh) + .with_stream("stream3", SyncMode.full_refresh) + .build() + ) .set_expected_check_status("FAILED") .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py index af1318dba647..4ff096954523 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py @@ -2,7 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError + +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder @@ -272,6 +273,10 @@ ] } ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -416,6 +421,10 @@ ] } ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -492,19 +501,9 @@ }, ] ) - .set_expected_logs( - { - "read": [ - { - "level": "ERROR", - "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=c.csv line_no=2 n_skipped=0", - }, - { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", - }, - ] - } + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", ) ).build() @@ -640,23 +639,9 @@ }, ] ) - .set_expected_logs( - { - "read": [ - { - "level": "ERROR", - "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", - }, - { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", - }, - { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", - }, - ] - } + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", ) ).build() @@ -676,29 +661,7 @@ ] } ) - .set_expected_records( - [ - { - "data": { - "col1": "val_a_11", - "col2": 1, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", - }, - "stream": "stream1", - }, - { - "data": { - "col1": "val_a_12", - "col2": 2, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", - }, - "stream": "stream1", - }, - # No records past that because the first record for the second file did not conform to the schema - ] - ) + .set_expected_records(None) # When syncing streams concurrently we don't know how many records will be emitted before the sync stops .set_expected_logs( { "read": [ @@ -737,56 +700,7 @@ ] } ) - .set_expected_records( - [ - { - "data": { - "col1": "val_aa1_11", - "col2": 1, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a/a1.csv", - }, - "stream": "stream1", - }, - { - "data": { - "col1": "val_aa1_12", - "col2": 2, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a/a1.csv", - }, - "stream": "stream1", - }, - # {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, - { - "data": { - "col1": "val_bb1_11", - "col2": 1, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b/b1.csv", - }, - "stream": "stream2", - }, - { - "data": { - "col1": "val_bb1_12", - "col2": 2, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b/b1.csv", - }, - "stream": "stream2", - }, - # {"data": {"col1": "val_bb2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, - # {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, - # {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - # {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - ] - ) + .set_expected_records(None) # When syncing streams concurrently we don't know how many records will be emitted before the sync stops .set_expected_logs( { "read": [ diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/__init__.py similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json rename to airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_adapters.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_adapters.py new file mode 100644 index 000000000000..2d93e73ced56 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_adapters.py @@ -0,0 +1,373 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +import unittest +from datetime import datetime +from unittest.mock import MagicMock, Mock + +import pytest +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, Level, SyncMode +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.file_based.availability_strategy import DefaultFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.discovery_policy import DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector +from airbyte_cdk.sources.file_based.file_types import default_parsers +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_validation_policies import EmitRecordPolicy +from airbyte_cdk.sources.file_based.stream import DefaultFileBasedStream +from airbyte_cdk.sources.file_based.stream.concurrent.adapters import ( + FileBasedStreamFacade, + FileBasedStreamPartition, + FileBasedStreamPartitionGenerator, +) +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from freezegun import freeze_time + +_ANY_SYNC_MODE = SyncMode.full_refresh +_ANY_STATE = {"state_key": "state_value"} +_ANY_CURSOR_FIELD = ["a", "cursor", "key"] +_STREAM_NAME = "stream" +_ANY_CURSOR = Mock(spec=FileBasedNoopCursor) + + +@pytest.mark.parametrize( + "sync_mode", + [ + pytest.param(SyncMode.full_refresh, id="test_full_refresh"), + pytest.param(SyncMode.incremental, id="test_incremental"), + ], +) +def test_file_based_stream_partition_generator(sync_mode): + stream = Mock() + message_repository = Mock() + stream_slices = [ + {"files": [RemoteFile(uri="1", last_modified=datetime.now())]}, + {"files": [RemoteFile(uri="2", last_modified=datetime.now())]}, + ] + stream.stream_slices.return_value = stream_slices + + partition_generator = FileBasedStreamPartitionGenerator( + stream, message_repository, _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR + ) + + partitions = list(partition_generator.generate()) + slices = [partition.to_slice() for partition in partitions] + assert slices == stream_slices + stream.stream_slices.assert_called_once_with(sync_mode=_ANY_SYNC_MODE, cursor_field=_ANY_CURSOR_FIELD, stream_state=_ANY_STATE) + + +@pytest.mark.parametrize( + "transformer, expected_records", + [ + pytest.param( + TypeTransformer(TransformConfig.NoTransform), + [Record({"data": "1"}, _STREAM_NAME), Record({"data": "2"}, _STREAM_NAME)], + id="test_no_transform", + ), + pytest.param( + TypeTransformer(TransformConfig.DefaultSchemaNormalization), + [Record({"data": 1}, _STREAM_NAME), Record({"data": 2}, _STREAM_NAME)], + id="test_default_transform", + ), + ], +) +def test_file_based_stream_partition(transformer, expected_records): + stream = Mock() + stream.name = _STREAM_NAME + stream.get_json_schema.return_value = {"type": "object", "properties": {"data": {"type": ["integer"]}}} + stream.transformer = transformer + message_repository = InMemoryMessageRepository() + _slice = None + sync_mode = SyncMode.full_refresh + cursor_field = None + state = None + partition = FileBasedStreamPartition(stream, _slice, message_repository, sync_mode, cursor_field, state, _ANY_CURSOR) + + a_log_message = AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message='slice:{"partition": 1}', + ), + ) + + stream_data = [a_log_message, {"data": "1"}, {"data": "2"}] + stream.read_records.return_value = stream_data + + records = list(partition.read()) + messages = list(message_repository.consume_queue()) + + assert records == expected_records + assert messages == [a_log_message] + + +@pytest.mark.parametrize( + "exception_type, expected_display_message", + [ + pytest.param(Exception, None, id="test_exception_no_display_message"), + pytest.param(ExceptionWithDisplayMessage, "display_message", id="test_exception_no_display_message"), + ], +) +def test_file_based_stream_partition_raising_exception(exception_type, expected_display_message): + stream = Mock() + stream.get_error_display_message.return_value = expected_display_message + + message_repository = InMemoryMessageRepository() + _slice = None + + partition = FileBasedStreamPartition(stream, _slice, message_repository, _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR) + + stream.read_records.side_effect = Exception() + + with pytest.raises(exception_type) as e: + list(partition.read()) + if isinstance(e, ExceptionWithDisplayMessage): + assert e.display_message == "display message" + + +@freeze_time("2023-06-09T00:00:00Z") +@pytest.mark.parametrize( + "_slice, expected_hash", + [ + pytest.param( + {"files": [RemoteFile(uri="1", last_modified=datetime.strptime("2023-06-09T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"))]}, + hash(("stream", "2023-06-09T00:00:00.000000Z_1")), + id="test_hash_with_slice", + ), + pytest.param(None, hash("stream"), id="test_hash_no_slice"), + ], +) +def test_file_based_stream_partition_hash(_slice, expected_hash): + stream = Mock() + stream.name = "stream" + partition = FileBasedStreamPartition(stream, _slice, Mock(), _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR) + + _hash = partition.__hash__() + assert _hash == expected_hash + + +class StreamFacadeTest(unittest.TestCase): + def setUp(self): + self._abstract_stream = Mock() + self._abstract_stream.name = "stream" + self._abstract_stream.as_airbyte_stream.return_value = AirbyteStream( + name="stream", + json_schema={"type": "object"}, + supported_sync_modes=[SyncMode.full_refresh], + ) + self._legacy_stream = DefaultFileBasedStream( + cursor=FileBasedNoopCursor(MagicMock()), + config=FileBasedStreamConfig(name="stream", format=CsvFormat()), + catalog_schema={}, + stream_reader=MagicMock(), + availability_strategy=DefaultFileBasedAvailabilityStrategy(MagicMock()), + discovery_policy=DefaultDiscoveryPolicy(), + parsers=default_parsers, + validation_policy=EmitRecordPolicy(), + errors_collector=FileBasedErrorsCollector(), + ) + self._cursor = Mock(spec=Cursor) + self._logger = Mock() + self._slice_logger = Mock() + self._slice_logger.should_log_slice_message.return_value = False + self._facade = FileBasedStreamFacade(self._abstract_stream, self._legacy_stream, self._cursor, self._slice_logger, self._logger) + self._source = Mock() + + self._stream = Mock() + self._stream.primary_key = "id" + + def test_name_is_delegated_to_wrapped_stream(self): + assert self._facade.name == self._abstract_stream.name + + def test_cursor_field_is_a_string(self): + self._abstract_stream.cursor_field = "cursor_field" + assert self._facade.cursor_field == "cursor_field" + + def test_source_defined_cursor_is_true(self): + assert self._facade.source_defined_cursor + + def test_json_schema_is_delegated_to_wrapped_stream(self): + json_schema = {"type": "object"} + self._abstract_stream.get_json_schema.return_value = json_schema + assert self._facade.get_json_schema() == json_schema + self._abstract_stream.get_json_schema.assert_called_once_with() + + def test_given_cursor_is_noop_when_supports_incremental_then_return_legacy_stream_response(self): + assert ( + FileBasedStreamFacade( + self._abstract_stream, self._legacy_stream, _ANY_CURSOR, Mock(spec=SliceLogger), Mock(spec=logging.Logger) + ).supports_incremental + == self._legacy_stream.supports_incremental + ) + + def test_given_cursor_is_not_noop_when_supports_incremental_then_return_true(self): + assert FileBasedStreamFacade( + self._abstract_stream, self._legacy_stream, Mock(spec=Cursor), Mock(spec=SliceLogger), Mock(spec=logging.Logger) + ).supports_incremental + + def test_full_refresh(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_records(SyncMode.full_refresh, None, {}, None)) + + assert actual_stream_data == expected_stream_data + + def test_read_records_full_refresh(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_full_refresh(None, None, None)) + + assert actual_stream_data == expected_stream_data + + def test_read_records_incremental(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_incremental(None, None, None, None, None, None, None)) + + assert actual_stream_data == expected_stream_data + + def test_create_from_stream_stream(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = "cursor" + + facade = FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + assert facade.name == "stream" + assert facade.cursor_field == "cursor" + assert facade._abstract_stream._primary_key == ["id"] + + def test_create_from_stream_stream_with_none_primary_key(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = None + stream.cursor_field = [] + + facade = FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade._abstract_stream._primary_key == [] + + def test_create_from_stream_with_composite_primary_key(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = ["id", "name"] + stream.cursor_field = [] + + facade = FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade._abstract_stream._primary_key == ["id", "name"] + + def test_create_from_stream_with_empty_list_cursor(self): + stream = Mock() + stream.primary_key = "id" + stream.cursor_field = [] + + facade = FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + assert facade.cursor_field == [] + + def test_create_from_stream_raises_exception_if_primary_key_is_nested(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = [["field", "id"]] + + with self.assertRaises(ValueError): + FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_raises_exception_if_primary_key_has_invalid_type(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = 123 + + with self.assertRaises(ValueError): + FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_raises_exception_if_cursor_field_is_nested(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = ["field", "cursor"] + + with self.assertRaises(ValueError): + FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_with_cursor_field_as_list(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = ["cursor"] + + facade = FileBasedStreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade.cursor_field == "cursor" + + def test_create_from_stream_none_message_repository(self): + self._stream.name = "stream" + self._stream.primary_key = "id" + self._stream.cursor_field = "cursor" + self._source.message_repository = None + + with self.assertRaises(ValueError): + FileBasedStreamFacade.create_from_stream(self._stream, self._source, self._logger, {}, self._cursor) + + def test_get_error_display_message_no_display_message(self): + self._stream.get_error_display_message.return_value = "display_message" + + facade = FileBasedStreamFacade.create_from_stream(self._stream, self._source, self._logger, _ANY_STATE, self._cursor) + + expected_display_message = None + e = Exception() + + display_message = facade.get_error_display_message(e) + + assert expected_display_message == display_message + + def test_get_error_display_message_with_display_message(self): + self._stream.get_error_display_message.return_value = "display_message" + + facade = FileBasedStreamFacade.create_from_stream(self._stream, self._source, self._logger, _ANY_STATE, self._cursor) + + expected_display_message = "display_message" + e = ExceptionWithDisplayMessage("display_message") + + display_message = facade.get_error_display_message(e) + + assert expected_display_message == display_message + + +@pytest.mark.parametrize( + "exception, expected_display_message", + [ + pytest.param(Exception("message"), None, id="test_no_display_message"), + pytest.param(ExceptionWithDisplayMessage("message"), "message", id="test_no_display_message"), + ], +) +def test_get_error_display_message(exception, expected_display_message): + stream = Mock() + legacy_stream = Mock() + cursor = Mock(spec=Cursor) + facade = FileBasedStreamFacade(stream, legacy_stream, cursor, Mock().Mock(), Mock()) + + display_message = facade.get_error_display_message(exception) + + assert display_message == expected_display_message diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py new file mode 100644 index 000000000000..027038b2ae98 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py @@ -0,0 +1,458 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +from datetime import datetime +from typing import Any, Dict, List, MutableMapping, Optional, Tuple +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import AirbyteStateMessage, SyncMode +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from freezegun import freeze_time + +DATE_TIME_FORMAT = FileBasedConcurrentCursor.DATE_TIME_FORMAT +MOCK_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = 3 + + +def _make_cursor(input_state: Optional[MutableMapping[str, Any]]) -> FileBasedConcurrentCursor: + stream = MagicMock() + stream.name = "test" + stream.namespace = None + stream_config = MagicMock() + stream_config.days_to_sync_if_history_is_full = MOCK_DAYS_TO_SYNC_IF_HISTORY_IS_FULL + cursor = FileBasedConcurrentCursor( + stream_config, + stream.name, + None, + input_state, + MagicMock(), + ConnectorStateManager( + stream_instance_map={stream.name: stream}, + state=[AirbyteStateMessage.parse_obj(input_state)] if input_state is not None else None, + ), + CursorField(FileBasedConcurrentCursor.CURSOR_FIELD), + ) + return cursor + + +@pytest.mark.parametrize( + "input_state, expected_cursor_value", + [ + pytest.param({}, (datetime.min, ""), id="no-state-gives-min-cursor"), + pytest.param({"history": {}}, (datetime.min, ""), id="missing-cursor-field-gives-min-cursor"), + pytest.param( + {"history": {"a.csv": "2021-01-01T00:00:00.000000Z"}, "_ab_source_file_last_modified": "2021-01-01T00:00:00.000000Z_a.csv"}, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + id="cursor-value-matches-earliest-file", + ), + pytest.param( + {"history": {"a.csv": "2021-01-01T00:00:00.000000Z"}, "_ab_source_file_last_modified": "2020-01-01T00:00:00.000000Z_a.csv"}, + (datetime.strptime("2020-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + id="cursor-value-is-earlier", + ), + pytest.param( + {"history": {"a.csv": "2022-01-01T00:00:00.000000Z"}, "_ab_source_file_last_modified": "2021-01-01T00:00:00.000000Z_a.csv"}, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + id="cursor-value-is-later", + ), + pytest.param( + { + "history": { + "a.csv": "2021-01-01T00:00:00.000000Z", + "b.csv": "2021-01-02T00:00:00.000000Z", + "c.csv": "2021-01-03T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2021-01-04T00:00:00.000000Z_d.csv", + }, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + id="cursor-not-earliest", + ), + pytest.param( + {"history": {"b.csv": "2020-12-31T00:00:00.000000Z"}, "_ab_source_file_last_modified": "2021-01-01T00:00:00.000000Z_a.csv"}, + (datetime.strptime("2020-12-31T00:00:00.000000Z", DATE_TIME_FORMAT), "b.csv"), + id="state-with-cursor-and-earlier-history", + ), + pytest.param( + {"history": {"b.csv": "2021-01-02T00:00:00.000000Z"}, "_ab_source_file_last_modified": "2021-01-01T00:00:00.000000Z_a.csv"}, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + id="state-with-cursor-and-later-history", + ), + ], +) +def test_compute_prev_sync_cursor(input_state: MutableMapping[str, Any], expected_cursor_value: Tuple[datetime, str]): + cursor = _make_cursor(input_state) + assert cursor._compute_prev_sync_cursor(input_state) == expected_cursor_value + + +@pytest.mark.parametrize( + "initial_state, pending_files, file_to_add, expected_history, expected_pending_files, expected_cursor_value", + [ + pytest.param( + {"history": {}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [], + "2021-01-05T00:00:00.000000Z_newfile.csv", + id="add-to-empty-history-single-pending-file", + ), + pytest.param( + {"history": {}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z"), ("pending.csv", "2020-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [("pending.csv", "2020-01-05T00:00:00.000000Z")], + "2020-01-05T00:00:00.000000Z_pending.csv", + id="add-to-empty-history-pending-file-is-older", + ), + pytest.param( + {"history": {}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z"), ("pending.csv", "2022-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [("pending.csv", "2022-01-05T00:00:00.000000Z")], + "2022-01-05T00:00:00.000000Z_pending.csv", + id="add-to-empty-history-pending-file-is-newer", + ), + pytest.param( + {"history": {"existing.csv": "2021-01-04T00:00:00.000000Z"}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"existing.csv": "2021-01-04T00:00:00.000000Z", "newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [], + "2021-01-05T00:00:00.000000Z_newfile.csv", + id="add-to-nonempty-history-single-pending-file", + ), + pytest.param( + {"history": {"existing.csv": "2021-01-04T00:00:00.000000Z"}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z"), ("pending.csv", "2020-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"existing.csv": "2021-01-04T00:00:00.000000Z", "newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [("pending.csv", "2020-01-05T00:00:00.000000Z")], + "2020-01-05T00:00:00.000000Z_pending.csv", + id="add-to-nonempty-history-pending-file-is-older", + ), + pytest.param( + {"history": {"existing.csv": "2021-01-04T00:00:00.000000Z"}}, + [("newfile.csv", "2021-01-05T00:00:00.000000Z"), ("pending.csv", "2022-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"existing.csv": "2021-01-04T00:00:00.000000Z", "newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [("pending.csv", "2022-01-05T00:00:00.000000Z")], + "2022-01-05T00:00:00.000000Z_pending.csv", + id="add-to-nonempty-history-pending-file-is-newer", + ), + ], +) +def test_add_file( + initial_state: MutableMapping[str, Any], + pending_files: List[Tuple[str, str]], + file_to_add: Tuple[str, str], + expected_history: Dict[str, Any], + expected_pending_files: List[Tuple[str, str]], + expected_cursor_value: str, +): + cursor = _make_cursor(initial_state) + mock_message_repository = MagicMock() + cursor._message_repository = mock_message_repository + stream = MagicMock() + + cursor.set_pending_partitions( + [ + FileBasedStreamPartition( + stream, + {"files": [RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT))]}, + mock_message_repository, + SyncMode.full_refresh, + FileBasedConcurrentCursor.CURSOR_FIELD, + initial_state, + cursor, + ) + for uri, timestamp in pending_files + ] + ) + + uri, timestamp = file_to_add + cursor.add_file(RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT))) + assert cursor._file_to_datetime_history == expected_history + assert cursor._pending_files == { + uri: RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT)) for uri, timestamp in expected_pending_files + } + assert ( + mock_message_repository.emit_message.call_args_list[0].args[0].state.data["test"]["_ab_source_file_last_modified"] + == expected_cursor_value + ) + + +@pytest.mark.parametrize( + "initial_state, pending_files, file_to_add, expected_history, expected_pending_files, expected_cursor_value", + [ + pytest.param( + {"history": {}}, + [], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [], + "2021-01-05T00:00:00.000000Z_newfile.csv", + id="add-to-empty-history-no-pending-files", + ), + pytest.param( + {"history": {}}, + [("pending.csv", "2021-01-05T00:00:00.000000Z")], + ("newfile.csv", "2021-01-05T00:00:00.000000Z"), + {"newfile.csv": "2021-01-05T00:00:00.000000Z"}, + [("pending.csv", "2021-01-05T00:00:00.000000Z")], + "2021-01-05T00:00:00.000000Z_pending.csv", + id="add-to-empty-history-file-not-in-pending-files", + ), + ], +) +def test_add_file_invalid( + initial_state: MutableMapping[str, Any], + pending_files: List[Tuple[str, str]], + file_to_add: Tuple[str, str], + expected_history: Dict[str, Any], + expected_pending_files: List[Tuple[str, str]], + expected_cursor_value: str, +): + cursor = _make_cursor(initial_state) + cursor._pending_files = { + uri: RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT)) for uri, timestamp in pending_files + } + mock_message_repository = MagicMock() + cursor._message_repository = mock_message_repository + + uri, timestamp = file_to_add + cursor.add_file(RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT))) + assert cursor._file_to_datetime_history == expected_history + assert cursor._pending_files == { + uri: RemoteFile(uri=uri, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT)) for uri, timestamp in expected_pending_files + } + assert mock_message_repository.emit_message.call_args_list[0].args[0].log.level.value == "WARN" + assert ( + mock_message_repository.emit_message.call_args_list[1].args[0].state.data["test"]["_ab_source_file_last_modified"] + == expected_cursor_value + ) + + +@pytest.mark.parametrize( + "input_state, pending_files, expected_cursor_value", + [ + pytest.param({}, [], f"{datetime.min.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}_", id="no-state-no-pending"), + pytest.param( + {"history": {"a.csv": "2021-01-01T00:00:00.000000Z"}}, [], "2021-01-01T00:00:00.000000Z_a.csv", id="no-pending-with-history" + ), + pytest.param( + {"history": {}}, [("b.csv", "2021-01-02T00:00:00.000000Z")], "2021-01-02T00:00:00.000000Z_b.csv", id="pending-no-history" + ), + pytest.param( + {"history": {"a.csv": "2022-01-01T00:00:00.000000Z"}}, + [("b.csv", "2021-01-02T00:00:00.000000Z")], + "2021-01-01T00:00:00.000000Z_a.csv", + id="with-pending-before-history", + ), + pytest.param( + {"history": {"a.csv": "2021-01-01T00:00:00.000000Z"}}, + [("b.csv", "2022-01-02T00:00:00.000000Z")], + "2022-01-01T00:00:00.000000Z_a.csv", + id="with-pending-after-history", + ), + ], +) +def test_get_new_cursor_value(input_state: MutableMapping[str, Any], pending_files: List[Tuple[str, str]], expected_cursor_value: str): + cursor = _make_cursor(input_state) + pending_partitions = [] + for url, timestamp in pending_files: + partition = MagicMock() + partition.to_slice = lambda *args, **kwargs: { + "files": [RemoteFile(uri=url, last_modified=datetime.strptime(timestamp, DATE_TIME_FORMAT))] + } + pending_partitions.append(partition) + + cursor.set_pending_partitions(pending_partitions) + + +@pytest.mark.parametrize( + "all_files, history, is_history_full, prev_cursor_value, expected_files_to_sync", + [ + pytest.param( + [RemoteFile(uri="new.csv", last_modified=datetime.strptime("2021-01-03T00:00:00.000000Z", "%Y-%m-%dT%H:%M:%S.%fZ"))], + {}, + False, + (datetime.min, ""), + ["new.csv"], + id="empty-history-one-new-file", + ), + pytest.param( + [RemoteFile(uri="a.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000000Z", "%Y-%m-%dT%H:%M:%S.%fZ"))], + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + (datetime.min, ""), + ["a.csv"], + id="non-empty-history-file-in-history-modified", + ), + pytest.param( + [RemoteFile(uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000000Z", "%Y-%m-%dT%H:%M:%S.%fZ"))], + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + (datetime.min, ""), + [], + id="non-empty-history-file-in-history-not-modified", + ), + ], +) +def test_get_files_to_sync(all_files, history, is_history_full, prev_cursor_value, expected_files_to_sync): + cursor = _make_cursor({}) + cursor._file_to_datetime_history = history + cursor._prev_cursor_value = prev_cursor_value + cursor._is_history_full = MagicMock(return_value=is_history_full) + files_to_sync = list(cursor.get_files_to_sync(all_files, MagicMock())) + assert [f.uri for f in files_to_sync] == expected_files_to_sync + + +@freeze_time("2023-06-16T00:00:00Z") +@pytest.mark.parametrize( + "file_to_check, history, is_history_full, prev_cursor_value, sync_start, expected_should_sync", + [ + pytest.param( + RemoteFile(uri="new.csv", last_modified=datetime.strptime("2021-01-03T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + False, + (datetime.min, ""), + datetime.min, + True, + id="file-not-in-history-not-full-old-cursor", + ), + pytest.param( + RemoteFile(uri="new.csv", last_modified=datetime.strptime("2021-01-03T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + False, + (datetime.strptime("2024-01-02T00:00:00.000000Z", DATE_TIME_FORMAT), ""), + datetime.min, + True, + id="file-not-in-history-not-full-new-cursor", + ), + pytest.param( + RemoteFile(uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + (datetime.min, ""), + datetime.min, + False, + id="file-in-history-not-modified", + ), + pytest.param( + RemoteFile(uri="a.csv", last_modified=datetime.strptime("2020-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + (datetime.min, ""), + datetime.min, + False, + id="file-in-history-modified-before", + ), + pytest.param( + RemoteFile(uri="a.csv", last_modified=datetime.strptime("2022-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + (datetime.min, ""), + datetime.min, + True, + id="file-in-history-modified-after", + ), + pytest.param( + RemoteFile(uri="new.csv", last_modified=datetime.strptime("2022-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + True, + (datetime.strptime("2021-01-02T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + datetime.min, + True, + id="history-full-file-modified-after-cursor", + ), + pytest.param( + RemoteFile(uri="new1.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + True, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "new0.csv"), + datetime.min, + True, + id="history-full-modified-eq-cursor-uri-gt", + ), + pytest.param( + RemoteFile(uri="new0.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + True, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "new1.csv"), + datetime.min, + False, + id="history-full-modified-eq-cursor-uri-lt", + ), + pytest.param( + RemoteFile(uri="new.csv", last_modified=datetime.strptime("2020-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + True, + (datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + datetime.min, + True, + id="history-full-modified-before-cursor-and-after-sync-start", + ), + pytest.param( + RemoteFile(uri="new.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT)), + {}, + True, + (datetime.strptime("2022-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), "a.csv"), + datetime.strptime("2024-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), + False, + id="history-full-modified-before-cursor-and-before-sync-start", + ), + ], +) +def test_should_sync_file( + file_to_check: RemoteFile, + history: Dict[str, Any], + is_history_full: bool, + prev_cursor_value: Tuple[datetime, str], + sync_start: datetime, + expected_should_sync: bool, +): + cursor = _make_cursor({}) + cursor._file_to_datetime_history = history + cursor._prev_cursor_value = prev_cursor_value + cursor._sync_start = sync_start + cursor._is_history_full = MagicMock(return_value=is_history_full) + should_sync = cursor._should_sync_file(file_to_check, MagicMock()) + assert should_sync == expected_should_sync + + +@freeze_time("2023-06-16T00:00:00Z") +@pytest.mark.parametrize( + "input_history, is_history_full, expected_start_time", + [ + pytest.param({}, False, datetime.min, id="empty-history"), + pytest.param( + {"a.csv": "2021-01-01T00:00:00.000000Z"}, + False, + datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), + id="non-full-history", + ), + pytest.param( + {f"file{i}.csv": f"2021-01-0{i}T00:00:00.000000Z" for i in range(1, 4)}, # all before the time window + True, + datetime.strptime("2021-01-01T00:00:00.000000Z", DATE_TIME_FORMAT), # Time window start time + id="full-history-earliest-before-window", + ), + pytest.param( + {f"file{i}.csv": f"2024-01-0{i}T00:00:00.000000Z" for i in range(1, 4)}, # all after the time window + True, + datetime.strptime("2023-06-13T00:00:00.000000Z", DATE_TIME_FORMAT), # Earliest file time + id="full-history-earliest-after-window", + ), + ], +) +def test_compute_start_time(input_history, is_history_full, expected_start_time, monkeypatch): + cursor = _make_cursor({"history": input_history}) + cursor._file_to_datetime_history = input_history + cursor._is_history_full = MagicMock(return_value=is_history_full) + assert cursor._compute_start_time() == expected_start_time diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py index e0c5f59623f5..e93eb6bbfc5e 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py @@ -2,15 +2,18 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import traceback import unittest from datetime import datetime, timezone from typing import Any, Iterable, Iterator, Mapping from unittest.mock import Mock import pytest -from airbyte_cdk.models import Level +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -55,7 +58,10 @@ class MockFormat: ), pytest.param( {"type": "object", "properties": {"prop": {"type": "string"}}}, - {"type": ["null", "object"], "properties": {"prop": {"type": ["null", "string"]}}}, + { + "type": ["null", "object"], + "properties": {"prop": {"type": ["null", "string"]}}, + }, id="deeply-nested-schema", ), ], @@ -90,6 +96,7 @@ def setUp(self) -> None: parsers={MockFormat: self._parser}, validation_policy=self._validation_policy, cursor=self._cursor, + errors_collector=FileBasedErrorsCollector(), ) def test_when_read_records_from_slice_then_return_records(self) -> None: @@ -97,14 +104,19 @@ def test_when_read_records_from_slice_then_return_records(self) -> None: messages = list(self._stream.read_records_from_slice({"files": [RemoteFile(uri="uri", last_modified=self._NOW)]})) assert list(map(lambda message: message.record.data["data"], messages)) == [self._A_RECORD] - def test_given_exception_when_read_records_from_slice_then_do_process_other_files(self) -> None: + def test_given_exception_when_read_records_from_slice_then_do_process_other_files( + self, + ) -> None: """ The current behavior for source-s3 v3 does not fail sync on some errors and hence, we will keep this behaviour for now. One example we can easily reproduce this is by having a file with gzip extension that is not actually a gzip file. The reader will fail to open the file but the sync won't fail. Ticket: https://github.com/airbytehq/airbyte/issues/29680 """ - self._parser.parse_records.side_effect = [ValueError("An error"), [self._A_RECORD]] + self._parser.parse_records.side_effect = [ + ValueError("An error"), + [self._A_RECORD], + ] messages = list( self._stream.read_records_from_slice( @@ -120,7 +132,9 @@ def test_given_exception_when_read_records_from_slice_then_do_process_other_file assert messages[0].log.level == Level.ERROR assert messages[1].record.data["data"] == self._A_RECORD - def test_given_traced_exception_when_read_records_from_slice_then_fail(self) -> None: + def test_given_traced_exception_when_read_records_from_slice_then_fail( + self, + ) -> None: """ When a traced exception is raised, the stream shouldn't try to handle but pass it on to the caller. """ @@ -138,7 +152,9 @@ def test_given_traced_exception_when_read_records_from_slice_then_fail(self) -> ) ) - def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning(self) -> None: + def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning( + self, + ) -> None: self._stream_config.schemaless = False self._validation_policy.record_passes_validation_policy.return_value = False self._parser.parse_records.side_effect = [self._iter([self._A_RECORD, ValueError("An error")])] @@ -183,3 +199,46 @@ def _iter(self, x: Iterable[Any]) -> Iterator[Any]: if isinstance(item, Exception): raise item yield item + + +class TestFileBasedErrorCollector: + test_error_collector: FileBasedErrorsCollector = FileBasedErrorsCollector() + + @pytest.mark.parametrize( + "stream, file, line_no, n_skipped, collector_expected_len", + ( + ("stream_1", "test.csv", 1, 1, 1), + ("stream_2", "test2.csv", 2, 2, 2), + ), + ids=[ + "Single error", + "Multiple errors", + ], + ) + def test_collect_parsing_error(self, stream, file, line_no, n_skipped, collector_expected_len) -> None: + test_error_pattern = "Error parsing record." + # format the error body + test_error = ( + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={stream} file={file} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), + ), + ) + # collecting the error + self.test_error_collector.collect(test_error) + # check the error has been collected + assert len(self.test_error_collector.errors) == collector_expected_len + # check for the patern presence for the collected errors + for error in self.test_error_collector.errors: + assert test_error_pattern in error[0].log.message + + def test_yield_and_raise_collected(self) -> None: + # we expect the following method will raise the AirbyteTracedException + with pytest.raises(AirbyteTracedException) as parse_error: + list(self.test_error_collector.yield_and_raise_collected()) + assert parse_error.value.message == "Some errors occured while reading from the source." + assert parse_error.value.internal_message == "Please check the logged errors for more information." diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py index 6c2327444acd..5a7a7b72ff9b 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py @@ -26,6 +26,29 @@ success_multi_stream_scenario, success_user_provided_schema_scenario, ) +from unit_tests.sources.file_based.scenarios.concurrent_incremental_scenarios import ( + multi_csv_different_timestamps_scenario_concurrent, + multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer, + multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older, + multi_csv_per_timestamp_scenario_concurrent, + multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newer, + multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_older, + multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_newer, + multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_older, + multi_csv_same_timestamp_scenario_concurrent, + multi_csv_skip_file_if_already_in_history_concurrent, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_newer, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_older, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_newer, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_older, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_newer, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_older, + single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurrent, + single_csv_input_state_is_earlier_scenario_concurrent, + single_csv_input_state_is_later_scenario_concurrent, + single_csv_no_input_state_scenario_concurrent, +) from unit_tests.sources.file_based.scenarios.csv_scenarios import ( csv_autogenerate_column_names_scenario, csv_custom_bool_values_scenario, @@ -48,6 +71,7 @@ csv_strings_can_be_null_not_quoted_scenario, earlier_csv_scenario, empty_schema_inference_scenario, + invalid_csv_multi_scenario, invalid_csv_scenario, multi_csv_scenario, multi_csv_stream_n_file_exceeds_limit_for_inference, @@ -102,6 +126,7 @@ corrupted_file_scenario, no_file_extension_unstructured_scenario, simple_markdown_scenario, + simple_txt_scenario, simple_unstructured_scenario, unstructured_invalid_file_type_discover_scenario_no_skip, unstructured_invalid_file_type_discover_scenario_skip, @@ -131,6 +156,7 @@ csv_multi_stream_scenario, csv_single_stream_scenario, invalid_csv_scenario, + invalid_csv_multi_scenario, single_csv_scenario, multi_csv_scenario, multi_csv_stream_n_file_exceeds_limit_for_inference, @@ -204,12 +230,34 @@ parquet_with_invalid_config_scenario, single_partitioned_parquet_scenario, simple_markdown_scenario, + simple_txt_scenario, simple_unstructured_scenario, corrupted_file_scenario, no_file_extension_unstructured_scenario, unstructured_invalid_file_type_discover_scenario_no_skip, unstructured_invalid_file_type_discover_scenario_skip, unstructured_invalid_file_type_read_scenario, + multi_csv_different_timestamps_scenario_concurrent, + multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer, + multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older, + multi_csv_per_timestamp_scenario_concurrent, + multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newer, + multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_older, + multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_newer, + multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor_is_older, + multi_csv_same_timestamp_scenario_concurrent, + multi_csv_skip_file_if_already_in_history_concurrent, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_newer, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario_concurrent_cursor_is_older, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_newer, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario_concurrent_cursor_is_older, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_newer, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_is_older, + single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurrent, + single_csv_input_state_is_earlier_scenario_concurrent, + single_csv_input_state_is_later_scenario_concurrent, + single_csv_no_input_state_scenario_concurrent, ] read_scenarios = discover_scenarios + [ diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py index 747d22a31a1f..47df14a09403 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py @@ -13,6 +13,7 @@ from airbyte_cdk.entrypoint import launch from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -72,9 +73,27 @@ def assert_exception(expected_exception: type[BaseException], output: Entrypoint def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[AbstractSource]) -> None: records, log_messages = output.records_and_state_messages, output.logs logs = [message.log for message in log_messages if message.log.level.value in scenario.log_levels] - expected_records = scenario.expected_records - assert len(records) == len(expected_records) - for actual, expected in zip(records, expected_records): + if scenario.expected_records is None: + return + + expected_records = [r for r in scenario.expected_records] if scenario.expected_records else [] + + sorted_expected_records = sorted( + filter(lambda e: "data" in e, expected_records), + key=lambda record: ",".join( + f"{k}={v}" for k, v in sorted(record["data"].items(), key=lambda items: (items[0], items[1])) if k != "emitted_at" + ), + ) + sorted_records = sorted( + filter(lambda r: r.record, records), + key=lambda record: ",".join( + f"{k}={v}" for k, v in sorted(record.record.data.items(), key=lambda items: (items[0], items[1])) if k != "emitted_at" + ), + ) + + assert len(sorted_records) == len(sorted_expected_records) + + for actual, expected in zip(sorted_records, sorted_expected_records): if actual.record: assert len(actual.record.data) == len(expected["data"]) for key, value in actual.record.data.items(): @@ -83,7 +102,16 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac else: assert value == expected["data"][key] assert actual.record.stream == expected["stream"] - elif actual.state: + + expected_states = list(filter(lambda e: "data" not in e, expected_records)) + states = list(filter(lambda r: r.state, records)) + + if hasattr(scenario.source, "cursor_cls") and issubclass(scenario.source.cursor_cls, AbstractConcurrentFileBasedCursor): + # Only check the last state emitted because we don't know the order the others will be in. + # This may be needed for non-file-based concurrent scenarios too. + assert states[-1].state.data == expected_states[-1] + else: + for actual, expected in zip(states, expected_states): # states should be emitted in sorted order assert actual.state.data == expected if scenario.expected_logs: @@ -129,7 +157,7 @@ def verify_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: Tes output = check(capsys, tmp_path, scenario) if expected_msg: # expected_msg is a string. what's the expected value field? - assert expected_msg.value in output["message"] # type: ignore + assert expected_msg in output["message"] # type: ignore assert output["status"] == scenario.expected_check_status else: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py index 2ab346d9814d..af3161e07199 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py @@ -2,7 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from airbyte_cdk.sources.streams.concurrent.cursor import CursorField -from airbyte_cdk.sources.streams.concurrent.state_converter import ConcurrencyCompatibleStateType +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder from unit_tests.sources.streams.concurrent.scenarios.utils import MockStream @@ -72,10 +73,11 @@ [ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 1}], "state_type": ConcurrencyCompatibleStateType.date_range.value, "legacy": {}}}, + {"stream1": {"cursor_field": 1}}, {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 2}], "state_type": ConcurrencyCompatibleStateType.date_range.value, "legacy": {}}}, + {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -84,7 +86,7 @@ ) -LEGACY_STATE = [{"type": "STREAM", "stream": {"stream_state": {"created": 0}, "stream_descriptor": {"name": "stream1"}}}] +LEGACY_STATE = StateBuilder().with_stream_state("stream1", {"cursor_field": 0}).build() test_incremental_stream_without_slice_boundaries_with_legacy_state = ( TestScenarioBuilder() .set_name("test_incremental_stream_without_slice_boundaries_with_legacy_state") @@ -141,29 +143,18 @@ ) ] ) - .set_incremental(CursorField(["cursor_field"]), ("from", "to")) + .set_incremental(CursorField("cursor_field"), ("from", "to")) .set_input_state(LEGACY_STATE) ) .set_expected_records( [ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, - { - "stream1": { - "slices": [{"start": 0, "end": 1}], - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "legacy": {"created": 0}, - } - }, + {"stream1": {"cursor_field": 1}}, {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, - { - "stream1": { - "slices": [{"start": 0, "end": 2}], - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "legacy": {"created": 0}, - } - }, + {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -172,18 +163,17 @@ ) -CONCURRENT_STATE = [ - { - "type": "STREAM", - "stream": { - "stream_state": { - "slices": [{"start": 0, "end": 0}], - "state_type": ConcurrencyCompatibleStateType.date_range.value, - }, - "stream_descriptor": {"name": "stream1"}, +CONCURRENT_STATE = ( + StateBuilder() + .with_stream_state( + "stream1", + { + "slices": [{"start": 0, "end": 0}], + "state_type": ConcurrencyCompatibleStateType.date_range.value, }, - }, -] + ) + .build() +) test_incremental_stream_without_slice_boundaries_with_concurrent_state = ( TestScenarioBuilder() .set_name("test_incremental_stream_without_slice_boundaries_with_concurrent_state") @@ -240,17 +230,18 @@ ) ] ) - .set_incremental(CursorField(["cursor_field"]), ("from", "to")) + .set_incremental(CursorField("cursor_field"), ("from", "to")) .set_input_state(CONCURRENT_STATE) ) .set_expected_records( [ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 1}], "state_type": ConcurrencyCompatibleStateType.date_range.value}}, + {"stream1": {"cursor_field": 1}}, {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 2}], "state_type": ConcurrencyCompatibleStateType.date_range.value}}, + {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py index 601b33d25ab6..10c93aebb334 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py @@ -5,20 +5,29 @@ import logging from typing import Any, List, Mapping, Optional, Tuple, Union -from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ( + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, NoopCursor -from airbyte_cdk.sources.streams.concurrent.state_converter import EpochValueConcurrentStreamStateConverter +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter from airbyte_protocol.models import ConfiguredAirbyteStream from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger +_CURSOR_FIELD = "cursor_field" _NO_STATE = None @@ -49,28 +58,34 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in self._streams}, state=self._state) - state_converter = StreamFacadeConcurrentConnectorStateConverter("created") + state_manager = ConnectorStateManager( + stream_instance_map={s.name: AirbyteStream(name=s.name, namespace=None, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]) for s in self._streams}, + state=self._state, + ) # The input values into the AirbyteStream are dummy values; the connector state manager only uses `name` and `namespace` + + state_converter = StreamFacadeConcurrentConnectorStateConverter() + stream_states = [state_manager.get_stream_state(stream.name, stream.namespace) for stream in self._streams] return [ StreamFacade.create_from_stream( stream, self, stream.logger, - state_converter.get_concurrent_stream_state(state_manager.get_stream_state(stream.name, stream.namespace)), + state, ConcurrentCursor( stream.name, stream.namespace, - state_converter.get_concurrent_stream_state(state_manager.get_stream_state(stream.name, stream.namespace)), + state, self.message_repository, # type: ignore # for this source specifically, we always return `InMemoryMessageRepository` state_manager, state_converter, self._cursor_field, self._cursor_boundaries, + None, ) if self._cursor_field else NoopCursor(), ) - for stream in self._streams + for stream, state in zip(self._streams, stream_states) ] @property @@ -120,6 +135,6 @@ def set_input_state(self, state: List[Mapping[str, Any]]) -> "StreamFacadeSource self._input_state = state return self - def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> StreamFacadeSource: + def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> StreamFacadeSource: threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool") - return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, self._input_state) + return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py index b6ef948ef187..8964024d2ca0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py @@ -357,10 +357,11 @@ [ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 1}], "state_type": "date-range", "legacy": {}}}, + {"stream1": {"cursor_field": 1}}, {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 2}], "state_type": "date-range", "legacy": {}}}, + {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -402,7 +403,8 @@ [ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 3}, "stream": "stream1"}, - {"stream1": {"slices": [{"start": 0, "end": 3}], "state_type": "date-range", "legacy": {}}}, + {"stream1": {"cursor_field": 3}}, + {"stream1": {"cursor_field": 3}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py index 2f4ab9b9fccb..e1eb81445d4a 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py @@ -4,6 +4,7 @@ import logging from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder @@ -29,6 +30,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_with_slice_logger = DefaultStream( @@ -46,6 +48,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_with_primary_key = DefaultStream( @@ -63,6 +66,7 @@ primary_key=["id"], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_multiple_partitions = DefaultStream( @@ -83,6 +87,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_multiple_partitions_concurrency_level_two = DefaultStream( @@ -103,6 +108,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _stream_raising_exception = DefaultStream( @@ -120,6 +126,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) test_concurrent_cdk_single_stream = ( @@ -246,6 +253,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ), ] ) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py index 943aea30dbba..87a65ea6efd8 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py @@ -119,7 +119,7 @@ def __init__(self): self._streams: List[DefaultStream] = [] self._message_repository = None - def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> ConcurrentCdkSource: + def build(self, configured_catalog: Optional[Mapping[str, Any]], _, __) -> ConcurrentCdkSource: return ConcurrentCdkSource(self._streams, self._message_repository, 1, 1) def set_streams(self, streams: List[DefaultStream]) -> "ConcurrentSourceBuilder": diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py deleted file mode 100644 index 397c8a194840..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from queue import Queue -from unittest.mock import Mock - -import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel -from airbyte_cdk.sources.streams.concurrent.adapters import StreamPartition -from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer - - -@pytest.mark.parametrize( - "slices", [pytest.param([], id="test_no_partitions"), pytest.param([{"partition": 1}, {"partition": 2}], id="test_two_partitions")] -) -def test_partition_generator(slices): - queue = Queue() - partition_generator = PartitionEnqueuer(queue) - - stream = Mock() - message_repository = Mock() - sync_mode = SyncMode.full_refresh - cursor_field = None - state = None - cursor = Mock() - partitions = [StreamPartition(stream, s, message_repository, sync_mode, cursor_field, state, cursor) for s in slices] - stream.generate_partitions.return_value = iter(partitions) - - partition_generator.generate_partitions(stream) - - actual_partitions = [] - while partition := queue.get(False): - if isinstance(partition, PartitionGenerationCompletedSentinel): - break - actual_partitions.append(partition) - - assert actual_partitions == partitions diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py index a520cc7d9c7e..3e0e00b62d32 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py @@ -3,7 +3,7 @@ # import logging import unittest -from unittest.mock import Mock +from unittest.mock import Mock, call import freezegun from airbyte_cdk.models import ( @@ -32,6 +32,7 @@ _STREAM_NAME = "stream" _ANOTHER_STREAM_NAME = "stream2" +_ANY_AIRBYTE_MESSAGE = Mock(spec=AirbyteMessage) class TestConcurrentReadProcessor(unittest.TestCase): @@ -40,13 +41,11 @@ def setUp(self): self._thread_pool_manager = Mock(spec=ThreadPoolManager) self._an_open_partition = Mock(spec=Partition) - self._an_open_partition.is_closed.return_value = False self._log_message = Mock(spec=LogMessage) self._an_open_partition.to_slice.return_value = self._log_message self._an_open_partition.stream_name.return_value = _STREAM_NAME self._a_closed_partition = Mock(spec=Partition) - self._a_closed_partition.is_closed.return_value = True self._a_closed_partition.stream_name.return_value = _ANOTHER_STREAM_NAME self._logger = Mock(spec=logging.Logger) @@ -76,6 +75,19 @@ def setUp(self): self._record.stream_name = _STREAM_NAME self._record.data = self._record_data + def test_stream_is_not_done_initially(self): + stream_instances_to_read_from = [self._stream] + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + assert not handler._is_stream_done(self._stream.name) + def test_handle_partition_done_no_other_streams_to_generate_partitions_for(self): stream_instances_to_read_from = [self._stream] @@ -99,6 +111,10 @@ def test_handle_partition_done_no_other_streams_to_generate_partitions_for(self) @freezegun.freeze_time("2020-01-01T00:00:00") def test_handle_last_stream_partition_done(self): + in_order_validation_mock = Mock() + in_order_validation_mock.attach_mock(self._another_stream, "_another_stream") + in_order_validation_mock.attach_mock(self._message_repository, "_message_repository") + self._message_repository.consume_queue.return_value = iter([_ANY_AIRBYTE_MESSAGE]) stream_instances_to_read_from = [self._another_stream] handler = ConcurrentReadProcessor( @@ -111,12 +127,12 @@ def test_handle_last_stream_partition_done(self): self._partition_reader, ) handler.start_next_partition_generator() - handler.on_partition(self._a_closed_partition) sentinel = PartitionGenerationCompletedSentinel(self._another_stream) - messages = handler.on_partition_generation_completed(sentinel) + messages = list(handler.on_partition_generation_completed(sentinel)) expected_messages = [ + _ANY_AIRBYTE_MESSAGE, AirbyteMessage( type=MessageType.TRACE, trace=AirbyteTraceMessage( @@ -127,9 +143,12 @@ def test_handle_last_stream_partition_done(self): status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE), ), ), - ) + ), ] assert expected_messages == messages + assert in_order_validation_mock.mock_calls.index( + call._another_stream.cursor.ensure_at_least_one_state_emitted + ) < in_order_validation_mock.mock_calls.index(call._message_repository.consume_queue) def test_handle_partition(self): stream_instances_to_read_from = [self._another_stream] @@ -147,7 +166,7 @@ def test_handle_partition(self): handler.on_partition(self._a_closed_partition) self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._a_closed_partition) - assert self._a_closed_partition in handler._streams_to_partitions[_ANOTHER_STREAM_NAME] + assert self._a_closed_partition in handler._streams_to_running_partitions[_ANOTHER_STREAM_NAME] def test_handle_partition_emits_log_message_if_it_should_be_logged(self): stream_instances_to_read_from = [self._stream] @@ -169,15 +188,16 @@ def test_handle_partition_emits_log_message_if_it_should_be_logged(self): self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._an_open_partition) self._message_repository.emit_message.assert_called_with(self._log_message) - assert self._an_open_partition in handler._streams_to_partitions[_STREAM_NAME] + assert self._an_open_partition in handler._streams_to_running_partitions[_STREAM_NAME] + + @freezegun.freeze_time("2020-01-01T00:00:00") def test_handle_on_partition_complete_sentinel_with_messages_from_repository(self): stream_instances_to_read_from = [self._stream] partition = Mock(spec=Partition) log_message = Mock(spec=LogMessage) partition.to_slice.return_value = log_message partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True handler = ConcurrentReadProcessor( stream_instances_to_read_from, @@ -189,6 +209,7 @@ def test_handle_on_partition_complete_sentinel_with_messages_from_repository(sel self._partition_reader, ) handler.start_next_partition_generator() + handler.on_partition(partition) sentinel = PartitionCompleteSentinel(partition) @@ -223,7 +244,8 @@ def test_handle_on_partition_complete_sentinel_yields_status_message_if_the_stre self._partition_reader, ) handler.start_next_partition_generator() - handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream)) + handler.on_partition(self._a_closed_partition) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))) sentinel = PartitionCompleteSentinel(self._a_closed_partition) @@ -254,7 +276,6 @@ def test_handle_on_partition_complete_sentinel_yields_no_status_message_if_the_s log_message = Mock(spec=LogMessage) partition.to_slice.return_value = log_message partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True handler = ConcurrentReadProcessor( stream_instances_to_read_from, @@ -282,7 +303,6 @@ def test_on_record_no_status_message_no_repository_messge(self): log_message = Mock(spec=LogMessage) partition.to_slice.return_value = log_message partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True self._message_repository.consume_queue.return_value = [] handler = ConcurrentReadProcessor( @@ -319,7 +339,6 @@ def test_on_record_with_repository_messge(self): log_message = Mock(spec=LogMessage) partition.to_slice.return_value = log_message partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True slice_logger = Mock(spec=SliceLogger) slice_logger.should_log_slice_message.return_value = True slice_logger.create_slice_log_message.return_value = log_message @@ -370,7 +389,6 @@ def test_on_record_emits_status_message_on_first_record_no_repository_message(se stream_instances_to_read_from = [self._stream] partition = Mock(spec=Partition) partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True handler = ConcurrentReadProcessor( stream_instances_to_read_from, @@ -413,7 +431,6 @@ def test_on_record_emits_status_message_on_first_record_with_repository_message( log_message = Mock(spec=LogMessage) partition.to_slice.return_value = log_message partition.stream_name.return_value = _STREAM_NAME - partition.is_closed.return_value = True self._message_repository.consume_queue.return_value = [ AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")) ] @@ -474,7 +491,70 @@ def test_on_exception_stops_streams_and_raises_an_exception(self): self._message_repository, self._partition_reader, ) - handler._streams_to_partitions = {_STREAM_NAME: {self._an_open_partition}, _ANOTHER_STREAM_NAME: {self._a_closed_partition}} + + handler.start_next_partition_generator() + + another_stream = Mock(spec=AbstractStream) + another_stream.name = _STREAM_NAME + another_stream.as_airbyte_stream.return_value = AirbyteStream( + name=_ANOTHER_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + exception = RuntimeError("Something went wrong") + + messages = [] + + with self.assertRaises(RuntimeError): + for m in handler.on_exception(exception): + messages.append(m) + + expected_message = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_ANOTHER_STREAM_NAME), + status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE), + ), + ), + ), + ] + + assert messages == expected_message + self._thread_pool_manager.shutdown.assert_called_once() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_exception_does_not_stop_streams_that_are_already_done(self): + stream_instances_to_read_from = [self._stream, self._another_stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.start_next_partition_generator() + handler.on_partition(self._an_open_partition) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream))) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))) another_stream = Mock(spec=AbstractStream) another_stream.name = _STREAM_NAME diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py index 3e34dde83d99..94ed5211eabb 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py @@ -11,11 +11,10 @@ from airbyte_cdk.sources.streams.concurrent.cursor import Comparable, ConcurrentCursor, CursorField from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record -from airbyte_cdk.sources.streams.concurrent.state_converter import ConcurrencyCompatibleStateType, EpochValueConcurrentStreamStateConverter +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter _A_STREAM_NAME = "a stream name" _A_STREAM_NAMESPACE = "a stream namespace" -_ANY_STATE = {} _A_CURSOR_FIELD_KEY = "a_cursor_field_key" _NO_PARTITION_IDENTIFIER = None _NO_SLICE = None @@ -40,34 +39,37 @@ class ConcurrentCursorTest(TestCase): def setUp(self) -> None: self._message_repository = Mock(spec=MessageRepository) self._state_manager = Mock(spec=ConnectorStateManager) - self._state_converter = EpochValueConcurrentStreamStateConverter("created") + self._state_converter = EpochValueConcurrentStreamStateConverter() def _cursor_with_slice_boundary_fields(self) -> ConcurrentCursor: return ConcurrentCursor( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - self._state_converter.get_concurrent_stream_state(_ANY_STATE), + {}, self._message_repository, self._state_manager, self._state_converter, CursorField(_A_CURSOR_FIELD_KEY), _SLICE_BOUNDARY_FIELDS, + None, ) def _cursor_without_slice_boundary_fields(self) -> ConcurrentCursor: return ConcurrentCursor( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - self._state_converter.get_concurrent_stream_state(_ANY_STATE), + {}, self._message_repository, self._state_manager, self._state_converter, CursorField(_A_CURSOR_FIELD_KEY), None, + None, ) def test_given_boundary_fields_when_close_partition_then_emit_state(self) -> None: - self._cursor_with_slice_boundary_fields().close_partition( + cursor = self._cursor_with_slice_boundary_fields() + cursor.close_partition( _partition( {_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30}, ) @@ -77,16 +79,21 @@ def test_given_boundary_fields_when_close_partition_then_emit_state(self) -> Non self._state_manager.update_state_for_stream.assert_called_once_with( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - { - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "legacy": _ANY_STATE, - "slices": [ - { - "start": 12, - "end": 30, - }, - ], - }, + {_A_CURSOR_FIELD_KEY: 0}, # State message is updated to the legacy format before being emitted + ) + + def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None: + self._cursor_with_slice_boundary_fields().close_partition( + _partition( + {_LOWER_SLICE_BOUNDARY_FIELD: 0, _UPPER_SLICE_BOUNDARY_FIELD: 30}, + ) + ) + + self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value) + self._state_manager.update_state_for_stream.assert_called_once_with( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {_A_CURSOR_FIELD_KEY: 30}, # State message is updated to the legacy format before being emitted ) def test_given_boundary_fields_and_record_observed_when_close_partition_then_ignore_records(self) -> None: @@ -95,7 +102,7 @@ def test_given_boundary_fields_and_record_observed_when_close_partition_then_ign cursor.close_partition(_partition({_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30})) - assert self._state_manager.update_state_for_stream.call_args_list[0].args[2]["slices"][0]["end"] != _A_VERY_HIGH_CURSOR_VALUE + assert self._state_manager.update_state_for_stream.call_args_list[0].args[2][_A_CURSOR_FIELD_KEY] != _A_VERY_HIGH_CURSOR_VALUE def test_given_no_boundary_fields_when_close_partition_then_emit_state(self) -> None: cursor = self._cursor_without_slice_boundary_fields() @@ -105,16 +112,7 @@ def test_given_no_boundary_fields_when_close_partition_then_emit_state(self) -> self._state_manager.update_state_for_stream.assert_called_once_with( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - { - "state_type": ConcurrencyCompatibleStateType.date_range.value, - "legacy": {}, - "slices": [ - { - "start": 0, - "end": 10, - }, - ], - }, + {"a_cursor_field_key": 10}, ) def test_given_no_boundary_fields_when_close_multiple_partitions_then_raise_exception(self) -> None: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py new file mode 100644 index 000000000000..534dbd580787 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py @@ -0,0 +1,369 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone + +import pytest +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( + EpochValueConcurrentStreamStateConverter, + IsoMillisConcurrentStreamStateConverter, +) + + +@pytest.mark.parametrize( + "converter, input_state, is_compatible", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + {"state_type": "date-range"}, + True, + id="no-input-state-is-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": ConcurrencyCompatibleStateType.date_range.value, + }, + True, + id="input-state-with-date_range-is-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": "fake", + }, + False, + id="input-state-with-fake-state-type-is-not-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + }, + False, + id="input-state-without-state_type-is-not-compatible-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + {"state_type": "date-range"}, + True, + id="no-input-state-is-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": ConcurrencyCompatibleStateType.date_range.value, + }, + True, + id="input-state-with-date_range-is-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": "fake", + }, + False, + id="input-state-with-fake-state-type-is-not-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + }, + False, + id="input-state-without-state_type-is-not-compatible-isomillis", + ), + ], +) +def test_concurrent_stream_state_converter_is_state_message_compatible(converter, input_state, is_compatible): + assert converter.is_state_message_compatible(input_state) == is_compatible + + +@pytest.mark.parametrize( + "converter,start,state,expected_start", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + None, + {}, + EpochValueConcurrentStreamStateConverter().zero_value, + id="epoch-converter-no-state-no-start-start-is-zero-value", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {}, + datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + id="epoch-converter-no-state-with-start-start-is-start", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + None, + {"created_at": 1617030404}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-without-start-start-is-from-state", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030404, + {"created_at": 1617030403}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-before-start-start-is-start", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {"created_at": 1617030404}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-after-start-start-is-from-state", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + None, + {}, + IsoMillisConcurrentStreamStateConverter().zero_value, + id="isomillis-converter-no-state-no-start-start-is-zero-value", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2021-08-22T05:03:27.000Z", + {}, + datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-no-state-with-start-start-is-start", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + None, + {"created_at": "2021-08-22T05:03:27.000Z"}, + datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-without-start-start-is-from-state", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2022-08-22T05:03:27.000Z", + {"created_at": "2021-08-22T05:03:27.000Z"}, + datetime(2022, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-before-start-start-is-start", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2022-08-22T05:03:27.000Z", + {"created_at": "2023-08-22T05:03:27.000Z"}, + datetime(2023, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-after-start-start-is-from-state", + ), + ], +) +def test_get_sync_start(converter, start, state, expected_start): + assert converter._get_sync_start(CursorField("created_at"), state, start) == expected_start + + +@pytest.mark.parametrize( + "converter, start, sequential_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 0, + {}, + { + "legacy": {}, + "slices": [ + { + "start": EpochValueConcurrentStreamStateConverter().zero_value, + "end": EpochValueConcurrentStreamStateConverter().zero_value, + } + ], + "state_type": "date-range", + }, + id="empty-input-state-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {"created": 1617030403}, + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + } + ], + "legacy": {"created": 1617030403}, + }, + id="with-input-state-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2020-01-01T00:00:00.000Z", + {"created": "2021-08-22T05:03:27.000Z"}, + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + "end": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + } + ], + "legacy": {"created": "2021-08-22T05:03:27.000Z"}, + }, + id="with-input-state-isomillis", + ), + ], +) +def test_convert_from_sequential_state(converter, start, sequential_state, expected_output_state): + comparison_format = "%Y-%m-%dT%H:%M:%S.%f" + if expected_output_state["slices"]: + _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start) + assert conversion["state_type"] == expected_output_state["state_type"] + assert conversion["legacy"] == expected_output_state["legacy"] + for actual, expected in zip(conversion["slices"], expected_output_state["slices"]): + assert actual["start"].strftime(comparison_format) == expected["start"].strftime(comparison_format) + assert actual["end"].strftime(comparison_format) == expected["end"].strftime(comparison_format) + else: + _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start) + assert conversion == expected_output_state + + +@pytest.mark.parametrize( + "converter, concurrent_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + } + ], + }, + {"created": 1617030403}, + id="epoch-single-slice", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + { + "start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + ], + }, + {"created": 1648566403}, + id="epoch-overlapping-slices", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + { + "start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + ], + }, + {"created": 1617030403}, + id="epoch-multiple-slices", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + } + ], + }, + {"created": "2021-03-29T15:06:43.000Z"}, + id="isomillis-single-slice", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + { + "start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + ], + }, + {"created": "2022-03-29T15:06:43.000Z"}, + id="isomillis-overlapping-slices", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [ + { + "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + { + "start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + }, + ], + }, + {"created": "2021-03-29T15:06:43.000Z"}, + id="isomillis-multiple-slices", + ), + ], +) +def test_convert_to_sequential_state(converter, concurrent_state, expected_output_state): + assert converter.convert_to_sequential_state(CursorField("created"), concurrent_state) == expected_output_state + + +@pytest.mark.parametrize( + "converter, concurrent_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "start": EpochValueConcurrentStreamStateConverter().zero_value, + }, + {"created": 0}, + id="empty-slices-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + }, + {"created": "2021-08-22T05:03:27.000Z"}, + id="empty-slices-isomillis", + ), + ], +) +def test_convert_to_sequential_state_no_slices_returns_legacy_state(converter, concurrent_state, expected_output_state): + with pytest.raises(RuntimeError): + converter.convert_to_sequential_state(CursorField("created"), concurrent_state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py index 818c2862bb8b..fb40368d98b3 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py @@ -6,7 +6,7 @@ from airbyte_cdk.models import AirbyteStream, SyncMode from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE -from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream @@ -28,6 +28,7 @@ def setUp(self): self._primary_key, self._cursor_field, self._logger, + NoopCursor(), ) def test_get_json_schema(self): @@ -88,6 +89,7 @@ def test_as_airbyte_stream_with_primary_key(self): ["id"], self._cursor_field, self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -119,6 +121,7 @@ def test_as_airbyte_stream_with_composite_primary_key(self): ["id_a", "id_b"], self._cursor_field, self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -150,6 +153,7 @@ def test_as_airbyte_stream_with_a_cursor(self): self._primary_key, "date", self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -174,6 +178,7 @@ def test_as_airbyte_stream_with_namespace(self): self._primary_key, self._cursor_field, self._logger, + NoopCursor(), namespace="test", ) expected_airbyte_stream = AirbyteStream( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py new file mode 100644 index 000000000000..bdcd9ad43318 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import unittest +from queue import Queue +from typing import Callable, Iterable, List +from unittest.mock import Mock, patch + +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem + +_SOME_PARTITIONS: List[Partition] = [Mock(spec=Partition), Mock(spec=Partition)] + + +class PartitionEnqueuerTest(unittest.TestCase): + def setUp(self) -> None: + self._queue: Queue[QueueItem] = Queue() + self._thread_pool_manager = Mock(spec=ThreadPoolManager) + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False + self._partition_generator = PartitionEnqueuer(self._queue, self._thread_pool_manager) + + @patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep") + def test_given_no_partitions_when_generate_partitions_then_do_not_wait(self, mocked_sleep): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True # shouldn't be called but just in case + stream = self._a_stream([]) + + self._partition_generator.generate_partitions(stream) + + assert mocked_sleep.call_count == 0 + + def test_given_no_partitions_when_generate_partitions_then_only_push_sentinel(self): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True + stream = self._a_stream([]) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == [PartitionGenerationCompletedSentinel(stream)] + + def test_given_partitions_when_generate_partitions_then_return_partitions_before_sentinel(self): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False + stream = self._a_stream(_SOME_PARTITIONS) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == _SOME_PARTITIONS + [PartitionGenerationCompletedSentinel(stream)] + + @patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep") + def test_given_partition_but_limit_reached_when_generate_partitions_then_wait_until_not_hitting_limit(self, mocked_sleep): + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.side_effect = [True, True, False] + stream = self._a_stream([Mock(spec=Partition)]) + + self._partition_generator.generate_partitions(stream) + + assert mocked_sleep.call_count == 2 + + def test_given_exception_when_generate_partitions_then_raise(self): + stream = Mock(spec=AbstractStream) + exception = ValueError() + stream.generate_partitions.side_effect = self._partitions_before_raising(_SOME_PARTITIONS, exception) + + self._partition_generator.generate_partitions(stream) + + assert self._consume_queue() == _SOME_PARTITIONS + [exception] + + def _partitions_before_raising(self, partitions: List[Partition], exception: Exception) -> Callable[[], Iterable[Partition]]: + def inner_function() -> Iterable[Partition]: + for partition in partitions: + yield partition + raise exception + + return inner_function + + @staticmethod + def _a_stream(partitions: List[Partition]) -> AbstractStream: + stream = Mock(spec=AbstractStream) + stream.generate_partitions.return_value = iter(partitions) + return stream + + def _consume_queue(self) -> List[QueueItem]: + queue_content: List[QueueItem] = [] + while queue_item := self._queue.get(): + if isinstance(queue_item, (PartitionGenerationCompletedSentinel, Exception)): + queue_content.append(queue_item) + break + queue_content.append(queue_item) + return queue_content diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py index df82432c415f..9e9fb8973949 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py @@ -1,32 +1,67 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import unittest from queue import Queue +from typing import Callable, Iterable, List from unittest.mock import Mock +import pytest from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record -from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem +_RECORDS = [ + Record({"id": 1, "name": "Jack"}, "stream"), + Record({"id": 2, "name": "John"}, "stream"), +] -def test_partition_reader(): - queue = Queue() - partition_reader = PartitionReader(queue) - stream_partition = Mock() - records = [ - Record({"id": 1, "name": "Jack"}, "stream"), - Record({"id": 2, "name": "John"}, "stream"), - ] - stream_partition.read.return_value = iter(records) +class PartitionReaderTest(unittest.TestCase): + def setUp(self) -> None: + self._queue: Queue[QueueItem] = Queue() + self._partition_reader = PartitionReader(self._queue) - partition_reader.process_partition(stream_partition) + def test_given_no_records_when_process_partition_then_only_emit_sentinel(self): + self._partition_reader.process_partition(self._a_partition([])) - actual_records = [] - while record := queue.get(): - if isinstance(record, PartitionCompleteSentinel): + while queue_item := self._queue.get(): + if not isinstance(queue_item, PartitionCompleteSentinel): + pytest.fail("Only one PartitionCompleteSentinel is expected") break - actual_records.append(record) - assert records == actual_records + def test_given_read_partition_successful_when_process_partition_then_queue_records_and_sentinel(self): + self._partition_reader.process_partition(self._a_partition(_RECORDS)) + + actual_records = [] + while queue_item := self._queue.get(): + if isinstance(queue_item, PartitionCompleteSentinel): + break + actual_records.append(queue_item) + + assert _RECORDS == actual_records + + def test_given_exception_when_process_partition_then_queue_records_and_raise_exception(self): + partition = Mock() + exception = ValueError() + partition.read.side_effect = self._read_with_exception(_RECORDS, exception) + + self._partition_reader.process_partition(partition) + + for i in range(len(_RECORDS)): + assert self._queue.get() == _RECORDS[i] + assert self._queue.get() == exception + + def _a_partition(self, records: List[Record]) -> Partition: + partition = Mock(spec=Partition) + partition.read.return_value = iter(records) + return partition + + @staticmethod + def _read_with_exception(records: List[Record], exception: Exception) -> Callable[[], Iterable[Record]]: + def mocked_function() -> Iterable[Record]: + yield from records + raise exception + + return mocked_function diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_state_converter.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_state_converter.py deleted file mode 100644 index da7181861ce4..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_state_converter.py +++ /dev/null @@ -1,225 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, MutableMapping - -import pytest -from airbyte_cdk.models import ( - AirbyteStateBlob, - AirbyteStateMessage, - AirbyteStateType, - AirbyteStream, - AirbyteStreamState, - StreamDescriptor, - SyncMode, -) -from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager -from airbyte_cdk.sources.streams.concurrent.state_converter import ( - ConcurrencyCompatibleStateType, - ConcurrentStreamStateConverter, - EpochValueConcurrentStreamStateConverter, -) - - -class MockConcurrentConnectorStateConverter(ConcurrentStreamStateConverter): - def convert_from_sequential_state(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - state["state_type"] = ConcurrencyCompatibleStateType.date_range.value - return state - - def convert_to_sequential_state(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - state.pop("state_type") - return state - - @staticmethod - def increment(timestamp: Any) -> Any: - return timestamp + 1 - - -@pytest.mark.parametrize( - "stream, input_state, expected_output_state", - [ - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - [], - {"state_type": ConcurrencyCompatibleStateType.date_range.value}, - id="no-input-state", - ), - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - [ - AirbyteStateMessage( - type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="stream1", namespace=None), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2022_05_22"}), - ), - ), - ], - {"created_at": "2022_05_22", "state_type": ConcurrencyCompatibleStateType.date_range.value}, - id="incompatible-input-state", - ), - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - [ - AirbyteStateMessage( - type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="stream1", namespace=None), - stream_state=AirbyteStateBlob.parse_obj( - { - "created_at": "2022_05_22", - "state_type": ConcurrencyCompatibleStateType.date_range.value, - }, - ), - ), - ), - ], - {"created_at": "2022_05_22", "state_type": ConcurrencyCompatibleStateType.date_range.value}, - id="compatible-input-state", - ), - ], -) -def test_concurrent_connector_state_manager_get_stream_state(stream, input_state, expected_output_state): - state_manager = ConnectorStateManager({"stream1": stream}, input_state) - state_converter = MockConcurrentConnectorStateConverter() - assert state_converter.get_concurrent_stream_state(state_manager.get_stream_state("stream1", None)) == expected_output_state - - -@pytest.mark.parametrize( - "input_state, is_compatible", - [ - pytest.param( - {}, - False, - id="no-input-state-is-not-compatible", - ), - pytest.param( - { - "created_at": "2022_05_22", - "state_type": ConcurrencyCompatibleStateType.date_range.value, - }, - True, - id="input-state-with-date_range-is-compatible", - ), - pytest.param( - { - "created_at": "2022_05_22", - "state_type": "fake", - }, - False, - id="input-state-with-fake-state-type-is-not-compatible", - ), - pytest.param( - { - "created_at": "2022_05_22", - }, - False, - id="input-state-without-state_type-is-not-compatible", - ), - ], -) -def test_concurrent_stream_state_converter_is_state_message_compatible(input_state, is_compatible): - assert ConcurrentStreamStateConverter.is_state_message_compatible(input_state) == is_compatible - - -@pytest.mark.parametrize( - "input_intervals, expected_merged_intervals", - [ - pytest.param( - [], - [], - id="no-intervals", - ), - pytest.param( - [{"start": 0, "end": 1}], - [{"start": 0, "end": 1}], - id="single-interval", - ), - pytest.param( - [{"start": 0, "end": 1}, {"start": 0, "end": 1}], - [{"start": 0, "end": 1}], - id="duplicate-intervals", - ), - pytest.param( - [{"start": 0, "end": 1}, {"start": 0, "end": 2}], - [{"start": 0, "end": 2}], - id="overlapping-intervals", - ), - pytest.param( - [{"start": 0, "end": 3}, {"start": 1, "end": 2}], - [{"start": 0, "end": 3}], - id="enclosed-intervals", - ), - pytest.param( - [{"start": 1, "end": 2}, {"start": 0, "end": 1}], - [{"start": 0, "end": 2}], - id="unordered-intervals", - ), - pytest.param( - [{"start": 0, "end": 1}, {"start": 2, "end": 3}], - [{"start": 0, "end": 3}], - id="adjacent-intervals", - ), - pytest.param( - [{"start": 3, "end": 4}, {"start": 0, "end": 1}], - [{"start": 0, "end": 1}, {"start": 3, "end": 4}], - id="nonoverlapping-intervals", - ), - pytest.param( - [{"start": 0, "end": 1}, {"start": 2, "end": 3}, {"start": 10, "end": 11}, {"start": 1, "end": 4}], - [{"start": 0, "end": 4}, {"start": 10, "end": 11}], - id="overlapping-and-nonoverlapping-intervals", - ), - ], -) -def test_concurrent_stream_state_converter_merge_intervals(input_intervals, expected_merged_intervals): - return MockConcurrentConnectorStateConverter.merge_intervals(input_intervals) == expected_merged_intervals - - -@pytest.mark.parametrize( - "stream, sequential_state, expected_output_state", - [ - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - {}, - {"slices": [], "state_type": ConcurrencyCompatibleStateType.date_range.value, "legacy": {}}, - id="empty-input-state", - ), - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - {"created": 1617030403}, - { - "state_type": "date-range", - "slices": [{"start": 0, "end": 1617030403}], - "legacy": {"created": 1617030403}, - }, - id="with-input-state", - ), - ], -) -def test_epoch_state_converter_convert_from_sequential_state(stream, sequential_state, expected_output_state): - state_manager = EpochValueConcurrentStreamStateConverter("created") - assert state_manager.convert_from_sequential_state(sequential_state) == expected_output_state - - -@pytest.mark.parametrize( - "stream, concurrent_state, expected_output_state", - [ - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - {"state_type": ConcurrencyCompatibleStateType.date_range.value}, - {}, - id="empty-input-state", - ), - pytest.param( - AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.incremental]), - {"state_type": "date-range", "slices": [{"start": 0, "end": 1617030403}]}, - {"created": 1617030403}, - id="with-input-state", - ), - ], -) -def test_epoch_state_converter_convert_to_sequential_state(stream, concurrent_state, expected_output_state): - state_manager = EpochValueConcurrentStreamStateConverter("created") - assert state_manager.convert_to_sequential_state(concurrent_state) == expected_output_state diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py index 12caaceba2d9..102cf7cdd448 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py @@ -3,17 +3,15 @@ # from concurrent.futures import Future, ThreadPoolExecutor from unittest import TestCase -from unittest.mock import Mock, patch +from unittest.mock import Mock from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager -_SLEEP_TIME = 2 - class ThreadPoolManagerTest(TestCase): def setUp(self): self._threadpool = Mock(spec=ThreadPoolExecutor) - self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1, sleep_time=_SLEEP_TIME) + self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1) self._fn = lambda x: x self._arg = "arg" @@ -23,28 +21,38 @@ def test_submit_calls_underlying_thread_pool(self): assert len(self._thread_pool_manager._futures) == 1 - def test_submit_too_many_concurrent_tasks(self): + def test_given_no_exceptions_when_shutdown_if_exception_then_do_not_raise(self): future = Mock(spec=Future) future.exception.return_value = None - future.done.side_effect = [False, True] + future.done.side_effect = [True, True] - with patch("time.sleep") as sleep_mock: - self._thread_pool_manager._futures = [future] - self._thread_pool_manager.submit(self._fn, self._arg) - self._threadpool.submit.assert_called_with(self._fn, self._arg) - sleep_mock.assert_called_with(_SLEEP_TIME) + self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() - assert len(self._thread_pool_manager._futures) == 1 + self._thread_pool_manager.shutdown_if_exception() # do not raise - def test_submit_task_previous_task_failed(self): + def test_given_exception_when_shutdown_if_exception_then_raise(self): future = Mock(spec=Future) future.exception.return_value = RuntimeError - future.done.side_effect = [False, True] + future.done.side_effect = [True, True] self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() with self.assertRaises(RuntimeError): - self._thread_pool_manager.submit(self._fn, self._arg) + self._thread_pool_manager.shutdown_if_exception() + + def test_given_exception_during_pruning_when_check_for_errors_and_shutdown_then_shutdown_and_raise(self): + future = Mock(spec=Future) + future.exception.return_value = RuntimeError + future.done.side_effect = [True, True] + + self._thread_pool_manager._futures = [future] + self._thread_pool_manager.prune_to_validate_has_reached_futures_limit() + + with self.assertRaises(RuntimeError): + self._thread_pool_manager.check_for_errors_and_shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) def test_shutdown(self): self._thread_pool_manager.shutdown() diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index 4315f488112d..f1489c43860a 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -13,6 +13,7 @@ from airbyte_cdk.models import ( AirbyteCatalog, AirbyteConnectionStatus, + AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, @@ -27,6 +28,7 @@ ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, + FailureType, Level, Status, StreamDescriptor, @@ -40,6 +42,7 @@ from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message +from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest import fixture @@ -54,12 +57,14 @@ def __init__( per_stream: bool = True, message_repository: MessageRepository = None, exception_on_missing_stream: bool = True, + stop_sync_on_stream_failure: bool = False, ): self._streams = streams self.check_lambda = check_lambda self.per_stream = per_stream self.exception_on_missing_stream = exception_on_missing_stream self._message_repository = message_repository + self._stop_sync_on_stream_failure = stop_sync_on_stream_failure def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: if self.check_lambda: @@ -84,6 +89,12 @@ def message_repository(self): return self._message_repository +class MockSourceWithStopSyncFalseOverride(MockSource): + @property + def stop_sync_on_stream_failure(self) -> bool: + return False + + class StreamNoStateMethod(Stream): name = "managers" primary_key = None @@ -115,8 +126,11 @@ class StreamRaisesException(Stream): name = "lamentations" primary_key = None + def __init__(self, exception_to_raise): + self._exception_to_raise = exception_to_raise + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: - raise AirbyteTracedException(message="I was born only to crash like Icarus") + raise self._exception_to_raise MESSAGE_FROM_REPOSITORY = Mock() @@ -291,7 +305,7 @@ def test_read_stream_emits_repository_message_on_error(mocker, message_repositor source = MockSource(streams=[stream], message_repository=message_repository) - with pytest.raises(RuntimeError): + with pytest.raises(AirbyteTracedException): messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]))) assert MESSAGE_FROM_REPOSITORY in messages @@ -306,14 +320,14 @@ def test_read_stream_with_error_gets_display_message(mocker): catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]) # without get_error_display_message - with pytest.raises(RuntimeError, match="oh no!"): + with pytest.raises(AirbyteTracedException): list(source.read(logger, {}, catalog)) mocker.patch.object(MockStream, "get_error_display_message", return_value="my message") - with pytest.raises(AirbyteTracedException, match="oh no!") as exc: + with pytest.raises(AirbyteTracedException) as exc: list(source.read(logger, {}, catalog)) - assert exc.value.message == "my message" + assert "oh no!" in exc.value.message GLOBAL_EMITTED_AT = 1 @@ -358,6 +372,24 @@ def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_stat return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state_data)) +def _as_error_trace( + stream: str, error_message: str, internal_message: Optional[str], failure_type: Optional[FailureType], stack_trace: Optional[str] +) -> AirbyteMessage: + trace_message = AirbyteTraceMessage( + emitted_at=datetime.datetime.now().timestamp() * 1000.0, + type=TraceType.ERROR, + error=AirbyteErrorTraceMessage( + stream_descriptor=StreamDescriptor(name=stream), + message=error_message, + internal_message=internal_message, + failure_type=failure_type, + stack_trace=stack_trace, + ), + ) + + return AirbyteMessage(type=MessageType.TRACE, trace=trace_message) + + def _configured_stream(stream: Stream, sync_mode: SyncMode): return ConfiguredAirbyteStream( stream=stream.as_airbyte_stream(), @@ -1154,7 +1186,17 @@ def test_emit_non_records(self, mocker, per_stream_enabled): def test_checkpoint_state_from_stream_instance(): teams_stream = MockStreamOverridesStateMethod() managers_stream = StreamNoStateMethod() - state_manager = ConnectorStateManager({"teams": teams_stream, "managers": managers_stream}, []) + state_manager = ConnectorStateManager( + { + "teams": AirbyteStream( + name="teams", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] + ), + "managers": AirbyteStream( + name="managers", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] + ), + }, + [], + ) # The stream_state passed to checkpoint_state() should be ignored since stream implements state function teams_stream.state = {"updated_at": "2022-09-11"} @@ -1168,21 +1210,37 @@ def test_checkpoint_state_from_stream_instance(): ) -def test_continue_sync_with_failed_streams(mocker): +@pytest.mark.parametrize( + "exception_to_raise,expected_error_message,expected_internal_message", + [ + pytest.param( + AirbyteTracedException(message="I was born only to crash like Icarus"), + "I was born only to crash like Icarus", + None, + id="test_raises_traced_exception", + ), + pytest.param( + Exception("Generic connector error message"), + "Something went wrong in the connector. See the logs for more details.", + "Generic connector error message", + id="test_raises_generic_exception", + ), + ], +) +def test_continue_sync_with_failed_streams(mocker, exception_to_raise, expected_error_message, expected_internal_message): """ - Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues - syncing even when one stream fails with an error. + Tests that running a sync for a connector with multiple streams will continue syncing when one stream fails + with an error. This source does not override the default behavior defined in the AbstractSource class. """ stream_output = [{"k1": "v1"}, {"k2": "v2"}] s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") - s2 = StreamRaisesException() + s2 = StreamRaisesException(exception_to_raise=exception_to_raise) s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") mocker.patch.object(MockStream, "get_json_schema", return_value={}) mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={}) src = MockSource(streams=[s1, s2, s3]) - mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True) catalog = ConfiguredAirbyteCatalog( streams=[ _configured_stream(s1, SyncMode.full_refresh), @@ -1199,6 +1257,7 @@ def test_continue_sync_with_failed_streams(mocker): _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + _as_error_trace("lamentations", expected_error_message, expected_internal_message, FailureType.system_error, None), _as_stream_status("s3", AirbyteStreamStatus.STARTED), _as_stream_status("s3", AirbyteStreamStatus.RUNNING), *_as_records("s3", stream_output), @@ -1206,26 +1265,77 @@ def test_continue_sync_with_failed_streams(mocker): ] ) - messages = [] with pytest.raises(AirbyteTracedException) as exc: - # We can't use list comprehension or list() here because we are still raising a final exception for the - # failed streams and that disrupts parsing the generator into the messages emitted before - for message in src.read(logger, {}, catalog): - messages.append(message) + messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)] + messages = _fix_emitted_at(messages) + + assert expected == messages + + assert "lamentations" in exc.value.message + assert exc.value.failure_type == FailureType.config_error + + +def test_continue_sync_source_override_false(mocker): + """ + Tests that running a sync for a connector explicitly overriding the default AbstractSource.stop_sync_on_stream_failure + property to be False which will continue syncing stream even if one encountered an exception. + """ + update_secrets(["API_KEY_VALUE"]) + + stream_output = [{"k1": "v1"}, {"k2": "v2"}] + s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") + s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="I was born only to crash like Icarus")) + s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") + + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={}) + + src = MockSourceWithStopSyncFalseOverride(streams=[s1, s2, s3]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + _configured_stream(s1, SyncMode.full_refresh), + _configured_stream(s2, SyncMode.full_refresh), + _configured_stream(s3, SyncMode.full_refresh), + ] + ) + + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + *_as_records("s1", stream_output), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), + _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None), + _as_stream_status("s3", AirbyteStreamStatus.STARTED), + _as_stream_status("s3", AirbyteStreamStatus.RUNNING), + *_as_records("s3", stream_output), + _as_stream_status("s3", AirbyteStreamStatus.COMPLETE), + ] + ) + + with pytest.raises(AirbyteTracedException) as exc: + messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)] + messages = _fix_emitted_at(messages) + + assert expected == messages - messages = _fix_emitted_at(messages) - assert expected == messages assert "lamentations" in exc.value.message + assert exc.value.failure_type == FailureType.config_error -def test_stop_sync_with_failed_streams(mocker): +def test_sync_error_trace_messages_obfuscate_secrets(mocker): """ - Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops - syncing once a stream fails with an error. + Tests that exceptions emitted as trace messages by a source have secrets properly sanitized """ + update_secrets(["API_KEY_VALUE"]) + stream_output = [{"k1": "v1"}, {"k2": "v2"}] s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") - s2 = StreamRaisesException() + s2 = StreamRaisesException( + exception_to_raise=AirbyteTracedException(message="My api_key value API_KEY_VALUE flew too close to the sun.") + ) s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") mocker.patch.object(MockStream, "get_json_schema", return_value={}) @@ -1248,15 +1358,73 @@ def test_stop_sync_with_failed_streams(mocker): _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + _as_error_trace("lamentations", "My api_key value **** flew too close to the sun.", None, FailureType.system_error, None), + _as_stream_status("s3", AirbyteStreamStatus.STARTED), + _as_stream_status("s3", AirbyteStreamStatus.RUNNING), + *_as_records("s3", stream_output), + _as_stream_status("s3", AirbyteStreamStatus.COMPLETE), ] ) - messages = [] - with pytest.raises(AirbyteTracedException): - # We can't use list comprehension or list() here because we are still raising a final exception for the - # failed streams and that disrupts parsing the generator into the messages emitted before - for message in src.read(logger, {}, catalog): - messages.append(message) + with pytest.raises(AirbyteTracedException) as exc: + messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)] + messages = _fix_emitted_at(messages) - messages = _fix_emitted_at(messages) - assert expected == messages + assert expected == messages + + assert "lamentations" in exc.value.message + assert exc.value.failure_type == FailureType.config_error + + +def test_continue_sync_with_failed_streams_with_override_false(mocker): + """ + Tests that running a sync for a connector with multiple streams and stop_sync_on_stream_failure enabled stops + the sync when one stream fails with an error. + """ + stream_output = [{"k1": "v1"}, {"k2": "v2"}] + s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") + s2 = StreamRaisesException(AirbyteTracedException(message="I was born only to crash like Icarus")) + s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") + + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={}) + + src = MockSource(streams=[s1, s2, s3]) + mocker.patch.object(MockSource, "stop_sync_on_stream_failure", return_value=True) + catalog = ConfiguredAirbyteCatalog( + streams=[ + _configured_stream(s1, SyncMode.full_refresh), + _configured_stream(s2, SyncMode.full_refresh), + _configured_stream(s3, SyncMode.full_refresh), + ] + ) + + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + *_as_records("s1", stream_output), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), + _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None), + ] + ) + + with pytest.raises(AirbyteTracedException) as exc: + messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)] + messages = _fix_emitted_at(messages) + + assert expected == messages + + assert "lamentations" in exc.value.message + assert exc.value.failure_type == FailureType.config_error + + +def _remove_stack_trace(message: AirbyteMessage) -> AirbyteMessage: + """ + Helper method that removes the stack trace from Airbyte trace messages to make asserting against expected records easier + """ + if message.trace and message.trace.error and message.trace.error.stack_trace: + message.trace.error.stack_trace = None + return message diff --git a/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py index ca5c669a27c6..ebd082a2b152 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability, StreamAvailable, StreamUnavailable +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from airbyte_protocol.models import AirbyteStream @@ -72,6 +73,10 @@ def as_airbyte_stream(self) -> AirbyteStream: def log_stream_sync_configuration(self) -> None: raise NotImplementedError + @property + def cursor(self) -> Cursor: + return NoopCursor() + class _MockPartition(Partition): def __init__(self, name: str): diff --git a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py index f0c80a0cd0d3..9bee58eb69ee 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py +++ b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py @@ -3,21 +3,21 @@ # from contextlib import nullcontext as does_not_raise -from typing import Any, Iterable, List, Mapping +from typing import List import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStream, + AirbyteStreamState, + StreamDescriptor, + SyncMode, +) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor -from airbyte_cdk.sources.streams import Stream - - -class StreamWithNamespace(Stream): - primary_key = None - namespace = "public" - - def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: - return {} @pytest.mark.parametrize( @@ -156,7 +156,14 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: ), ) def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error): - stream_to_instance_map = {"actors": StreamWithNamespace()} + stream_to_instance_map = { + "actors": AirbyteStream( + name="actors", + namespace="public", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + ) + } if isinstance(input_stream_state, List): input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)] @@ -264,7 +271,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp ], ) def test_get_stream_state(input_state, stream_name, namespace, expected_state): - stream_to_instance_map = {"users": StreamWithNamespace()} + stream_to_instance_map = { + stream_name: AirbyteStream( + name=stream_name, namespace=namespace, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] + ) + } state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)] state_manager = ConnectorStateManager(stream_to_instance_map, state_messages) diff --git a/airbyte-cdk/python/unit_tests/sources/test_integration_source.py b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py index 64b322eb53c3..17628a0263cd 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_integration_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py @@ -2,13 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import json import os +from typing import Any, List, Mapping from unittest import mock from unittest.mock import patch import pytest import requests from airbyte_cdk.entrypoint import launch +from airbyte_cdk.utils import AirbyteTracedException from unit_tests.sources.fixtures.source_test_fixture import ( HttpTestStream, SourceFixtureOauthAuthenticator, @@ -21,9 +24,9 @@ "deployment_mode, url_base, expected_records, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), - pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), - pytest.param("CLOUD", "https://localhost:80/api/v1/", [], ValueError, id="test_cloud_read_with_localhost"), + pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/", [], "config_error", id="test_cloud_read_with_localhost"), pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), ], @@ -36,8 +39,10 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec with mock.patch.object(HttpTestStream, "url_base", url_base): args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"] if expected_error: - with pytest.raises(expected_error): + with pytest.raises(AirbyteTracedException): launch(source, args) + messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert contains_error_trace_message(messages, expected_error) else: launch(source, args) @@ -46,14 +51,14 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec "deployment_mode, token_refresh_url, expected_records, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), - pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"), pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), ], ) @patch.object(requests.Session, "send", fixture_mock_send) -def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error): +def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_url, expected_records, expected_error): oauth_authenticator = SourceFixtureOauthAuthenticator( client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url ) @@ -62,7 +67,20 @@ def test_external_oauth_request_source(deployment_mode, token_refresh_url, expec with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"] if expected_error: - with pytest.raises(expected_error): + with pytest.raises(AirbyteTracedException): launch(source, args) + messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert contains_error_trace_message(messages, expected_error) else: launch(source, args) + + +def contains_error_trace_message(messages: List[Mapping[str, Any]], expected_error: str) -> bool: + for message in messages: + if message.get("type") != "TRACE": + continue + elif message.get("trace").get("type") != "ERROR": + continue + elif message.get("trace").get("error").get("failure_type") == expected_error: + return True + return False diff --git a/airbyte-cdk/python/unit_tests/sources/test_source_read.py b/airbyte-cdk/python/unit_tests/sources/test_source_read.py index 752c4640d3ed..dd08c4d18dac 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source_read.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source_read.py @@ -343,7 +343,7 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger) config = {} catalog = _create_configured_catalog(source._streams) - messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError) + messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, AirbyteTracedException) messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError) expected_messages = [ diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py index 5987dfe69544..61a9ecfec2f9 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py @@ -22,24 +22,32 @@ def test_given_request_matches_when_matches_then_has_expected_match_count(self): def test_given_request_does_not_match_when_matches_then_does_not_have_expected_match_count(self): self._a_request.matches.return_value = False self._matcher.matches(self._a_request) + assert not self._matcher.has_expected_match_count() + assert self._matcher.actual_number_of_matches == 0 def test_given_many_requests_with_some_match_when_matches_then_has_expected_match_count(self): - self._a_request.matches.side_effect = [True, False] + self._a_request.matches.return_value = True + self._another_request.matches.return_value = False self._matcher.matches(self._a_request) self._matcher.matches(self._another_request) + assert self._matcher.has_expected_match_count() + assert self._matcher.actual_number_of_matches == 1 - def test_given_many_matches_the_expected_number_of_requests_when_matches_then_has_expected_match_count(self): + def test_given_expected_number_of_requests_met_when_matches_then_has_expected_match_count(self): _matcher = HttpRequestMatcher(self._request_to_match, 2) self._a_request.matches.return_value = True _matcher.matches(self._a_request) _matcher.matches(self._a_request) + assert _matcher.has_expected_match_count() + assert _matcher.actual_number_of_matches == 2 - def test_given_many_matches_the_expected_number_of_requests_when_matches_then_does_not_have_expected_match_count(self): + def test_given_expected_number_of_requests_not_met_when_matches_then_does_not_have_expected_match_count(self): _matcher = HttpRequestMatcher(self._request_to_match, 2) self._a_request.matches.side_effect = [True, False] _matcher.matches(self._a_request) _matcher.matches(self._a_request) + assert not _matcher.has_expected_match_count() diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py index 06a50c75106d..e9e583e564d4 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py @@ -10,51 +10,91 @@ # see https://github.com/psf/requests/blob/0b4d494192de489701d3a2e32acef8fb5d3f042e/src/requests/models.py#L424-L429 _A_URL = "http://test.com/" _ANOTHER_URL = "http://another-test.com/" -_A_BODY = "a body" -_ANOTHER_BODY = "another body" +_A_RESPONSE_BODY = "a body" +_ANOTHER_RESPONSE_BODY = "another body" _A_RESPONSE = HttpResponse("any response") _SOME_QUERY_PARAMS = {"q1": "query value"} _SOME_HEADERS = {"h1": "header value"} +_OTHER_HEADERS = {"h2": "another header value"} +_SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2} +_SOME_REQUEST_BODY_STR = "some_request_body" class HttpMockerTest(TestCase): @HttpMocker() - def test_given_request_match_when_decorate_then_return_response(self, http_mocker): + def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker): http_mocker.get( HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), - HttpResponse(_A_BODY, 474), + HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS), ) response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) - assert response.text == _A_BODY + assert response.text == _A_RESPONSE_BODY assert response.status_code == 474 + assert response.headers == _OTHER_HEADERS @HttpMocker() - def test_given_multiple_responses_when_decorate_then_return_response(self, http_mocker): + def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker): http_mocker.get( HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), - [HttpResponse(_A_BODY, 1), HttpResponse(_ANOTHER_BODY, 2)], + HttpResponse(_A_RESPONSE_BODY, 474), + ) + + requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS | {"more strict query param key": "any value"}) + + @HttpMocker() + def test_given_post_request_match_when_decorate_then_return_response(self, http_mocker): + http_mocker.post( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR), + HttpResponse(_A_RESPONSE_BODY, 474), + ) + + response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + + assert response.text == _A_RESPONSE_BODY + assert response.status_code == 474 + + @HttpMocker() + def test_given_multiple_responses_when_decorate_get_request_then_return_response(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], ) first_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) second_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) - assert first_response.text == _A_BODY + assert first_response.text == _A_RESPONSE_BODY + assert first_response.status_code == 1 + assert second_response.text == _ANOTHER_RESPONSE_BODY + assert second_response.status_code == 2 + + @HttpMocker() + def test_given_multiple_responses_when_decorate_post_request_then_return_response(self, http_mocker): + http_mocker.post( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR), + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], + ) + + first_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + second_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + + assert first_response.text == _A_RESPONSE_BODY assert first_response.status_code == 1 - assert second_response.text == _ANOTHER_BODY + assert second_response.text == _ANOTHER_RESPONSE_BODY assert second_response.status_code == 2 @HttpMocker() def test_given_more_requests_than_responses_when_decorate_then_raise_error(self, http_mocker): http_mocker.get( HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), - [HttpResponse(_A_BODY, 1), HttpResponse(_ANOTHER_BODY, 2)], + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], ) last_response = [requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) for _ in range(10)][-1] - assert last_response.text == _ANOTHER_BODY + assert last_response.text == _ANOTHER_RESPONSE_BODY assert last_response.status_code == 2 @HttpMocker() @@ -135,3 +175,40 @@ def decorated_function(http_mocker): with pytest.raises(ValueError) as exc_info: decorated_function() assert "more_granular" in str(exc_info.value) # the matcher corresponding to the first `http_mocker.get` is not matched + + def test_given_exact_number_of_call_provided_when_assert_number_of_calls_then_do_not_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + request = HttpRequest(_A_URL) + http_mocker.get(request, _A_RESPONSE) + + requests.get(_A_URL) + requests.get(_A_URL) + + http_mocker.assert_number_of_calls(request, 2) + + decorated_function() + # then do not raise + + def test_given_invalid_number_of_call_provided_when_assert_number_of_calls_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + request = HttpRequest(_A_URL) + http_mocker.get(request, _A_RESPONSE) + + requests.get(_A_URL) + requests.get(_A_URL) + + http_mocker.assert_number_of_calls(request, 1) + + with pytest.raises(AssertionError): + decorated_function() + + def test_given_unknown_request_when_assert_number_of_calls_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get(HttpRequest(_A_URL), _A_RESPONSE) + http_mocker.assert_number_of_calls(HttpRequest(_ANOTHER_URL), 1) + + with pytest.raises(ValueError): + decorated_function() diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py index e724894b4002..a5a94ea05580 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py @@ -18,40 +18,100 @@ def test_given_query_params_in_url_and_also_provided_then_raise_error(self): def test_given_same_url_query_params_and_subset_headers_when_matches_then_return_true(self): request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1"}) - request_received = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1", "second_header": "h2"}) - assert request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1", "second_header": "h2"}) + assert actual_request.matches(request_to_match) def test_given_url_differs_when_matches_then_return_false(self): assert not HttpRequest("mock://test.com/another_path").matches(HttpRequest("mock://test.com/path")) def test_given_query_params_differs_when_matches_then_return_false(self): request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) - request_received = HttpRequest("mock://test.com/path", {"another_query_param": "q2"}) - assert not request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", {"another_query_param": "q2"}) + assert not actual_request.matches(request_to_match) def test_given_query_params_is_subset_differs_when_matches_then_return_false(self): request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) - request_received = HttpRequest("mock://test.com/path", {"a_query_param": "q1", "another_query_param": "q2"}) - assert not request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1", "another_query_param": "q2"}) + assert not actual_request.matches(request_to_match) def test_given_headers_is_subset_differs_when_matches_then_return_true(self): request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"}) - request_received = HttpRequest("mock://test.com/path", headers={"first_header": "h1", "second_header": "h2"}) - assert request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "h1", "second_header": "h2"}) + assert actual_request.matches(request_to_match) def test_given_headers_value_does_not_match_differs_when_matches_then_return_false(self): request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"}) - request_received = HttpRequest("mock://test.com/path", headers={"first_header": "value does not match"}) - assert not request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "value does not match"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_body_mappings_value_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2}) + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2}) + assert actual_request.matches(request_to_match) + + def test_given_bodies_are_mapping_and_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "value does not match"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_mapping_and_bytes_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + assert actual_request.matches(request_to_match) + + def test_given_different_mapping_and_bytes_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "another value"}') + assert not actual_request.matches(request_to_match) + + def test_given_same_mapping_and_str_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + assert actual_request.matches(request_to_match) + + def test_given_different_mapping_and_str_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "another value"}') + assert not actual_request.matches(request_to_match) + + def test_given_same_bytes_and_mapping_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + assert actual_request.matches(request_to_match) + + def test_given_different_bytes_and_mapping_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_str_and_mapping_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + assert actual_request.matches(request_to_match) + + def test_given_different_str_and_mapping_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_body_str_value_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body="some_request_body") + actual_request = HttpRequest("mock://test.com/path", body="some_request_body") + assert actual_request.matches(request_to_match) + + def test_given_body_str_value_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body="some_request_body") + actual_request = HttpRequest("mock://test.com/path", body="another_request_body") + assert not actual_request.matches(request_to_match) def test_given_any_matcher_for_query_param_when_matches_then_return_true(self): request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) - request_received = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) + actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) - assert request_received.matches(request_to_match) - assert request_to_match.matches(request_received) + assert actual_request.matches(request_to_match) + assert request_to_match.matches(actual_request) def test_given_any_matcher_for_both_when_matches_then_return_true(self): request_to_match = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) - request_received = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) - assert request_received.matches(request_to_match) + actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) + assert actual_request.matches(request_to_match) diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py index b06d1423f1bd..328db535ca36 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py @@ -16,7 +16,8 @@ PaginationStrategy, Path, RecordBuilder, - create_builders_from_resource, + create_record_builder, + create_response_builder, find_template, ) @@ -37,7 +38,11 @@ def _record_builder( record_id_path: Optional[Path] = None, record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None, ) -> RecordBuilder: - return create_builders_from_resource(deepcopy(response_template), records_path, record_id_path, record_cursor_path)[0] + return create_record_builder(deepcopy(response_template), records_path, record_id_path, record_cursor_path) + + +def _any_record_builder() -> RecordBuilder: + return create_record_builder({"record_path": [{"a_record": "record value"}]}, FieldPath("record_path")) def _response_builder( @@ -45,7 +50,7 @@ def _response_builder( records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy] = None ) -> HttpResponseBuilder: - return create_builders_from_resource(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy)[1] + return create_response_builder(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy) def _body(response: HttpResponse) -> Dict[str, Any]: @@ -90,6 +95,16 @@ def test_given_nested_cursor_when_build_then_set_cursor(self) -> None: record = builder.with_cursor("another cursor").build() assert record["nested"][_CURSOR_FIELD] == "another cursor" + def test_given_with_field_when_build_then_write_field(self) -> None: + builder = _any_record_builder() + record = builder.with_field(FieldPath("to_write_field"), "a field value").build() + assert record["to_write_field"] == "a field value" + + def test_given_nested_cursor_when_build_then_write_field(self) -> None: + builder = _any_record_builder() + record = builder.with_field(NestedPath(["path", "to_write_field"]), "a field value").build() + assert record["path"]["to_write_field"] == "a field value" + def test_given_cursor_path_not_provided_but_with_id_when_build_then_raise_error(self) -> None: builder = _record_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) with pytest.raises(ValueError): @@ -159,4 +174,4 @@ def test_given_cwd_doesnt_have_unit_tests_as_parent_when_from_resource_file__the def test_given_records_path_invalid_when_create_builders_from_resource_then_raise_exception(self) -> None: with pytest.raises(ValueError): - create_builders_from_resource(_A_RESPONSE_TEMPLATE, NestedPath(["invalid", "record", "path"])) + create_record_builder(_A_RESPONSE_TEMPLATE, NestedPath(["invalid", "record", "path"])) diff --git a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py index b62dd5478f95..d0564cdf93f6 100644 --- a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py +++ b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py @@ -3,12 +3,13 @@ import json import logging import os -from typing import Iterator, List +from typing import Any, Iterator, List from unittest import TestCase from unittest.mock import Mock, patch from airbyte_cdk.sources.abstract_source import AbstractSource from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import ( AirbyteAnalyticsTraceMessage, AirbyteErrorTraceMessage, @@ -26,18 +27,30 @@ Type, ) + +def _a_state_message(state: Any) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state)) + + +def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteMessage: + return AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=stream_name), + status=status, + ), + ), + ) + + _A_RECORD = AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream="stream", data={"record key": "record value"}, emitted_at=0) -) -_A_STATE_MESSAGE = AirbyteMessage( - type=Type.STATE, - state=AirbyteStateMessage(data={"state key": "state value"},) -) -_A_LOG = AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage(level=Level.INFO, message="This is an Airbyte log message") + type=Type.RECORD, record=AirbyteRecordMessage(stream="stream", data={"record key": "record value"}, emitted_at=0) ) +_A_STATE_MESSAGE = _a_state_message({"state key": "state value for _A_STATE_MESSAGE"}) +_A_LOG = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is an Airbyte log message")) _AN_ERROR_MESSAGE = AirbyteMessage( type=Type.TRACE, trace=AirbyteTraceMessage( @@ -72,24 +85,10 @@ ] } ) -_A_STATE = {"state_key": "state_value"} +_A_STATE = StateBuilder().with_stream_state(_A_STREAM_NAME, {"state_key": "state_value"}).build() _A_LOG_MESSAGE = "a log message" -def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteMessage: - return AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.STREAM_STATUS, - emitted_at=0, - stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name=stream_name), - status=status, - ), - ), - ) - - def _to_entrypoint_output(messages: List[AirbyteMessage]) -> Iterator[str]: return (message.json(exclude_unset=True) for message in messages) @@ -115,6 +114,7 @@ def _validate_tmp_files(self): _validate_tmp_catalog(expected_catalog, entrypoint.return_value.parse_args.call_args.args[0][4]) _validate_tmp_json_file(expected_state, entrypoint.return_value.parse_args.call_args.args[0][6]) return entrypoint.return_value.run.return_value + return _validate_tmp_files @@ -148,6 +148,7 @@ def test_given_logging_during_run_when_read_then_output_has_logs(self, entrypoin def _do_some_logging(self): logging.getLogger("any logger").info(_A_LOG_MESSAGE) return entrypoint.return_value.run.return_value + entrypoint.return_value.run.side_effect = _do_some_logging output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) @@ -173,6 +174,15 @@ def test_given_state_message_and_records_when_read_then_output_has_records_and_s output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) assert output.records_and_state_messages == [_A_RECORD, _A_STATE_MESSAGE] + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_many_state_messages_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): + last_emitted_state = {"last state key": "last state value"} + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE, _a_state_message(last_emitted_state)]) + + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + + assert output.most_recent_state == last_emitted_state + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_log_when_read_then_output_has_log(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) @@ -189,7 +199,7 @@ def test_given_trace_message_when_read_then_output_has_trace_messages(self, entr def test_given_stream_statuses_when_read_then_return_statuses(self, entrypoint): status_messages = [ _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.STARTED), - _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE) + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE), ] entrypoint.return_value.run.return_value = _to_entrypoint_output(status_messages) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) @@ -200,20 +210,20 @@ def test_given_stream_statuses_for_many_streams_when_read_then_filter_other_stre status_messages = [ _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.STARTED), _a_status_message("another stream name", AirbyteStreamStatus.INCOMPLETE), - _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE) + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE), ] entrypoint.return_value.run.return_value = _to_entrypoint_output(status_messages) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) assert len(output.get_stream_statuses(_A_STREAM_NAME)) == 2 - @patch('airbyte_cdk.test.entrypoint_wrapper.print', create=True) + @patch("airbyte_cdk.test.entrypoint_wrapper.print", create=True) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_unexpected_exception_when_read_then_print(self, entrypoint, print_mock): entrypoint.return_value.run.side_effect = ValueError("This error should be printed") read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) assert print_mock.call_count > 0 - @patch('airbyte_cdk.test.entrypoint_wrapper.print', create=True) + @patch("airbyte_cdk.test.entrypoint_wrapper.print", create=True) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_expected_exception_when_read_then_do_not_print(self, entrypoint, print_mock): entrypoint.return_value.run.side_effect = ValueError("This error should be printed") diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index 7acceff69a19..7451a320d404 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -28,6 +28,7 @@ Type, ) from airbyte_cdk.sources import Source +from airbyte_cdk.utils import AirbyteTracedException class MockSource(Source): @@ -246,7 +247,9 @@ def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock) assert spec_mock.called -def test_given_message_emitted_during_config_when_read_then_emit_message_before_next_steps(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): +def test_given_message_emitted_during_config_when_read_then_emit_message_before_next_steps( + entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock +): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") mocker.patch.object(MockSource, "read_catalog", side_effect=ValueError) @@ -276,12 +279,12 @@ def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): "deployment_mode, url, expected_error", [ pytest.param("CLOUD", "https://airbyte.com", None, id="test_cloud_public_endpoint_is_successful"), - pytest.param("CLOUD", "https://192.168.27.30", ValueError, id="test_cloud_private_ip_address_is_rejected"), - pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", ValueError, id="test_cloud_private_endpoint_is_rejected"), + pytest.param("CLOUD", "https://192.168.27.30", AirbyteTracedException, id="test_cloud_private_ip_address_is_rejected"), + pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", AirbyteTracedException, id="test_cloud_private_endpoint_is_rejected"), pytest.param("CLOUD", "http://past.lives.net/api/v1/inyun", ValueError, id="test_cloud_unsecured_endpoint_is_rejected"), pytest.param("CLOUD", "https://not:very/cash:443.money", ValueError, id="test_cloud_invalid_url_format"), pytest.param("CLOUD", "https://192.168.27.30 ", ValueError, id="test_cloud_incorrect_ip_format_is_rejected"), - pytest.param("cloud", "https://192.168.27.30", ValueError, id="test_case_insensitive_cloud_environment_variable"), + pytest.param("cloud", "https://192.168.27.30", AirbyteTracedException, id="test_case_insensitive_cloud_environment_variable"), pytest.param("OSS", "https://airbyte.com", None, id="test_oss_public_endpoint_is_successful"), pytest.param("OSS", "https://192.168.27.30", None, id="test_oss_private_endpoint_is_successful"), pytest.param("OSS", "https://localhost:8080/api/v1/cast", None, id="test_oss_private_endpoint_is_successful"), diff --git a/airbyte-cdk/python/unit_tests/utils/test_secret_utils.py b/airbyte-cdk/python/unit_tests/utils/test_secret_utils.py index 08cc81778da0..39c6ff735da0 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_secret_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_secret_utils.py @@ -3,7 +3,7 @@ # import pytest -from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets, get_secret_paths, get_secrets, update_secrets +from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets, filter_secrets, get_secret_paths, get_secrets, update_secrets SECRET_STRING_KEY = "secret_key1" SECRET_STRING_VALUE = "secret_value" @@ -121,3 +121,15 @@ def test_secret_filtering(): update_secrets([SECRET_STRING_VALUE, SECRET_STRING_2_VALUE]) filtered = filter_secrets(sensitive_str) assert filtered == f"**** {NOT_SECRET_VALUE} **** ****" + + +def test_secrets_added_are_filtered(): + ADDED_SECRET = "only_a_secret_if_added" + sensitive_str = f"{ADDED_SECRET} {NOT_SECRET_VALUE}" + + filtered = filter_secrets(sensitive_str) + assert filtered == sensitive_str + + add_to_secrets(ADDED_SECRET) + filtered = filter_secrets(sensitive_str) + assert filtered == f"**** {NOT_SECRET_VALUE}" diff --git a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml b/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml deleted file mode 100644 index 7957973df4e7..000000000000 --- a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml +++ /dev/null @@ -1,11 +0,0 @@ -paths: - "airbyte-integrations/connectors/**": - - PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention) - - "[Breaking changes are considered](https://docs.airbyte.com/contributing-to-airbyte/change-cdk-connector/#breaking-changes-to-connectors). If a **Breaking Change** is being introduced, ensure an Airbyte engineer has created a Breaking Change Plan." - - Connector version has been incremented in the Dockerfile and metadata.yaml according to our [Semantic Versioning for Connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors) guidelines - - You've updated the connector's `metadata.yaml` file any other relevant changes, including a `breakingChanges` entry for major version bumps. See [metadata.yaml docs](https://docs.airbyte.com/connector-development/connector-metadata-file/) - - Secrets in the connector's spec are annotated with `airbyte_secret` - - All documentation files are up to date. (README.md, bootstrap.md, docs.md, etc...) - - Changelog updated in `docs/integrations//.md` with an entry for the new version. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog) - - Migration guide updated in `docs/integrations//-migrations.md` with an entry for the new version, if the version is a breaking change. See migration guide [example](https://docs.airbyte.io/integrations/sources/faker-migrations) - - If set, you've ensured the icon is present in the `platform-internal` repo. ([Docs](https://docs.airbyte.com/connector-development/connector-metadata-file/#the-icon-field)) diff --git a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py index 86b24786ae17..5411c8c269dc 100644 --- a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py @@ -76,11 +76,7 @@ async def check_python_image_has_expected_env_vars(python_image_container: dagge "PATH", "LANG", "GPG_KEY", - "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "PYTHON_SETUPTOOLS_VERSION", - "OTEL_TRACES_EXPORTER", - "OTEL_TRACE_PARENT", - "TRACEPARENT", } # It's not suboptimal to call printenv multiple times because the printenv output is cached. for expected_env_var in expected_env_vars: diff --git a/airbyte-ci/connectors/base_images/poetry.lock b/airbyte-ci/connectors/base_images/poetry.lock index 79faad2f5303..44a8b475dca2 100644 --- a/airbyte-ci/connectors/base_images/poetry.lock +++ b/airbyte-ci/connectors/base_images/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "ansicon" @@ -13,24 +13,25 @@ files = [ [[package]] name = "anyio" -version = "4.0.0" +version = "4.2.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" files = [ - {file = "anyio-4.0.0-py3-none-any.whl", hash = "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f"}, - {file = "anyio-4.0.0.tar.gz", hash = "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"}, + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] -doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (>=0.22)"] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] [[package]] name = "atomicwrites" @@ -44,21 +45,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -73,13 +75,13 @@ files = [ [[package]] name = "beartype" -version = "0.16.3" +version = "0.16.4" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.16.3-py3-none-any.whl", hash = "sha256:dc7b3fd28d4998771b4ff8eb41eccb70aa665a8dd505b8db43ba03c191450dd6"}, - {file = "beartype-0.16.3.tar.gz", hash = "sha256:085591b5b77807229b65a137fd473c6891c45287fe0ca6565b3250dead00380b"}, + {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, + {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, ] [package.extras] @@ -107,49 +109,49 @@ wcwidth = ">=0.1.4" [[package]] name = "cachetools" -version = "5.3.1" +version = "5.3.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, - {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, ] [[package]] name = "cattrs" -version = "23.1.2" +version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, - {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, ] [package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] -bson = ["pymongo (>=4.2.0,<5.0.0)"] -cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] -msgpack = ["msgpack (>=1.0.2,<2.0.0)"] -orjson = ["orjson (>=3.5.2,<4.0.0)"] -pyyaml = ["PyYAML (>=6.0,<7.0)"] -tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] -ujson = ["ujson (>=5.4.0,<6.0.0)"] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] @@ -218,101 +220,101 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "3.3.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"}, - {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -379,7 +381,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.3.1" +version = "0.3.3" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" @@ -406,63 +408,63 @@ url = "../connector_ops" [[package]] name = "coverage" -version = "7.3.2" +version = "7.4.0" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf"}, - {file = "coverage-7.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9"}, - {file = "coverage-7.3.2-cp310-cp310-win32.whl", hash = "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f"}, - {file = "coverage-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312"}, - {file = "coverage-7.3.2-cp311-cp311-win32.whl", hash = "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640"}, - {file = "coverage-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a"}, - {file = "coverage-7.3.2-cp312-cp312-win32.whl", hash = "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb"}, - {file = "coverage-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76"}, - {file = "coverage-7.3.2-cp38-cp38-win32.whl", hash = "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92"}, - {file = "coverage-7.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083"}, - {file = "coverage-7.3.2-cp39-cp39-win32.whl", hash = "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce"}, - {file = "coverage-7.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f"}, - {file = "coverage-7.3.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637"}, - {file = "coverage-7.3.2.tar.gz", hash = "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] [package.dependencies] @@ -512,13 +514,13 @@ test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)" [[package]] name = "dagger-io" -version = "0.6.4" +version = "0.9.6" description = "A client package for running Dagger pipelines in Python." optional = false python-versions = ">=3.10" files = [ - {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, - {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, ] [package.dependencies] @@ -529,11 +531,8 @@ gql = ">=3.4.0" graphql-core = ">=3.2.3" httpx = ">=0.23.1" platformdirs = ">=2.6.2" -typing-extensions = ">=4.4.0" - -[package.extras] -cli = ["typer[all] (>=0.6.1)"] -server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" [[package]] name = "deprecated" @@ -552,15 +551,30 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] +[[package]] +name = "editor" +version = "1.6.5" +description = "🖋 Open the default text editor 🖋" +optional = false +python-versions = ">=3.8" +files = [ + {file = "editor-1.6.5-py3-none-any.whl", hash = "sha256:53c26dd78333b50b8cdcf67748956afa75fabcb5bb25e96a00515504f58e49a8"}, + {file = "editor-1.6.5.tar.gz", hash = "sha256:5a8ad611d2a05de34994df3781605e26e63492f82f04c2e93abdd330eed6fa8d"}, +] + +[package.dependencies] +runs = "*" +xmod = "*" + [[package]] name = "exceptiongroup" -version = "1.1.3" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -568,13 +582,13 @@ test = ["pytest (>=6)"] [[package]] name = "gitdb" -version = "4.0.10" +version = "4.0.11" description = "Git Object Database" optional = false python-versions = ">=3.7" files = [ - {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, - {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, ] [package.dependencies] @@ -582,30 +596,30 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.38" +version = "3.1.41" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.38-py3-none-any.whl", hash = "sha256:9e98b672ffcb081c2c8d5aa630d4251544fb040fb158863054242f24a2a2ba30"}, - {file = "GitPython-3.1.38.tar.gz", hash = "sha256:4d683e8957c8998b58ddb937e3e6cd167215a180e1ffd4da769ab81c620a89fe"}, + {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, + {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-instafail", "pytest-subtests", "pytest-sugar"] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] [[package]] name = "google-api-core" -version = "2.12.0" +version = "2.15.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.12.0.tar.gz", hash = "sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553"}, - {file = "google_api_core-2.12.0-py3-none-any.whl", hash = "sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160"}, + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, ] [package.dependencies] @@ -621,13 +635,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.23.3" +version = "2.26.2" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.23.3.tar.gz", hash = "sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3"}, - {file = "google_auth-2.23.3-py2.py3-none-any.whl", hash = "sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda"}, + {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, + {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, ] [package.dependencies] @@ -644,13 +658,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-core" -version = "2.3.3" +version = "2.4.1" description = "Google Cloud API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, - {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, ] [package.dependencies] @@ -658,17 +672,17 @@ google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)"] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.12.0" +version = "2.14.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.12.0.tar.gz", hash = "sha256:57c0bcda2f5e11f008a155d8636d8381d5abab46b58e0cae0e46dd5e595e6b46"}, - {file = "google_cloud_storage-2.12.0-py2.py3-none-any.whl", hash = "sha256:bc52563439d42981b6e21b071a76da2791672776eda3ba99d13a8061ebbd6e5e"}, + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, ] [package.dependencies] @@ -764,13 +778,13 @@ testing = ["pytest"] [[package]] name = "google-resumable-media" -version = "2.6.0" +version = "2.7.0" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">= 3.7" files = [ - {file = "google-resumable-media-2.6.0.tar.gz", hash = "sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7"}, - {file = "google_resumable_media-2.6.0-py2.py3-none-any.whl", hash = "sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b"}, + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, ] [package.dependencies] @@ -782,13 +796,13 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.61.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.61.0.tar.gz", hash = "sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b"}, - {file = "googleapis_common_protos-1.61.0-py2.py3-none-any.whl", hash = "sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -799,29 +813,31 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" @@ -847,39 +863,40 @@ files = [ [[package]] name = "httpcore" -version = "0.18.0" +version = "1.0.2" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-0.18.0-py3-none-any.whl", hash = "sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced"}, - {file = "httpcore-0.18.0.tar.gz", hash = "sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9"}, + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, ] [package.dependencies] -anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" [package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.25.0" +version = "0.26.0" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.25.0-py3-none-any.whl", hash = "sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100"}, - {file = "httpx-0.25.0.tar.gz", hash = "sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [package.dependencies] +anyio = "*" certifi = "*" -httpcore = ">=0.18.0,<0.19.0" +httpcore = "==1.*" idna = "*" sniffio = "*" @@ -891,13 +908,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -913,29 +930,29 @@ files = [ [[package]] name = "inquirer" -version = "3.1.3" +version = "3.2.1" description = "Collection of common interactive command line user interfaces, based on Inquirer.js" optional = false -python-versions = ">=3.8" +python-versions = ">=3.8.1" files = [ - {file = "inquirer-3.1.3-py3-none-any.whl", hash = "sha256:a7441fd74d06fcac4385218a1f5e8703f7a113f7944e01af47b8c58e84f95ce5"}, - {file = "inquirer-3.1.3.tar.gz", hash = "sha256:aac309406f5b49d4b8ab7c6872117f43bf082a552dc256aa16bc95e16bb58bec"}, + {file = "inquirer-3.2.1-py3-none-any.whl", hash = "sha256:e1a0a001b499633ca69d2ea64da712b449939e8fad8fa47caebc92b0ee212df4"}, + {file = "inquirer-3.2.1.tar.gz", hash = "sha256:d5ff9bb8cd07bd3f076eabad8ae338280886e93998ff10461975b768e3854fbc"}, ] [package.dependencies] blessed = ">=1.19.0" -python-editor = ">=1.0.4" +editor = ">=1.6.0" readchar = ">=3.0.6" [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -946,13 +963,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jinxed" -version = "1.2.0" +version = "1.2.1" description = "Jinxed Terminal Library" optional = false python-versions = "*" files = [ - {file = "jinxed-1.2.0-py2.py3-none-any.whl", hash = "sha256:cfc2b2e4e3b4326954d546ba6d6b9a7a796ddcb0aef8d03161d005177eb0d48b"}, - {file = "jinxed-1.2.0.tar.gz", hash = "sha256:032acda92d5c57cd216033cbbd53de731e6ed50deb63eb4781336ca55f72cda5"}, + {file = "jinxed-1.2.1-py2.py3-none-any.whl", hash = "sha256:37422659c4925969c66148c5e64979f553386a4226b9484d910d3094ced37d30"}, + {file = "jinxed-1.2.1.tar.gz", hash = "sha256:30c3f861b73279fea1ed928cfd4dfb1f273e16cd62c8a32acfac362da0f78f3f"}, ] [package.dependencies] @@ -1009,6 +1026,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1137,38 +1164,38 @@ files = [ [[package]] name = "mypy" -version = "1.6.1" +version = "1.8.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5012e5cc2ac628177eaac0e83d622b2dd499e28253d4107a08ecc59ede3fc2c"}, - {file = "mypy-1.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8fbb68711905f8912e5af474ca8b78d077447d8f3918997fecbf26943ff3cbb"}, - {file = "mypy-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a1ad938fee7d2d96ca666c77b7c494c3c5bd88dff792220e1afbebb2925b5e"}, - {file = "mypy-1.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b96ae2c1279d1065413965c607712006205a9ac541895004a1e0d4f281f2ff9f"}, - {file = "mypy-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:40b1844d2e8b232ed92e50a4bd11c48d2daa351f9deee6c194b83bf03e418b0c"}, - {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"}, - {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"}, - {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"}, - {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"}, - {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"}, - {file = "mypy-1.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82e469518d3e9a321912955cc702d418773a2fd1e91c651280a1bda10622f02f"}, - {file = "mypy-1.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4473c22cc296425bbbce7e9429588e76e05bc7342da359d6520b6427bf76660"}, - {file = "mypy-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a0d7d24dfb26729e0a068639a6ce3500e31d6655df8557156c51c1cb874ce7"}, - {file = "mypy-1.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfd13d47b29ed3bbaafaff7d8b21e90d827631afda134836962011acb5904b71"}, - {file = "mypy-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:eb4f18589d196a4cbe5290b435d135dee96567e07c2b2d43b5c4621b6501531a"}, - {file = "mypy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41697773aa0bf53ff917aa077e2cde7aa50254f28750f9b88884acea38a16169"}, - {file = "mypy-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7274b0c57737bd3476d2229c6389b2ec9eefeb090bbaf77777e9d6b1b5a9d143"}, - {file = "mypy-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbaf4662e498c8c2e352da5f5bca5ab29d378895fa2d980630656178bd607c46"}, - {file = "mypy-1.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bb8ccb4724f7d8601938571bf3f24da0da791fe2db7be3d9e79849cb64e0ae85"}, - {file = "mypy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:68351911e85145f582b5aa6cd9ad666c8958bcae897a1bfda8f4940472463c45"}, - {file = "mypy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:49ae115da099dcc0922a7a895c1eec82c1518109ea5c162ed50e3b3594c71208"}, - {file = "mypy-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b27958f8c76bed8edaa63da0739d76e4e9ad4ed325c814f9b3851425582a3cd"}, - {file = "mypy-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925cd6a3b7b55dfba252b7c4561892311c5358c6b5a601847015a1ad4eb7d332"}, - {file = "mypy-1.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8f57e6b6927a49550da3d122f0cb983d400f843a8a82e65b3b380d3d7259468f"}, - {file = "mypy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a43ef1c8ddfdb9575691720b6352761f3f53d85f1b57d7745701041053deff30"}, - {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"}, - {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, ] [package.dependencies] @@ -1179,6 +1206,7 @@ typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] install-types = ["pip"] +mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] @@ -1194,77 +1222,47 @@ files = [ [[package]] name = "numpy" -version = "1.25.2" +version = "1.26.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, - {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, - {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, - {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, - {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, - {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, - {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, - {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, - {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, - {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, - {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, - {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, -] - -[[package]] -name = "numpy" -version = "1.26.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = "<3.13,>=3.9" -files = [ - {file = "numpy-1.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af"}, - {file = "numpy-1.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575"}, - {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244"}, - {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67"}, - {file = "numpy-1.26.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2"}, - {file = "numpy-1.26.1-cp310-cp310-win32.whl", hash = "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297"}, - {file = "numpy-1.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab"}, - {file = "numpy-1.26.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a"}, - {file = "numpy-1.26.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9"}, - {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3"}, - {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974"}, - {file = "numpy-1.26.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c"}, - {file = "numpy-1.26.1-cp311-cp311-win32.whl", hash = "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b"}, - {file = "numpy-1.26.1-cp311-cp311-win_amd64.whl", hash = "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53"}, - {file = "numpy-1.26.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f"}, - {file = "numpy-1.26.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24"}, - {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e"}, - {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124"}, - {file = "numpy-1.26.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c"}, - {file = "numpy-1.26.1-cp312-cp312-win32.whl", hash = "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66"}, - {file = "numpy-1.26.1-cp312-cp312-win_amd64.whl", hash = "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7"}, - {file = "numpy-1.26.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e"}, - {file = "numpy-1.26.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617"}, - {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e"}, - {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908"}, - {file = "numpy-1.26.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5"}, - {file = "numpy-1.26.1-cp39-cp39-win32.whl", hash = "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104"}, - {file = "numpy-1.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f"}, - {file = "numpy-1.26.1.tar.gz", hash = "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] [[package]] @@ -1280,107 +1278,50 @@ files = [ [[package]] name = "pandas" -version = "2.1.0" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pandas-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40dd20439ff94f1b2ed55b393ecee9cb6f3b08104c2c40b0cb7186a2f0046242"}, - {file = "pandas-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d4f38e4fedeba580285eaac7ede4f686c6701a9e618d8a857b138a126d067f2f"}, - {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e6a0fe052cf27ceb29be9429428b4918f3740e37ff185658f40d8702f0b3e09"}, - {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d81e1813191070440d4c7a413cb673052b3b4a984ffd86b8dd468c45742d3cc"}, - {file = "pandas-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eb20252720b1cc1b7d0b2879ffc7e0542dd568f24d7c4b2347cb035206936421"}, - {file = "pandas-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:38f74ef7ebc0ffb43b3d633e23d74882bce7e27bfa09607f3c5d3e03ffd9a4a5"}, - {file = "pandas-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cda72cc8c4761c8f1d97b169661f23a86b16fdb240bdc341173aee17e4d6cedd"}, - {file = "pandas-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d97daeac0db8c993420b10da4f5f5b39b01fc9ca689a17844e07c0a35ac96b4b"}, - {file = "pandas-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8c58b1113892e0c8078f006a167cc210a92bdae23322bb4614f2f0b7a4b510f"}, - {file = "pandas-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:629124923bcf798965b054a540f9ccdfd60f71361255c81fa1ecd94a904b9dd3"}, - {file = "pandas-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:70cf866af3ab346a10debba8ea78077cf3a8cd14bd5e4bed3d41555a3280041c"}, - {file = "pandas-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d53c8c1001f6a192ff1de1efe03b31a423d0eee2e9e855e69d004308e046e694"}, - {file = "pandas-2.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:86f100b3876b8c6d1a2c66207288ead435dc71041ee4aea789e55ef0e06408cb"}, - {file = "pandas-2.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28f330845ad21c11db51e02d8d69acc9035edfd1116926ff7245c7215db57957"}, - {file = "pandas-2.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9a6ccf0963db88f9b12df6720e55f337447aea217f426a22d71f4213a3099a6"}, - {file = "pandas-2.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d99e678180bc59b0c9443314297bddce4ad35727a1a2656dbe585fd78710b3b9"}, - {file = "pandas-2.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b31da36d376d50a1a492efb18097b9101bdbd8b3fbb3f49006e02d4495d4c644"}, - {file = "pandas-2.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:0164b85937707ec7f70b34a6c3a578dbf0f50787f910f21ca3b26a7fd3363437"}, - {file = "pandas-2.1.0.tar.gz", hash = "sha256:62c24c7fc59e42b775ce0679cfa7b14a5f9bfb7643cfbe708c960699e05fb918"}, -] - -[package.dependencies] -numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.1" - -[package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] -aws = ["s3fs (>=2022.05.0)"] -clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] -compression = ["zstandard (>=0.17.0)"] -computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2022.05.0)"] -gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] -hdf5 = ["tables (>=3.7.0)"] -html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] -mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] -spss = ["pyreadstat (>=1.1.5)"] -sql-other = ["SQLAlchemy (>=1.4.36)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.8.0)"] - -[[package]] -name = "pandas" -version = "2.1.1" +version = "2.1.4" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, - {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, - {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, - {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, - {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, - {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, - {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, - {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, - {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, - {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, - {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, ] [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" [package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] aws = ["s3fs (>=2022.05.0)"] clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] compression = ["zstandard (>=0.17.0)"] @@ -1400,18 +1341,18 @@ plot = ["matplotlib (>=3.6.1)"] postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] spss = ["pyreadstat (>=1.1.5)"] sql-other = ["SQLAlchemy (>=1.4.36)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] [[package]] name = "platformdirs" -version = "3.11.0" +version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, - {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, ] [package.extras] @@ -1435,24 +1376,22 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.4" +version = "4.25.2" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.24.4-cp310-abi3-win32.whl", hash = "sha256:ec9912d5cb6714a5710e28e592ee1093d68c5ebfeda61983b3f40331da0b1ebb"}, - {file = "protobuf-4.24.4-cp310-abi3-win_amd64.whl", hash = "sha256:1badab72aa8a3a2b812eacfede5020472e16c6b2212d737cefd685884c191085"}, - {file = "protobuf-4.24.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e61a27f362369c2f33248a0ff6896c20dcd47b5d48239cb9720134bef6082e4"}, - {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:bffa46ad9612e6779d0e51ae586fde768339b791a50610d85eb162daeb23661e"}, - {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b493cb590960ff863743b9ff1452c413c2ee12b782f48beca77c8da3e2ffe9d9"}, - {file = "protobuf-4.24.4-cp37-cp37m-win32.whl", hash = "sha256:dbbed8a56e56cee8d9d522ce844a1379a72a70f453bde6243e3c86c30c2a3d46"}, - {file = "protobuf-4.24.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6b7d2e1c753715dcfe9d284a25a52d67818dd43c4932574307daf836f0071e37"}, - {file = "protobuf-4.24.4-cp38-cp38-win32.whl", hash = "sha256:02212557a76cd99574775a81fefeba8738d0f668d6abd0c6b1d3adcc75503dbe"}, - {file = "protobuf-4.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:2fa3886dfaae6b4c5ed2730d3bf47c7a38a72b3a1f0acb4d4caf68e6874b947b"}, - {file = "protobuf-4.24.4-cp39-cp39-win32.whl", hash = "sha256:b77272f3e28bb416e2071186cb39efd4abbf696d682cbb5dc731308ad37fa6dd"}, - {file = "protobuf-4.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:9fee5e8aa20ef1b84123bb9232b3f4a5114d9897ed89b4b8142d81924e05d79b"}, - {file = "protobuf-4.24.4-py3-none-any.whl", hash = "sha256:80797ce7424f8c8d2f2547e2d42bfbb6c08230ce5832d6c099a37335c9c90a92"}, - {file = "protobuf-4.24.4.tar.gz", hash = "sha256:5a70731910cd9104762161719c3d883c960151eea077134458503723b60e3667"}, + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, ] [[package]] @@ -1468,13 +1407,13 @@ files = [ [[package]] name = "pyasn1" -version = "0.5.0" +version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, - {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, ] [[package]] @@ -1590,17 +1529,18 @@ requests = ">=2.14.0" [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" @@ -1692,13 +1632,13 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale [[package]] name = "pytest-mock" -version = "3.11.1" +version = "3.12.0" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, - {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, ] [package.dependencies] @@ -1721,18 +1661,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "python-editor" -version = "1.0.4" -description = "Programmatically open an editor, capture the result." -optional = false -python-versions = "*" -files = [ - {file = "python-editor-1.0.4.tar.gz", hash = "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b"}, - {file = "python_editor-1.0.4-py2-none-any.whl", hash = "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8"}, - {file = "python_editor-1.0.4-py3-none-any.whl", hash = "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d"}, -] - [[package]] name = "pytz" version = "2023.3.post1" @@ -1756,6 +1684,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1763,8 +1692,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1781,6 +1717,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1788,6 +1725,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1830,13 +1768,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "13.6.0" +version = "13.7.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245"}, - {file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef"}, + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, ] [package.dependencies] @@ -1860,6 +1798,20 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "runs" +version = "1.2.0" +description = "🏃 Run a block of text as a subprocess 🏃" +optional = false +python-versions = ">=3.8" +files = [ + {file = "runs-1.2.0-py3-none-any.whl", hash = "sha256:ec6fe3b24dfa20c5c4e5c4806d3b35bb880aad0e787a8610913c665c5a7cc07c"}, + {file = "runs-1.2.0.tar.gz", hash = "sha256:8804271011b7a2eeb0d77c3e3f556e5ce5f602fa0dd2a31ed0c1222893be69b7"}, +] + +[package.dependencies] +xmod = "*" + [[package]] name = "semver" version = "3.0.2" @@ -1873,17 +1825,17 @@ files = [ [[package]] name = "setuptools" -version = "68.2.2" +version = "69.0.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -1955,40 +1907,39 @@ files = [ [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] name = "tzdata" -version = "2023.3" +version = "2023.4" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] name = "urllib3" -version = "2.0.7" +version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, - {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2008,180 +1959,202 @@ toml = "*" [[package]] name = "wcwidth" -version = "0.2.8" +version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" files = [ - {file = "wcwidth-0.2.8-py2.py3-none-any.whl", hash = "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704"}, - {file = "wcwidth-0.2.8.tar.gz", hash = "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"}, + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] [[package]] name = "wrapt" -version = "1.15.0" +version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xmod" +version = "1.8.1" +description = "🌱 Turn any object into a module 🌱" +optional = false +python-versions = ">=3.8" +files = [ + {file = "xmod-1.8.1-py3-none-any.whl", hash = "sha256:a24e9458a4853489042522bdca9e50ee2eac5ab75c809a91150a8a7f40670d48"}, + {file = "xmod-1.8.1.tar.gz", hash = "sha256:38c76486b9d672c546d57d8035df0beb7f4a9b088bc3fb2de5431ae821444377"}, ] [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -2191,4 +2164,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "be8572e90eb9a6996fc554d7522c72b55e11f82107dd27162ac2a3e8559b4b5a" +content-hash = "e6f67b753371bdbe515e2326b68d32e46a492722b13a8b32a2636fe1e0c39028" diff --git a/airbyte-ci/connectors/base_images/pyproject.toml b/airbyte-ci/connectors/base_images/pyproject.toml index 94eced3e1e19..d853b591b277 100644 --- a/airbyte-ci/connectors/base_images/pyproject.toml +++ b/airbyte-ci/connectors/base_images/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "airbyte-connectors-base-images" -version = "0.1.2" +version = "1.0.1" description = "This package is used to generate and publish the base images for Airbyte Connectors." authors = ["Augustin Lafanechere "] readme = "README.md" @@ -8,7 +8,7 @@ packages = [{include = "base_images"}] include = ["generated"] [tool.poetry.dependencies] python = "^3.10" -dagger-io = "0.6.4" +dagger-io = "==0.9.6" gitpython = "^3.1.35" rich = "^13.5.2" semver = "^3.0.1" @@ -32,3 +32,10 @@ generate-docs = "base_images.commands:generate_docs" generate-release = "base_images.commands:generate_release" publish = "base_images.commands:publish_existing_version" +[tool.poe.tasks] +test = "pytest tests" + +[tool.airbyte_ci] +extra_poetry_groups = ["dev"] +poe_tasks = ["test"] +mount_docker_socket = true diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py index 2c02785957b4..c024caf01587 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py @@ -117,6 +117,9 @@ def __load_gsm_secrets(self) -> List[RemoteSecret]: enabled_versions = [version["name"] for version in versions_data["versions"] if version["state"] == "ENABLED"] if len(enabled_versions) > 1: self.logger.critical(f"{log_name} should have one enabled version at the same time!!!") + if not enabled_versions: + self.logger.warning(f"{log_name} doesn't have enabled versions for {secret_name}") + continue enabled_version = enabled_versions[0] secret_url = f"https://secretmanager.googleapis.com/v1/{enabled_version}:access" secret_data = self.api.get(secret_url) diff --git a/airbyte-ci/connectors/ci_credentials/pyproject.toml b/airbyte-ci/connectors/ci_credentials/pyproject.toml index 5ca538a9828d..a85705db5762 100644 --- a/airbyte-ci/connectors/ci_credentials/pyproject.toml +++ b/airbyte-ci/connectors/ci_credentials/pyproject.toml @@ -17,7 +17,7 @@ click = "^8.1.3" pyyaml = "^6.0" common_utils = { path = "../common_utils", develop = true } -[tool.poetry.group.test.dependencies] +[tool.poetry.group.dev.dependencies] requests-mock = "^1.10.0" pytest = "^7.3.1" @@ -27,3 +27,10 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] ci_credentials = "ci_credentials.main:ci_credentials" + +[tool.poe.tasks] +test = "pytest tests" + +[tool.airbyte_ci] +extra_poetry_groups = ["dev"] +poe_tasks = ["test"] diff --git a/airbyte-ci/connectors/common_utils/pyproject.toml b/airbyte-ci/connectors/common_utils/pyproject.toml index 97e23e9ffac5..ab21cfab1844 100644 --- a/airbyte-ci/connectors/common_utils/pyproject.toml +++ b/airbyte-ci/connectors/common_utils/pyproject.toml @@ -14,12 +14,19 @@ cryptography = "^3.4.7" requests = "^2.28.2" pyjwt = "^2.1.0" -[tool.poetry.group.test.dependencies] -pytest = "^7.2.2" [tool.poetry.group.dev.dependencies] requests-mock = "^1.9.3" +pytest = "^7.2.2" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.poe.tasks] +test = "pytest tests" + +[tool.airbyte_ci] +extra_poetry_groups = ["dev"] +# Disable poe tasks as tests are not passing ATM +poe_tasks = [] diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py deleted file mode 100644 index 99bd8e1786b0..000000000000 --- a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py +++ /dev/null @@ -1,320 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys -from pathlib import Path -from typing import Callable, Iterable, Optional, Set, Tuple - -from connector_ops.utils import Connector, ConnectorLanguage -from pydash.objects import get - - -def check_migration_guide(connector: Connector) -> bool: - """Check if a migration guide is available for the connector if a breaking change was introduced.""" - - breaking_changes = get(connector.metadata, "releases.breakingChanges") - if not breaking_changes: - return True - - migration_guide_file_path = connector.migration_guide_file_path - migration_guide_exists = migration_guide_file_path is not None and migration_guide_file_path.exists() - if not migration_guide_exists: - print( - f"Migration guide file is missing for {connector.name}. Please create a migration guide at {connector.migration_guide_file_path}" - ) - return False - - # Check that the migration guide begins with # {connector name} Migration Guide - expected_title = f"# {connector.name_from_metadata} Migration Guide" - expected_version_header_start = "## Upgrading to " - with open(migration_guide_file_path) as f: - first_line = f.readline().strip() - if not first_line == expected_title: - print( - f"Migration guide file for {connector.technical_name} does not start with the correct header. Expected '{expected_title}', got '{first_line}'" - ) - return False - - # Check that the migration guide contains a section for each breaking change key ## Upgrading to {version} - # Note that breaking change is a dict where the version is the key - # Note that the migration guide must have the sections in order of the version descending - # 3.0.0, 2.0.0, 1.0.0, etc - # This means we have to record the headings in the migration guide and then check that they are in order - # We also have to check that the headings are in the breaking changes dict - - ordered_breaking_changes = sorted(breaking_changes.keys(), reverse=True) - ordered_expected_headings = [f"{expected_version_header_start}{version}" for version in ordered_breaking_changes] - - ordered_heading_versions = [] - for line in f: - stripped_line = line.strip() - if stripped_line.startswith(expected_version_header_start): - version = stripped_line.replace(expected_version_header_start, "") - ordered_heading_versions.append(version) - - if ordered_breaking_changes != ordered_heading_versions: - print(f"Migration guide file for {connector.name} has incorrect version headings.") - print("Check for missing, extra, or misordered headings, or headers with typos.") - print(f"Expected headings: {ordered_expected_headings}") - return False - - return True - - -def check_documentation_file_exists(connector: Connector) -> bool: - """Check if a markdown file with connector documentation is available - in docs/integrations/s/.md - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether a documentation file was found. - """ - file_path = connector.documentation_file_path - - return file_path is not None and file_path.exists() - - -def check_documentation_follows_guidelines(connector: Connector) -> bool: - """Documentation guidelines are defined here https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw""" - follows_guidelines = True - with open(connector.documentation_file_path) as f: - doc_lines = [line.lower() for line in f.read().splitlines()] - if not doc_lines[0].startswith("# "): - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - # We usually don't have a metadata if the connector is not published. - if connector.metadata: - if doc_lines[0].strip() != f"# {connector.metadata['name'].lower()}": - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - elif not doc_lines[0].startswith("# "): - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - - expected_sections = ["## Prerequisites", "## Setup guide", "## Supported sync modes", "## Supported streams", "## Changelog"] - - for expected_section in expected_sections: - if expected_section.lower() not in doc_lines: - print(f"Connector documentation is missing a '{expected_section.replace('#', '').strip()}' section.") - follows_guidelines = False - return follows_guidelines - - -def check_changelog_entry_is_updated(connector: Connector) -> bool: - """Check that the changelog entry is updated for the latest connector version - in docs/integrations//.md - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether a the changelog is up to date. - """ - if not check_documentation_file_exists(connector): - return False - with open(connector.documentation_file_path) as f: - after_changelog = False - for line in f: - if "# changelog" in line.lower(): - after_changelog = True - if after_changelog and connector.version in line: - return True - return False - - -def check_connector_icon_is_available(connector: Connector) -> bool: - """Check an SVG icon exists for a connector in - in airbyte-config-oss/init-oss/src/main/resources/icons/.svg - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether an icon exists for this connector. - """ - return connector.icon_path.exists() - - -def read_all_files_in_directory( - directory: Path, ignored_directories: Optional[Set[str]] = None, ignored_filename_patterns: Optional[Set[str]] = None -) -> Iterable[Tuple[str, str]]: - ignored_directories = ignored_directories if ignored_directories is not None else {} - ignored_filename_patterns = ignored_filename_patterns if ignored_filename_patterns is not None else {} - - for path in directory.rglob("*"): - ignore_directory = any([ignored_directory in path.parts for ignored_directory in ignored_directories]) - ignore_filename = any([path.match(ignored_filename_pattern) for ignored_filename_pattern in ignored_filename_patterns]) - ignore = ignore_directory or ignore_filename - if path.is_file() and not ignore: - try: - for line in open(path, "r"): - yield path, line - except UnicodeDecodeError: - print(f"{path} could not be decoded as it is not UTF8.") - continue - - -IGNORED_DIRECTORIES_FOR_HTTPS_CHECKS = { - ".venv", - "tests", - "unit_tests", - "integration_tests", - "build", - "source-file", - ".pytest_cache", - "acceptance_tests_logs", - ".hypothesis", -} - -IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = { - "*Test.java", - "*.jar", - "*.pyc", - "*.gz", - "*.svg", - "expected_records.jsonl", - "expected_records.json", -} -IGNORED_URLS_PREFIX = { - "http://json-schema.org", - "http://localhost", -} - - -def is_comment(line: str, file_path: Path): - language_comments = { - ".py": "#", - ".yml": "#", - ".yaml": "#", - ".java": "//", - ".md": "build secret-->integration secret-->cat + secret-->airbyte_lib_validation build-->integration build-->cat end @@ -256,24 +292,43 @@ flowchart TD #### Options -| Option | Multiple | Default value | Description | -| ------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | -| `--fast-tests-only` | True | False | Run unit tests only, skipping integration tests or any tests explicitly tagged as slow. Use this for more frequent checks, when it is not feasible to run the entire test suite. | -| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. | -| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. | +| Option | Multiple | Default value | Description | +| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--skip-step/-x` | True | | Skip steps by id e.g. `-x unit -x acceptance` | +| `--only-step/-k` | True | | Only run specific steps by id e.g. `-k unit -k acceptance` | +| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | +| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. | +| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. | +| `--.=` | True | | You can pass extra parameters for specific test steps. More details in the extra parameters section below | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. Note: -- The above options are implemented for Java connectors but may not be available for Python connectors. If an option is not supported, the pipeline will not fail but instead the 'default' behavior will be executed. +- The above options are implemented for Java connectors but may not be available for Python + connectors. If an option is not supported, the pipeline will not fail but instead the 'default' + behavior will be executed. + +#### Extra parameters + +You can pass extra parameters to the following steps: + +- `unit` +- `integration` +- `acceptance` + +This allows you to override the default parameters of these steps. For example, you can only run the +`test_read` test of the acceptance test suite with: +`airbyte-ci connectors --name=source-pokeapi test --acceptance.-k=test_read` Here the `-k` parameter +is passed to the pytest command running acceptance tests. Please keep in mind that the extra +parameters are not validated by the CLI: if you pass an invalid parameter, you'll face a late +failure during the pipeline execution. ### `connectors build` command -Run a build pipeline for one or multiple connectors and export the built docker image to the local docker host. -It's mainly purposed for local use. +Run a build pipeline for one or multiple connectors and export the built docker image to the local +docker host. It's mainly purposed for local use. -Build a single connector: -`airbyte-ci connectors --name=source-pokeapi build` +Build a single connector: `airbyte-ci connectors --name=source-pokeapi build` Build a single connector with a custom image tag: `airbyte-ci connectors --name=source-pokeapi build --tag=my-custom-tag` @@ -289,11 +344,9 @@ You will get: Build multiple connectors: `airbyte-ci connectors --name=source-pokeapi --name=source-bigquery build` -Build certified connectors: -`airbyte-ci connectors --support-level=certified build` +Build certified connectors: `airbyte-ci connectors --support-level=certified build` -Build connectors changed on the current branch: -`airbyte-ci connectors --modified build` +Build connectors changed on the current branch: `airbyte-ci connectors --modified build` #### What it runs @@ -337,8 +390,8 @@ flowchart TD ### `connectors publish` command -Run a publish pipeline for one or multiple connectors. -It's mainly purposed for CI use to release a connector update. +Run a publish pipeline for one or multiple connectors. It's mainly purposed for CI use to release a +connector update. ### Examples @@ -346,15 +399,19 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod ### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------------------------ | -------- | --------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | -| `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | -| `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | -| `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | -| `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | -| `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | -| `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------------------ | -------- | ------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | +| `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | +| `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | +| `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | +| `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | +| `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | +| `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | +| `--python-registry-token` | False | | `PYTHON_REGISTRY_TOKEN` | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--python-registry-url` | False | https://upload.pypi.org/legacy/ | `PYTHON_REGISTRY_URL` | The python registry to publish to. Defaults to main pypi | +| `--python-registry-check-url` | False | https://pypi.org/pypi | `PYTHON_REGISTRY_CHECK_URL` | The python registry url to check whether a package is published already | I've added an empty "Default" column, and you can fill in the default values as needed. @@ -365,12 +422,36 @@ flowchart TD validate[Validate the metadata file] check[Check if the connector image already exists] build[Build the connector image for all platform variants] + publish_to_python_registry[Push the connector image to the python registry if enabled] upload_spec[Upload connector spec to the spec cache bucket] push[Push the connector image from DockerHub, with platform variants] pull[Pull the connector image from DockerHub to check SPEC can be run and the image layers are healthy] upload_metadata[Upload its metadata file to the metadata service bucket] - validate-->check-->build-->upload_spec-->push-->pull-->upload_metadata + validate-->check-->build-->upload_spec-->publish_to_python_registry-->push-->pull-->upload_metadata +``` + +#### Python registry publishing + +If `remoteRegistries.pypi.enabled` in the connector metadata is set to `true`, the connector will be +published to the python registry. To do so, the `--python-registry-token` and +`--python-registry-url` options are used to authenticate with the registry and publish the +connector. If the current version of the connector is already published to the registry, the publish +will be skipped (the `--python-registry-check-url` is used for the check). + +On a pre-release, the connector will be published as a `.dev` version. + +The `remoteRegistries.pypi.packageName` field holds the name of the used package name. It should be +set to `airbyte-source-`. Certified Python connectors are required to have PyPI +publishing enabled. + +An example `remoteRegistries` entry in a connector `metadata.yaml` looks like this: + +```yaml +remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pokeapi ``` ### `connectors bump_version` command @@ -379,7 +460,8 @@ Bump the version of the selected connectors. ### Examples -Bump source-openweather: `airbyte-ci connectors --name=source-openweather bump_version patch ""` +Bump source-openweather: +`airbyte-ci connectors --name=source-openweather bump_version patch ""` #### Arguments @@ -389,13 +471,29 @@ Bump source-openweather: `airbyte-ci connectors --name=source-openweather bump_v | `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | | `CHANGELOG_ENTRY` | The changelog entry that will get added to the connector documentation | +### `connectors upgrade_cdk` command + +Upgrade the CDK version of the selected connectors by updating the dependency in the setup.py file. + +### Examples + +Upgrade for source-openweather: +`airbyte-ci connectors --name=source-openweather upgrade_cdk ` + +#### Arguments + +| Argument | Description | +| ------------- | ------------------------------------------------------- | +| `CDK_VERSION` | CDK version to set (default to the most recent version) | + ### `connectors upgrade_base_image` command Modify the selected connector metadata to use the latest base image version. ### Examples -Upgrade the base image for source-openweather: `airbyte-ci connectors --name=source-openweather upgrade_base_image` +Upgrade the base image for source-openweather: +`airbyte-ci connectors --name=source-openweather upgrade_base_image` ### Options @@ -416,7 +514,8 @@ Make a connector using a Dockerfile migrate to the base image by: ### Examples -Migrate source-openweather to use the base image: `airbyte-ci connectors --name=source-openweather migrate_to_base_image` +Migrate source-openweather to use the base image: +`airbyte-ci connectors --name=source-openweather migrate_to_base_image` ### Arguments @@ -433,9 +532,10 @@ Available commands: ### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------ | -------- | ------- | --------------------------- | ---------------------------------------------- | -| `--quiet/-q` | False | False | | Hide formatter execution details in reporting. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `--quiet/-q` | False | False | | Hide formatter execution details in reporting. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | ### Examples @@ -444,15 +544,54 @@ Available commands: ### `format check all` command -This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix all`. +This command runs formatting checks, but does not format the code in place. It will exit 1 as soon +as a failure is encountered. To fix errors, use `airbyte-ci format fix all`. -Running `airbyte-ci format check` will run checks on all different types of code. Run `airbyte-ci format check --help` for subcommands to check formatting for only certain types of files. +Running `airbyte-ci format check` will run checks on all different types of code. Run +`airbyte-ci format check --help` for subcommands to check formatting for only certain types of +files. ### `format fix all` command -This command runs formatting checks and reformats any code that would be reformatted, so it's recommended to stage changes you might have before running this command. +This command runs formatting checks and reformats any code that would be reformatted, so it's +recommended to stage changes you might have before running this command. + +Running `airbyte-ci format fix all` will format all of the different types of code. Run +`airbyte-ci format fix --help` for subcommands to format only certain types of files. -Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. +### `poetry` command subgroup + +Available commands: + +- `airbyte-ci poetry publish` + +### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ---------------- | -------- | ------- | --------------------------- | -------------------------------------------------------------- | +| `--package-path` | True | | | The path to the python package to execute a poetry command on. | + +### Examples + +- Publish a python package: + `airbyte-ci poetry --package-path=path/to/package publish --publish-name=my-package --publish-version="1.2.3" --python-registry-token="..." --registry-url="http://host.docker.internal:8012/"` + +### `publish` command + +This command publishes poetry packages (using `pyproject.toml`) or python packages (using +`setup.py`) to a python registry. + +For poetry packages, the package name and version can be taken from the `pyproject.toml` file or be +specified as options. + +#### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ------------------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------- | +| `--publish-name` | False | | | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--publish-version` | False | | | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--python-registry-token` | True | | PYTHON_REGISTRY_TOKEN | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--python-registry-url` | False | https://upload.pypi.org/legacy/ | PYTHON_REGISTRY_URL | The python registry to publish to. Defaults to main pypi | ### `metadata` command subgroup @@ -462,8 +601,8 @@ Available commands: ### `metadata deploy orchestrator` command -This command deploys the metadata service orchestrator to production. -The `DAGSTER_CLOUD_METADATA_API_TOKEN` environment variable must be set. +This command deploys the metadata service orchestrator to production. The +`DAGSTER_CLOUD_METADATA_API_TOKEN` environment variable must be set. #### Example @@ -478,124 +617,187 @@ flowchart TD ### `tests` command -This command runs the Python tests for a airbyte-ci poetry package. - -#### Arguments +This command runs the poe tasks declared in the `[tool.airbyte-ci]` section of our internal poetry packages. +Feel free to checkout this [Pydantic model](https://github.com/airbytehq/airbyte/blob/main/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py#L9) to see the list of available options in `[tool.airbyte-ci]` section. -| Option | Required | Default | Mapped environment variable | Description | -| --------------------- | -------- | ------- | --------------------------- | ----------------------------------- | -| `poetry_package_path` | True | | | The path to poetry package to test. | +You can find the list of internal packages [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py#L1) #### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------ | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------------ | -| `--test-directory` | False | tests | | The path to the directory on which pytest should discover tests, relative to the poetry package. | +| Option | Required | Multiple | Description | +| -------------------------- | -------- | -------- | ------------------------------------------------------------------------------------------- | +| `--poetry-package-path/-p` | False | True | Poetry packages path to run the poe tasks for. | +| `--modified` | False | False | Run poe tasks of modified internal poetry packages. | +| `--ci-requirements` | False | False | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | -#### Example +#### Examples +You can pass multiple `--poetry-package-path` options to run poe tasks. + +E.G.: running Poe tasks on `airbyte-lib` and `airbyte-ci/connectors/pipelines`: +`airbyte-ci test --poetry-package-path=airbyte-ci/connectors/pipelines --poetry-package-path=airbyte-lib` + +E.G.: running Poe tasks on the modified internal packages of the current branch: +`airbyte-ci test --modified` -`airbyte-ci test airbyte-ci/connectors/pipelines --test-directory=tests` -`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests` ## Changelog -| Version | PR | Description | -| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.10.7 | [#33248](https://github.com/airbytehq/airbyte/pull/33248) | Fix bug which broke airbyte-ci connectors tests when optional DockerHub credentials env vars are not set. | -| 2.10.6 | [#33170](https://github.com/airbytehq/airbyte/pull/33170) | Remove Dagger logs from console output of `format`. | -| 2.10.5 | [#33097](https://github.com/airbytehq/airbyte/pull/33097) | Improve `format` performances, exit with 1 status code when `fix` changes files. | -| 2.10.4 | [#33206](https://github.com/airbytehq/airbyte/pull/33206) | Add "-y/--yes" Flag to allow preconfirmation of prompts | -| 2.10.3 | [#33080](https://github.com/airbytehq/airbyte/pull/33080) | Fix update failing due to SSL error on install. | -| 2.10.2 | [#33008](https://github.com/airbytehq/airbyte/pull/33008) | Fix local `connector build`. | -| 2.10.1 | [#32928](https://github.com/airbytehq/airbyte/pull/32928) | Fix BuildConnectorImages constructor. | -| 2.10.0 | [#32819](https://github.com/airbytehq/airbyte/pull/32819) | Add `--tag` option to connector build. | -| 2.9.0 | [#32816](https://github.com/airbytehq/airbyte/pull/32816) | Add `--architecture` option to connector build. | -| 2.8.1 | [#32999](https://github.com/airbytehq/airbyte/pull/32999) | Improve Java code formatting speed | -| 2.8.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Move pipx install to `airbyte-ci-dev`, and add auto-update feature targeting binary | -| 2.7.3 | [#32847](https://github.com/airbytehq/airbyte/pull/32847) | Improve --modified behaviour for pull requests. | -| 2.7.2 | [#32839](https://github.com/airbytehq/airbyte/pull/32839) | Revert changes in v2.7.1. | -| 2.7.1 | [#32806](https://github.com/airbytehq/airbyte/pull/32806) | Improve --modified behaviour for pull requests. | -| 2.7.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Merge airbyte-ci-internal into airbyte-ci | -| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | -| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | -| 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | -| 2.5.7 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | -| 2.5.6 | [#32139](https://github.com/airbytehq/airbyte/pull/32139) | Test coverage report on Python connector UnitTest. | -| 2.5.5 | [#32114](https://github.com/airbytehq/airbyte/pull/32114) | Create cache mount for `/var/lib/docker` to store images in `dind` context. | -| 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | -| 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | -| 2.5.2 | [#31871](https://github.com/airbytehq/airbyte/pull/31871) | Deactivate PR comments, add HTML report links to the PR status when its ready. | -| 2.5.1 | [#31774](https://github.com/airbytehq/airbyte/pull/31774) | Add a docker configuration check on `airbyte-ci` startup. | -| 2.5.0 | [#31766](https://github.com/airbytehq/airbyte/pull/31766) | Support local connectors secrets. | -| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | -| 2.3.1 | [#31748](https://github.com/airbytehq/airbyte/pull/31748) | Use AsyncClick library instead of base Click. | -| 2.3.0 | [#31699](https://github.com/airbytehq/airbyte/pull/31699) | Support optional concurrent CAT execution. | -| 2.2.6 | [#31752](https://github.com/airbytehq/airbyte/pull/31752) | Only authenticate when secrets are available. | -| 2.2.5 | [#31718](https://github.com/airbytehq/airbyte/pull/31718) | Authenticate the sidecar docker daemon to DockerHub. | -| 2.2.4 | [#31535](https://github.com/airbytehq/airbyte/pull/31535) | Improve gradle caching when building java connectors. | -| 2.2.3 | [#31688](https://github.com/airbytehq/airbyte/pull/31688) | Fix failing `CheckBaseImageUse` step when not running on PR. | -| 2.2.2 | [#31659](https://github.com/airbytehq/airbyte/pull/31659) | Support builds on x86_64 platform | -| 2.2.1 | [#31653](https://github.com/airbytehq/airbyte/pull/31653) | Fix CheckBaseImageIsUsed failing on non certified connectors. | -| 2.2.0 | [#30527](https://github.com/airbytehq/airbyte/pull/30527) | Add a new check for python connectors to make sure certified connectors use our base image. | -| 2.1.1 | [#31488](https://github.com/airbytehq/airbyte/pull/31488) | Improve `airbyte-ci` start time with Click Lazy load | -| 2.1.0 | [#31412](https://github.com/airbytehq/airbyte/pull/31412) | Run airbyte-ci from any where in airbyte project | -| 2.0.4 | [#31487](https://github.com/airbytehq/airbyte/pull/31487) | Allow for third party connector selections | -| 2.0.3 | [#31525](https://github.com/airbytehq/airbyte/pull/31525) | Refactor folder structure | -| 2.0.2 | [#31533](https://github.com/airbytehq/airbyte/pull/31533) | Pip cache volume by python version. | -| 2.0.1 | [#31545](https://github.com/airbytehq/airbyte/pull/31545) | Reword the changelog entry when using `migrate_to_base_image`. | -| 2.0.0 | [#31424](https://github.com/airbytehq/airbyte/pull/31424) | Remove `airbyte-ci connectors format` command. | -| 1.9.4 | [#31478](https://github.com/airbytehq/airbyte/pull/31478) | Fix running tests for connector-ops package. | -| 1.9.3 | [#31457](https://github.com/airbytehq/airbyte/pull/31457) | Improve the connector documentation for connectors migrated to our base image. | -| 1.9.2 | [#31426](https://github.com/airbytehq/airbyte/pull/31426) | Concurrent execution of java connectors tests. | -| 1.9.1 | [#31455](https://github.com/airbytehq/airbyte/pull/31455) | Fix `None` docker credentials on publish. | -| 1.9.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | -| 1.8.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | -| 1.7.2 | [#31343](https://github.com/airbytehq/airbyte/pull/31343) | Bind Pytest integration tests to a dockerhost. | -| 1.7.1 | [#31332](https://github.com/airbytehq/airbyte/pull/31332) | Disable Gradle step caching on source-postgres. | -| 1.7.0 | [#30526](https://github.com/airbytehq/airbyte/pull/30526) | Implement pre/post install hooks support. | -| 1.6.0 | [#30474](https://github.com/airbytehq/airbyte/pull/30474) | Test connector inside their containers. | -| 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | -| 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | -| 1.4.6 | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date | -| 1.4.5 | [#31133](https://github.com/airbytehq/airbyte/pull/31133) | Fix bug when building containers using `with_integration_base_java_and_normalization`. | -| 1.4.4 | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration. | -| 1.4.3 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Add --version and version check | -| 1.4.2 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement | -| 1.4.1 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants | -| 1.4.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add support for pyproject.toml as the prefered entry point for a connector package | -| 1.3.0 | [#30461](https://github.com/airbytehq/airbyte/pull/30461) | Add `--use-local-cdk` flag to all connectors commands | -| 1.2.3 | [#30477](https://github.com/airbytehq/airbyte/pull/30477) | Fix a test regression introduced the previous version. | -| 1.2.2 | [#30438](https://github.com/airbytehq/airbyte/pull/30438) | Add workaround to always stream logs properly with --is-local. | -| 1.2.1 | [#30384](https://github.com/airbytehq/airbyte/pull/30384) | Java connector test performance fixes. | -| 1.2.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add `--metadata-query` option to connectors command | -| 1.1.3 | [#30314](https://github.com/airbytehq/airbyte/pull/30314) | Stop patching gradle files to make them work with airbyte-ci. | -| 1.1.2 | [#30279](https://github.com/airbytehq/airbyte/pull/30279) | Fix correctness issues in layer caching by making atomic execution groupings | -| 1.1.1 | [#30252](https://github.com/airbytehq/airbyte/pull/30252) | Fix redundancies and broken logic in GradleTask, to speed up the CI runs. | -| 1.1.0 | [#29509](https://github.com/airbytehq/airbyte/pull/29509) | Refactor the airbyte-ci test command to run tests on any poetry package. | -| 1.0.0 | [#28000](https://github.com/airbytehq/airbyte/pull/29232) | Remove release stages in favor of support level from airbyte-ci. | -| 0.5.0 | [#28000](https://github.com/airbytehq/airbyte/pull/28000) | Run connector acceptance tests with dagger-in-dagger. | -| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | -| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | -| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | -| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | -| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | -| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | -| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | -| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | -| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | -| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | -| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | -| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | -| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | -| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | -| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | -| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | -| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | +| Version | PR | Description | +| ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 4.4.0 | [#35317](https://github.com/airbytehq/airbyte/pull/35317) | Augment java connector reports to include full logs and junit test results | +| 4.3.2 | [#35536](https://github.com/airbytehq/airbyte/pull/35536) | Make QA checks run correctly on `*-strict-encrypt` connectors. | +| 4.3.1 | [#35437](https://github.com/airbytehq/airbyte/pull/35437) | Do not run QA checks on publish, just MetadataValidation. | +| 4.3.0 | [#35438](https://github.com/airbytehq/airbyte/pull/35438) | Optionally disable telemetry with environment variable. | +| 4.2.4 | [#35325](https://github.com/airbytehq/airbyte/pull/35325) | Use `connectors_qa` for QA checks and remove redundant checks. | +| 4.2.3 | [#35322](https://github.com/airbytehq/airbyte/pull/35322) | Declare `connectors_qa` as an internal package for testing. | +| 4.2.2 | [#35364](https://github.com/airbytehq/airbyte/pull/35364) | Fix connector tests following gradle changes in #35307. | +| 4.2.1 | [#35204](https://github.com/airbytehq/airbyte/pull/35204) | Run `poetry check` before `poetry install` on poetry package install. | +| 4.2.0 | [#35103](https://github.com/airbytehq/airbyte/pull/35103) | Java 21 support. | +| 4.1.4 | [#35039](https://github.com/airbytehq/airbyte/pull/35039) | Fix bug which prevented gradle test reports from being added. | +| 4.1.3 | [#35010](https://github.com/airbytehq/airbyte/pull/35010) | Use `poetry install --no-root` in the builder container. | +| 4.1.2 | [#34945](https://github.com/airbytehq/airbyte/pull/34945) | Only install main dependencies when running poetry install. | +| 4.1.1 | [#34430](https://github.com/airbytehq/airbyte/pull/34430) | Speed up airbyte-ci startup (and airbyte-ci format). | +| 4.1.0 | [#34923](https://github.com/airbytehq/airbyte/pull/34923) | Include gradle test reports in HTML connector test report. | +| 4.0.0 | [#34736](https://github.com/airbytehq/airbyte/pull/34736) | Run poe tasks declared in internal poetry packages. | +| 3.10.4 | [#34867](https://github.com/airbytehq/airbyte/pull/34867) | Remove connector ops team | +| 3.10.3 | [#34836](https://github.com/airbytehq/airbyte/pull/34836) | Add check for python registry publishing enabled for certified python sources. | +| 3.10.2 | [#34044](https://github.com/airbytehq/airbyte/pull/34044) | Add pypi validation testing. | +| 3.10.1 | [#34756](https://github.com/airbytehq/airbyte/pull/34756) | Enable connectors tests in draft PRs. | +| 3.10.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of separate check URL to check whether package exists already. | +| 3.9.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of python registry URL via environment variable. | +| 3.8.1 | [#34607](https://github.com/airbytehq/airbyte/pull/34607) | Improve gradle dependency cache volume protection. | +| 3.8.0 | [#34316](https://github.com/airbytehq/airbyte/pull/34316) | Expose Dagger engine image name in `--ci-requirements` and add `--ci-requirements` to the `airbyte-ci` root command group. | +| 3.7.3 | [#34560](https://github.com/airbytehq/airbyte/pull/34560) | Simplify Gradle task execution framework by removing local maven repo support. | +| 3.7.2 | [#34555](https://github.com/airbytehq/airbyte/pull/34555) | Override secret masking in some very specific special cases. | +| 3.7.1 | [#34441](https://github.com/airbytehq/airbyte/pull/34441) | Support masked secret scrubbing for java CDK v0.15+ | +| 3.7.0 | [#34343](https://github.com/airbytehq/airbyte/pull/34343) | allow running connector upgrade_cdk for java connectors | +| 3.6.1 | [#34490](https://github.com/airbytehq/airbyte/pull/34490) | Fix inconsistent dagger log path typing | +| 3.6.0 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Add python registry publishing | +| 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | +| 3.5.2 | [#34381](https://github.com/airbytehq/airbyte/pull/34381) | Bind a sidecar docker host for `airbyte-ci test` | +| 3.5.1 | [#34321](https://github.com/airbytehq/airbyte/pull/34321) | Upgrade to Dagger 0.9.6 . | +| 3.5.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Pass extra params after Gradle tasks. | +| 3.4.2 | [#34301](https://github.com/airbytehq/airbyte/pull/34301) | Pass extra params after Gradle tasks. | +| 3.4.1 | [#34067](https://github.com/airbytehq/airbyte/pull/34067) | Use dagster-cloud 1.5.7 for deploy | +| 3.4.0 | [#34276](https://github.com/airbytehq/airbyte/pull/34276) | Introduce `--only-step` option for connector tests. | +| 3.3.0 | [#34218](https://github.com/airbytehq/airbyte/pull/34218) | Introduce `--ci-requirements` option for client defined CI runners. | +| 3.2.0 | [#34050](https://github.com/airbytehq/airbyte/pull/34050) | Connector test steps can take extra parameters | +| 3.1.3 | [#34136](https://github.com/airbytehq/airbyte/pull/34136) | Fix issue where dagger excludes were not being properly applied | +| 3.1.2 | [#33972](https://github.com/airbytehq/airbyte/pull/33972) | Remove secrets scrubbing hack for --is-local and other small tweaks. | +| 3.1.1 | [#33979](https://github.com/airbytehq/airbyte/pull/33979) | Fix AssertionError on report existence again | +| 3.1.0 | [#33994](https://github.com/airbytehq/airbyte/pull/33994) | Log more context information in CI. | +| 3.0.2 | [#33987](https://github.com/airbytehq/airbyte/pull/33987) | Fix type checking issue when running --help | +| 3.0.1 | [#33981](https://github.com/airbytehq/airbyte/pull/33981) | Fix issues with deploying dagster, pin pendulum version in dagster-cli install | +| 3.0.0 | [#33582](https://github.com/airbytehq/airbyte/pull/33582) | Upgrade to Dagger 0.9.5 | +| 2.14.3 | [#33964](https://github.com/airbytehq/airbyte/pull/33964) | Reintroduce mypy with fixes for AssertionError on publish and missing report URL on connector test commit status. | +| 2.14.2 | [#33954](https://github.com/airbytehq/airbyte/pull/33954) | Revert mypy changes | +| 2.14.1 | [#33956](https://github.com/airbytehq/airbyte/pull/33956) | Exclude pnpm lock files from auto-formatting | +| 2.14.0 | [#33941](https://github.com/airbytehq/airbyte/pull/33941) | Enable in-connector normalization in destination-postgres | +| 2.13.1 | [#33920](https://github.com/airbytehq/airbyte/pull/33920) | Report different sentry environments | +| 2.13.0 | [#33784](https://github.com/airbytehq/airbyte/pull/33784) | Make `airbyte-ci test` able to run any poetry command | +| 2.12.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Add upgrade CDK command | +| 2.11.0 | [#32188](https://github.com/airbytehq/airbyte/pull/32188) | Add -x option to connector test to allow for skipping steps | +| 2.10.12 | [#33419](https://github.com/airbytehq/airbyte/pull/33419) | Make ClickPipelineContext handle dagger logging. | +| 2.10.11 | [#33497](https://github.com/airbytehq/airbyte/pull/33497) | Consider nested .gitignore rules in format. | +| 2.10.10 | [#33449](https://github.com/airbytehq/airbyte/pull/33449) | Add generated metadata models to the default format ignore list. | +| 2.10.9 | [#33370](https://github.com/airbytehq/airbyte/pull/33370) | Fix bug that broke airbyte-ci test | +| 2.10.8 | [#33249](https://github.com/airbytehq/airbyte/pull/33249) | Exclude git ignored files from formatting. | +| 2.10.7 | [#33248](https://github.com/airbytehq/airbyte/pull/33248) | Fix bug which broke airbyte-ci connectors tests when optional DockerHub credentials env vars are not set. | +| 2.10.6 | [#33170](https://github.com/airbytehq/airbyte/pull/33170) | Remove Dagger logs from console output of `format`. | +| 2.10.5 | [#33097](https://github.com/airbytehq/airbyte/pull/33097) | Improve `format` performances, exit with 1 status code when `fix` changes files. | +| 2.10.4 | [#33206](https://github.com/airbytehq/airbyte/pull/33206) | Add "-y/--yes" Flag to allow preconfirmation of prompts | +| 2.10.3 | [#33080](https://github.com/airbytehq/airbyte/pull/33080) | Fix update failing due to SSL error on install. | +| 2.10.2 | [#33008](https://github.com/airbytehq/airbyte/pull/33008) | Fix local `connector build`. | +| 2.10.1 | [#32928](https://github.com/airbytehq/airbyte/pull/32928) | Fix BuildConnectorImages constructor. | +| 2.10.0 | [#32819](https://github.com/airbytehq/airbyte/pull/32819) | Add `--tag` option to connector build. | +| 2.9.0 | [#32816](https://github.com/airbytehq/airbyte/pull/32816) | Add `--architecture` option to connector build. | +| 2.8.1 | [#32999](https://github.com/airbytehq/airbyte/pull/32999) | Improve Java code formatting speed | +| 2.8.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Move pipx install to `airbyte-ci-dev`, and add auto-update feature targeting binary | +| 2.7.3 | [#32847](https://github.com/airbytehq/airbyte/pull/32847) | Improve --modified behaviour for pull requests. | +| 2.7.2 | [#32839](https://github.com/airbytehq/airbyte/pull/32839) | Revert changes in v2.7.1. | +| 2.7.1 | [#32806](https://github.com/airbytehq/airbyte/pull/32806) | Improve --modified behaviour for pull requests. | +| 2.7.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Merge airbyte-ci-internal into airbyte-ci | +| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | +| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | +| 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | +| 2.5.7 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | +| 2.5.6 | [#32139](https://github.com/airbytehq/airbyte/pull/32139) | Test coverage report on Python connector UnitTest. | +| 2.5.5 | [#32114](https://github.com/airbytehq/airbyte/pull/32114) | Create cache mount for `/var/lib/docker` to store images in `dind` context. | +| 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | +| 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | +| 2.5.2 | [#31871](https://github.com/airbytehq/airbyte/pull/31871) | Deactivate PR comments, add HTML report links to the PR status when its ready. | +| 2.5.1 | [#31774](https://github.com/airbytehq/airbyte/pull/31774) | Add a docker configuration check on `airbyte-ci` startup. | +| 2.5.0 | [#31766](https://github.com/airbytehq/airbyte/pull/31766) | Support local connectors secrets. | +| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | +| 2.3.1 | [#31748](https://github.com/airbytehq/airbyte/pull/31748) | Use AsyncClick library instead of base Click. | +| 2.3.0 | [#31699](https://github.com/airbytehq/airbyte/pull/31699) | Support optional concurrent CAT execution. | +| 2.2.6 | [#31752](https://github.com/airbytehq/airbyte/pull/31752) | Only authenticate when secrets are available. | +| 2.2.5 | [#31718](https://github.com/airbytehq/airbyte/pull/31718) | Authenticate the sidecar docker daemon to DockerHub. | +| 2.2.4 | [#31535](https://github.com/airbytehq/airbyte/pull/31535) | Improve gradle caching when building java connectors. | +| 2.2.3 | [#31688](https://github.com/airbytehq/airbyte/pull/31688) | Fix failing `CheckBaseImageUse` step when not running on PR. | +| 2.2.2 | [#31659](https://github.com/airbytehq/airbyte/pull/31659) | Support builds on x86_64 platform | +| 2.2.1 | [#31653](https://github.com/airbytehq/airbyte/pull/31653) | Fix CheckBaseImageIsUsed failing on non certified connectors. | +| 2.2.0 | [#30527](https://github.com/airbytehq/airbyte/pull/30527) | Add a new check for python connectors to make sure certified connectors use our base image. | +| 2.1.1 | [#31488](https://github.com/airbytehq/airbyte/pull/31488) | Improve `airbyte-ci` start time with Click Lazy load | +| 2.1.0 | [#31412](https://github.com/airbytehq/airbyte/pull/31412) | Run airbyte-ci from any where in airbyte project | +| 2.0.4 | [#31487](https://github.com/airbytehq/airbyte/pull/31487) | Allow for third party connector selections | +| 2.0.3 | [#31525](https://github.com/airbytehq/airbyte/pull/31525) | Refactor folder structure | +| 2.0.2 | [#31533](https://github.com/airbytehq/airbyte/pull/31533) | Pip cache volume by python version. | +| 2.0.1 | [#31545](https://github.com/airbytehq/airbyte/pull/31545) | Reword the changelog entry when using `migrate_to_base_image`. | +| 2.0.0 | [#31424](https://github.com/airbytehq/airbyte/pull/31424) | Remove `airbyte-ci connectors format` command. | +| 1.9.4 | [#31478](https://github.com/airbytehq/airbyte/pull/31478) | Fix running tests for connector-ops package. | +| 1.9.3 | [#31457](https://github.com/airbytehq/airbyte/pull/31457) | Improve the connector documentation for connectors migrated to our base image. | +| 1.9.2 | [#31426](https://github.com/airbytehq/airbyte/pull/31426) | Concurrent execution of java connectors tests. | +| 1.9.1 | [#31455](https://github.com/airbytehq/airbyte/pull/31455) | Fix `None` docker credentials on publish. | +| 1.9.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | +| 1.8.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | +| 1.7.2 | [#31343](https://github.com/airbytehq/airbyte/pull/31343) | Bind Pytest integration tests to a dockerhost. | +| 1.7.1 | [#31332](https://github.com/airbytehq/airbyte/pull/31332) | Disable Gradle step caching on source-postgres. | +| 1.7.0 | [#30526](https://github.com/airbytehq/airbyte/pull/30526) | Implement pre/post install hooks support. | +| 1.6.0 | [#30474](https://github.com/airbytehq/airbyte/pull/30474) | Test connector inside their containers. | +| 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | +| 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | +| 1.4.6 | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date | +| 1.4.5 | [#31133](https://github.com/airbytehq/airbyte/pull/31133) | Fix bug when building containers using `with_integration_base_java_and_normalization`. | +| 1.4.4 | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration. | +| 1.4.3 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Add --version and version check | +| 1.4.2 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement | +| 1.4.1 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants | +| 1.4.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add support for pyproject.toml as the prefered entry point for a connector package | +| 1.3.0 | [#30461](https://github.com/airbytehq/airbyte/pull/30461) | Add `--use-local-cdk` flag to all connectors commands | +| 1.2.3 | [#30477](https://github.com/airbytehq/airbyte/pull/30477) | Fix a test regression introduced the previous version. | +| 1.2.2 | [#30438](https://github.com/airbytehq/airbyte/pull/30438) | Add workaround to always stream logs properly with --is-local. | +| 1.2.1 | [#30384](https://github.com/airbytehq/airbyte/pull/30384) | Java connector test performance fixes. | +| 1.2.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add `--metadata-query` option to connectors command | +| 1.1.3 | [#30314](https://github.com/airbytehq/airbyte/pull/30314) | Stop patching gradle files to make them work with airbyte-ci. | +| 1.1.2 | [#30279](https://github.com/airbytehq/airbyte/pull/30279) | Fix correctness issues in layer caching by making atomic execution groupings | +| 1.1.1 | [#30252](https://github.com/airbytehq/airbyte/pull/30252) | Fix redundancies and broken logic in GradleTask, to speed up the CI runs. | +| 1.1.0 | [#29509](https://github.com/airbytehq/airbyte/pull/29509) | Refactor the airbyte-ci test command to run tests on any poetry package. | +| 1.0.0 | [#28000](https://github.com/airbytehq/airbyte/pull/29232) | Remove release stages in favor of support level from airbyte-ci. | +| 0.5.0 | [#28000](https://github.com/airbytehq/airbyte/pull/28000) | Run connector acceptance tests with dagger-in-dagger. | +| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | +| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | +| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | +| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | +| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | +| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | +| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | +| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | +| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | +| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | +| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | +| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | +| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | +| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | +| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | +| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | ## More info -This project is owned by the Connectors Operations team. -We share project updates and remaining stories before its release to production in this [EPIC](https://github.com/airbytehq/airbyte/issues/24403). +This project is owned by the Connectors Operations team. We share project updates and remaining +stories before its release to production in this +[EPIC](https://github.com/airbytehq/airbyte/issues/24403). # Troubleshooting @@ -648,13 +850,16 @@ make tools.airbyte-ci.install To fix this, you can either: -- Ensure that airbyte-ci is installed with pipx. Run `pipx list` to check if airbyte-ci is installed. +- Ensure that airbyte-ci is installed with pipx. Run `pipx list` to check if airbyte-ci is + installed. - Run `pipx ensurepath` to add the pipx binary directory to your PATH. -- Add the pipx binary directory to your PATH manually. The pipx binary directory is usually `~/.local/bin`. +- Add the pipx binary directory to your PATH manually. The pipx binary directory is usually + `~/.local/bin`. ### python3.10 not found -If you get the following error when running `pipx install --editable --force --python=python3.10 airbyte-ci/connectors/pipelines/`: +If you get the following error when running +`pipx install --editable --force --python=python3.10 airbyte-ci/connectors/pipelines/`: ```bash $ pipx install --editable --force --python=python3.10 airbyte-ci/connectors/pipelines/ @@ -666,13 +871,15 @@ It means that you don't have Python 3.10 installed on your system. To fix this, you can either: - Install Python 3.10 with pyenv. Run `pyenv install 3.10` to install the latest Python version. -- Install Python 3.10 with your system package manager. For instance, on Ubuntu you can run `sudo apt install python3.10`. -- Ensure that Python 3.10 is in your PATH. Run `which python3.10` to check if Python 3.10 is installed and in your PATH. +- Install Python 3.10 with your system package manager. For instance, on Ubuntu you can run + `sudo apt install python3.10`. +- Ensure that Python 3.10 is in your PATH. Run `which python3.10` to check if Python 3.10 is + installed and in your PATH. ### Any type of pipeline failure -First you should check that the version of the CLI you are using is the latest one. -You can check the version of the CLI with the `--version` option: +First you should check that the version of the CLI you are using is the latest one. You can check +the version of the CLI with the `--version` option: ```bash $ airbyte-ci --version @@ -685,7 +892,8 @@ and compare it with the version in the pyproject.toml file: $ cat airbyte-ci/connectors/pipelines/pyproject.toml | grep version ``` -If you get any type of pipeline failure, you can run the pipeline with the `--show-dagger-logs` option to get more information about the failure. +If you get any type of pipeline failure, you can run the pipeline with the `--show-dagger-logs` +option to get more information about the failure. ```bash $ airbyte-ci --show-dagger-logs connectors --name=source-pokeapi test diff --git a/airbyte-ci/connectors/pipelines/pipelines/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/__init__.py index 44a71b8f1b49..4b1a6ecc74dd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/__init__.py @@ -5,7 +5,7 @@ """The pipelines package.""" import logging import os - +from typing import Union from rich.logging import RichHandler from .helpers import sentry_utils @@ -15,16 +15,16 @@ logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) -logging_handlers = [RichHandler(rich_tracebacks=True)] -if "CI" in os.environ: - # RichHandler does not work great in the CI - logging_handlers = [logging.StreamHandler()] + +# RichHandler does not work great in the CI environment, so we use a StreamHandler instead +logging_handler: Union[RichHandler, logging.StreamHandler] = RichHandler(rich_tracebacks=True) if "CI" not in os.environ else logging.StreamHandler() + logging.basicConfig( level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", - handlers=logging_handlers, + handlers=[logging_handler], ) main_logger = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py index 2b312bba127b..712062361242 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py @@ -6,14 +6,15 @@ from __future__ import annotations import anyio -from connector_ops.utils import ConnectorLanguage +from connector_ops.utils import ConnectorLanguage # type: ignore from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors, python_connectors from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost, StepStatus from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report from pipelines.models.steps import StepResult + class NoBuildStepForLanguageError(Exception): pass @@ -32,7 +33,7 @@ async def run_connector_build(context: ConnectorContext) -> StepResult: return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context) -async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore, image_tag: str) -> ConnectorReport: +async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore, image_tag: str) -> Report: """Run a build pipeline for a single connector. Args: @@ -46,10 +47,11 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any async with semaphore: async with context: build_result = await run_connector_build(context) - per_platform_built_containers = build_result.output_artifact + per_platform_built_containers = build_result.output step_results.append(build_result) if context.is_local and build_result.status is StepStatus.SUCCESS: load_image_result = await LoadContainerToLocalDockerHost(context, per_platform_built_containers, image_tag).run() step_results.append(load_image_result) - context.report = ConnectorReport(context, step_results, name="BUILD RESULTS") - return context.report + report = ConnectorReport(context, step_results, name="BUILD RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py index 9ab4c063538e..818aa3163843 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py @@ -7,7 +7,7 @@ from types import ModuleType from typing import List, Optional -from connector_ops.utils import Connector +from connector_ops.utils import Connector # type: ignore from dagger import Container BUILD_CUSTOMIZATION_MODULE_NAME = "build_customization" @@ -21,11 +21,15 @@ def get_build_customization_module(connector: Connector) -> Optional[ModuleType] Optional[ModuleType]: The build_customization.py module if it exists, None otherwise. """ build_customization_spec_path = connector.code_directory / BUILD_CUSTOMIZATION_SPEC_NAME - if not build_customization_spec_path.exists(): - return None - build_customization_spec = importlib.util.spec_from_file_location( + + if not build_customization_spec_path.exists() or not (build_customization_spec := importlib.util.spec_from_file_location( f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path - ) + )): + return None + + if build_customization_spec.loader is None: + return None + build_customization_module = importlib.util.module_from_spec(build_customization_spec) build_customization_spec.loader.exec_module(build_customization_module) return build_customization_module @@ -41,9 +45,12 @@ def get_main_file_name(connector: Connector) -> str: str: The main file name. """ build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "MAIN_FILE_NAME"): - return build_customization_module.MAIN_FILE_NAME - return DEFAULT_MAIN_FILE_NAME + + return ( + build_customization_module.MAIN_FILE_NAME + if build_customization_module and hasattr(build_customization_module, "MAIN_FILE_NAME") + else DEFAULT_MAIN_FILE_NAME + ) def get_entrypoint(connector: Connector) -> List[str]: @@ -64,7 +71,7 @@ async def pre_install_hooks(connector: Connector, base_container: Container, log Container: The mutated base_container. """ build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "pre_connector_install"): + if build_customization_module and hasattr(build_customization_module, "pre_connector_install"): base_container = await build_customization_module.pre_connector_install(base_container) logger.info(f"Connector {connector.technical_name} pre install hook executed.") return base_container @@ -83,7 +90,7 @@ async def post_install_hooks(connector: Connector, connector_container: Containe Container: The mutated connector_container. """ build_customization_module = get_build_customization_module(connector) - if hasattr(build_customization_module, "post_connector_install"): + if build_customization_module and hasattr(build_customization_module, "post_connector_install"): connector_container = await build_customization_module.post_connector_install(connector_container) logger.info(f"Connector {connector.technical_name} post install hook executed.") return connector_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py index d9371fdddfee..f7ae65bffcfc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py @@ -4,49 +4,53 @@ from __future__ import annotations from abc import ABC -from typing import List, Optional, Tuple +from typing import TYPE_CHECKING -import docker +import docker # type: ignore from dagger import Container, ExecError, Platform, QueryError from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.utils import export_container_to_tarball from pipelines.models.steps import Step, StepResult, StepStatus +if TYPE_CHECKING: + from typing import Any class BuildConnectorImagesBase(Step, ABC): """ A step to build connector images for a set of platforms. """ + context: ConnectorContext + @property - def title(self): + def title(self) -> str: return f"Build {self.context.connector.technical_name} docker image for platform(s) {', '.join(self.build_platforms)}" def __init__(self, context: ConnectorContext) -> None: - self.build_platforms: List[Platform] = context.targeted_platforms + self.build_platforms = context.targeted_platforms super().__init__(context) - async def _run(self, *args) -> StepResult: + async def _run(self, *args: Any) -> StepResult: build_results_per_platform = {} for platform in self.build_platforms: try: connector = await self._build_connector(platform, *args) try: await connector.with_exec(["spec"]) - except ExecError: + except ExecError as e: return StepResult( - self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}." + step=self, status=StepStatus.FAILURE, stderr=str(e), stdout=f"Failed to run the spec command on the connector container for platform {platform}." ) build_results_per_platform[platform] = connector except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}") + return StepResult(step=self, status=StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}") success_message = ( f"The {self.context.connector.technical_name} docker image " f"was successfully built for platform(s) {', '.join(self.build_platforms)}" ) - return StepResult(self, StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform) + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=success_message, output=build_results_per_platform) - async def _build_connector(self, platform: Platform, *args) -> Container: + async def _build_connector(self, platform: Platform, *args: Any, **kwargs: Any) -> Container: """Implement the generation of the image for the platform and return the corresponding container. Returns: @@ -56,12 +60,14 @@ async def _build_connector(self, platform: Platform, *args) -> Container: class LoadContainerToLocalDockerHost(Step): - def __init__(self, context: ConnectorContext, containers: dict[Platform, Container], image_tag: Optional[str] = "dev") -> None: + context: ConnectorContext + + def __init__(self, context: ConnectorContext, containers: dict[Platform, Container], image_tag: str = "dev") -> None: super().__init__(context) self.image_tag = image_tag self.containers = containers - def _generate_dev_tag(self, platform: Platform, multi_platforms: bool): + def _generate_dev_tag(self, platform: Platform, multi_platforms: bool) -> str: """ When building for multiple platforms, we need to tag the image with the platform name. There's no way to locally build a multi-arch image, so we need to tag the image with the platform name when the user passed multiple architecture options. @@ -69,11 +75,11 @@ def _generate_dev_tag(self, platform: Platform, multi_platforms: bool): return f"{self.image_tag}-{platform.replace('/', '-')}" if multi_platforms else self.image_tag @property - def title(self): + def title(self) -> str: return f"Load {self.image_name}:{self.image_tag} to the local docker host." @property - def image_name(self) -> Tuple: + def image_name(self) -> str: return f"airbyte/{self.context.connector.technical_name}" async def _run(self) -> StepResult: @@ -83,8 +89,8 @@ async def _run(self) -> StepResult: _, exported_tar_path = await export_container_to_tarball(self.context, container, platform) if not exported_tar_path: return StepResult( - self, - StepStatus.FAILURE, + step=self, + status=StepStatus.FAILURE, stderr=f"Failed to export the connector image {self.image_name}:{self.image_tag} to a tarball.", ) try: @@ -98,7 +104,7 @@ async def _run(self) -> StepResult: loaded_images.append(full_image_name) except docker.errors.DockerException as e: return StepResult( - self, StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}" + step=self, status=StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}" ) - return StepResult(self, StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).") + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py index f8a4c7ed0d61..8d31bd5a714a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py @@ -36,10 +36,10 @@ async def _run(self, dist_dir: Directory) -> StepResult: if num_files == 0 else "More than one distribution tar file was built for the current java connector." ) - return StepResult(self, StepStatus.FAILURE, stderr=error_message) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=error_message) dist_tar = dist_dir.file(tar_files[0]) except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) return await super()._run(dist_tar) async def _build_connector(self, platform: Platform, dist_tar: File) -> Container: @@ -59,7 +59,7 @@ async def run_connector_build(context: ConnectorContext) -> StepResult: build_connector_tar_result = await BuildConnectorDistributionTar(context).run() if build_connector_tar_result.status is not StepStatus.SUCCESS: return build_connector_tar_result - dist_dir = await build_connector_tar_result.output_artifact.directory(dist_tar_directory_path(context)) + dist_dir = await build_connector_tar_result.output.directory(dist_tar_directory_path(context)) return await BuildConnectorImages(context).run(dist_dir) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py index 8461375732e1..7a3bae6d0ac3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py @@ -13,6 +13,8 @@ class BuildOrPullNormalization(Step): """A step to build or pull the normalization image for a connector according to the image name.""" + context: ConnectorContext + def __init__(self, context: ConnectorContext, normalization_image: str, build_platform: Platform) -> None: """Initialize the step to build or pull the normalization image. @@ -24,11 +26,14 @@ def __init__(self, context: ConnectorContext, normalization_image: str, build_pl self.build_platform = build_platform self.use_dev_normalization = normalization_image.endswith(":dev") self.normalization_image = normalization_image - self.title = f"Build {self.normalization_image}" if self.use_dev_normalization else f"Pull {self.normalization_image}" + + @property + def title(self) -> str: + return f"Build {self.normalization_image}" if self.use_dev_normalization else f"Pull {self.normalization_image}" async def _run(self) -> StepResult: if self.use_dev_normalization: build_normalization_container = normalization.with_normalization(self.context, self.build_platform) else: build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image) - return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container) + return StepResult(step=self, status=StepStatus.SUCCESS, output=build_normalization_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py index 8b8e8b9a1855..cbb5e0f9ce6d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -3,6 +3,8 @@ # +from typing import Any + from dagger import Container, Platform from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase @@ -17,9 +19,10 @@ class BuildConnectorImages(BuildConnectorImagesBase): A spec command is run on the container to validate it was built successfully. """ + context: ConnectorContext PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code" - async def _build_connector(self, platform: Platform): + async def _build_connector(self, platform: Platform, *args: Any) -> Container: if ( "connectorBuildOptions" in self.context.connector.metadata and "baseImage" in self.context.connector.metadata["connectorBuildOptions"] @@ -42,14 +45,15 @@ async def _create_builder_container(self, base_container: Container) -> Containe Returns: Container: The builder container, with installed dependencies. """ - ONLY_PYTHON_BUILD_FILES = ["setup.py", "requirements.txt", "pyproject.toml", "poetry.lock"] + ONLY_BUILD_FILES = ["pyproject.toml", "poetry.lock", "poetry.toml", "setup.py", "requirements.txt", "README.md"] + builder = await with_python_connector_installed( self.context, base_container, str(self.context.connector.code_directory), - include=ONLY_PYTHON_BUILD_FILES, + install_root_package=False, + include=ONLY_BUILD_FILES ) - return builder async def _build_from_base_image(self, platform: Platform) -> Container: @@ -74,10 +78,10 @@ async def _build_from_base_image(self, platform: Platform) -> Container: # copy python dependencies from builder to connector container customized_base.with_directory("/usr/local", builder.directory("/usr/local")) .with_workdir(self.PATH_TO_INTEGRATION_CODE) - .with_file(main_file_name, (await self.context.get_connector_dir(include=main_file_name)).file(main_file_name)) + .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name)) .with_directory( connector_snake_case_name, - (await self.context.get_connector_dir(include=connector_snake_case_name)).directory(connector_snake_case_name), + (await self.context.get_connector_dir(include=[connector_snake_case_name])).directory(connector_snake_case_name), ) .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) .with_entrypoint(entrypoint) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py index f387e5a98284..329da37639b3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py @@ -1,18 +1,21 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import datetime -from copy import deepcopy +from typing import TYPE_CHECKING import semver -from dagger import Container +from dagger import Container, Directory from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.airbyte_ci.metadata.pipeline import MetadataValidation from pipelines.helpers import git from pipelines.helpers.connectors import metadata_change_helpers from pipelines.models.steps import Step, StepResult, StepStatus +if TYPE_CHECKING: + from anyio import Semaphore + def get_bumped_version(version: str, bump_type: str) -> str: current_version = semver.VersionInfo.parse(version) @@ -28,6 +31,7 @@ def get_bumped_version(version: str, bump_type: str) -> str: class AddChangelogEntry(Step): + context: ConnectorContext title = "Add changelog entry" def __init__( @@ -37,7 +41,7 @@ def __init__( new_version: str, changelog_entry: str, pull_request_number: str, - ): + ) -> None: super().__init__(context) self.repo_dir = repo_dir self.new_version = new_version @@ -48,104 +52,106 @@ async def _run(self) -> StepResult: doc_path = self.context.connector.documentation_file_path if not doc_path.exists(): return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Connector does not have a documentation file.", - output_artifact=self.repo_dir, + output=self.repo_dir, ) try: updated_doc = self.add_changelog_entry(doc_path.read_text()) except Exception as e: return StepResult( - self, - StepStatus.FAILURE, + step=self, + status=StepStatus.FAILURE, stdout=f"Could not add changelog entry: {e}", - output_artifact=self.container_with_airbyte_repo, + output=self.repo_dir, ) - updated_repo_dir = self.repo_dir.with_new_file(str(doc_path), updated_doc) + updated_repo_dir = self.repo_dir.with_new_file(str(doc_path), contents=updated_doc) return StepResult( - self, - StepStatus.SUCCESS, + step=self, + status=StepStatus.SUCCESS, stdout=f"Added changelog entry to {doc_path}", - output_artifact=updated_repo_dir, + output=updated_repo_dir, ) - def find_line_index_for_new_entry(self, markdown_text) -> int: + def find_line_index_for_new_entry(self, markdown_text: str) -> int: lines = markdown_text.splitlines() for line_index, line in enumerate(lines): if "version" in line.lower() and "date" in line.lower() and "pull request" in line.lower() and "subject" in line.lower(): return line_index + 2 raise Exception("Could not find the changelog section table in the documentation file.") - def add_changelog_entry(self, og_doc_content) -> str: + def add_changelog_entry(self, og_doc_content: str) -> str: today = datetime.date.today().strftime("%Y-%m-%d") lines = og_doc_content.splitlines() line_index_for_new_entry = self.find_line_index_for_new_entry(og_doc_content) new_entry = f"| {self.new_version} | {today} | [{self.pull_request_number}](https://github.com/airbytehq/airbyte/pull/{self.pull_request_number}) | {self.changelog_entry} |" lines.insert(line_index_for_new_entry, new_entry) - return "\n".join(lines) + return "\n".join(lines) + "\n" class BumpDockerImageTagInMetadata(Step): + context: ConnectorContext title = "Upgrade the dockerImageTag to the latest version in metadata.yaml" def __init__( self, context: ConnectorContext, - repo_dir: Container, + repo_dir: Directory, new_version: str, - ): + ) -> None: super().__init__(context) self.repo_dir = repo_dir self.new_version = new_version @staticmethod - def get_metadata_with_bumped_version(previous_version: str, new_version: str, current_metadata: dict) -> dict: - updated_metadata = deepcopy(current_metadata) - updated_metadata["data"]["dockerImageTag"] = new_version - # Bump strict versions - if current_metadata["data"].get("registries", {}).get("cloud", {}).get("dockerImageTag") == previous_version: - updated_metadata["data"]["registries"]["cloud"]["dockerImageTag"] = new_version - return updated_metadata + def get_metadata_with_bumped_version(previous_version: str, new_version: str, metadata_str: str) -> str: + return metadata_str.replace("dockerImageTag: " + previous_version, "dockerImageTag: " + new_version) async def _run(self) -> StepResult: metadata_path = self.context.connector.metadata_file_path current_metadata = await metadata_change_helpers.get_current_metadata(self.repo_dir, metadata_path) + current_metadata_str = await metadata_change_helpers.get_current_metadata_str(self.repo_dir, metadata_path) current_version = metadata_change_helpers.get_current_version(current_metadata) if current_version is None: return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Can't retrieve the connector current version.", - output_artifact=self.repo_dir, + output=self.repo_dir, ) - updated_metadata = self.get_metadata_with_bumped_version(current_version, self.new_version, current_metadata) - repo_dir_with_updated_metadata = metadata_change_helpers.get_repo_dir_with_updated_metadata( - self.repo_dir, metadata_path, updated_metadata + updated_metadata_str = self.get_metadata_with_bumped_version(current_version, self.new_version, current_metadata_str) + repo_dir_with_updated_metadata = metadata_change_helpers.get_repo_dir_with_updated_metadata_str( + self.repo_dir, metadata_path, updated_metadata_str ) + metadata_validation_results = await MetadataValidation(self.context).run() + # Exit early if the metadata file is invalid. + if metadata_validation_results.status is not StepStatus.SUCCESS: + return metadata_validation_results + return StepResult( - self, - StepStatus.SUCCESS, + step=self, + status=StepStatus.SUCCESS, stdout=f"Updated dockerImageTag from {current_version} to {self.new_version} in {metadata_path}", - output_artifact=repo_dir_with_updated_metadata, + output=repo_dir_with_updated_metadata, ) async def run_connector_version_bump_pipeline( context: ConnectorContext, - semaphore, + semaphore: "Semaphore", bump_type: str, changelog_entry: str, pull_request_number: str, -) -> ConnectorReport: +) -> Report: """Run a pipeline to upgrade for a single connector. Args: context (ConnectorContext): The initialized connector context. Returns: - ConnectorReport: The reports holding the base image version upgrade results. + Report: The reports holding the base image version upgrade results. """ async with semaphore: steps_results = [] @@ -158,7 +164,7 @@ async def run_connector_version_bump_pipeline( new_version, ) update_docker_image_tag_in_metadata_result = await update_docker_image_tag_in_metadata.run() - repo_dir_with_updated_metadata = update_docker_image_tag_in_metadata_result.output_artifact + repo_dir_with_updated_metadata = update_docker_image_tag_in_metadata_result.output steps_results.append(update_docker_image_tag_in_metadata_result) add_changelog_entry = AddChangelogEntry( @@ -170,7 +176,8 @@ async def run_connector_version_bump_pipeline( ) add_changelog_entry_result = await add_changelog_entry.run() steps_results.append(add_changelog_entry_result) - final_repo_dir = add_changelog_entry_result.output_artifact + final_repo_dir = add_changelog_entry_result.output await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) - context.report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS") - return context.report + report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index e9188a35d6b2..6bda567160fd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -7,13 +7,12 @@ from typing import List, Optional, Set, Tuple import asyncclick as click -from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo +from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore from pipelines import main_logger from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup -from pipelines.consts import CIContext from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors -from pipelines.helpers.git import get_modified_files_in_branch, get_modified_files_in_commit +from pipelines.helpers.git import get_modified_files from pipelines.helpers.utils import transform_strs_to_paths ALL_CONNECTORS = get_all_connectors_in_repo() @@ -93,7 +92,7 @@ def get_selected_connectors_with_modified_files( return selected_connectors_with_modified_files -def validate_environment(is_local: bool): +def validate_environment(is_local: bool) -> None: """Check if the required environment variables exist.""" if is_local: if not Path(".git").is_dir(): @@ -153,6 +152,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: "bump_version": "pipelines.airbyte_ci.connectors.bump_version.commands.bump_version", "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image", "upgrade_base_image": "pipelines.airbyte_ci.connectors.upgrade_base_image.commands.upgrade_base_image", + "upgrade_cdk": "pipelines.airbyte_ci.connectors.upgrade_cdk.commands.bump_version", }, ) @click.option( @@ -235,7 +235,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: @click_ignore_unused_kwargs async def connectors( ctx: click.Context, -): +) -> None: """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) @@ -262,18 +262,3 @@ async def connectors( ctx.obj["enable_dependency_scanning"], ) log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) - - -async def get_modified_files(git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext) -> Set[str]: - """Get the list of modified files in the current git branch. - If the current branch is master, it will return the list of modified files in the head commit. - The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge. - Pipelines like "publish on merge" are triggered on each new commit on master. - - If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff. - If the current branch is not master, it will return the list of modified files in the current branch. - This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event. - """ - if ci_context is CIContext.MASTER or (ci_context is CIContext.MANUAL and git_branch == "master"): - return await get_modified_files_in_commit(git_branch, git_revision, is_local) - return await get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py new file mode 100644 index 000000000000..34f0ce115811 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from enum import Enum + + +class CONNECTOR_TEST_STEP_ID(str, Enum): + """ + An enum for the different step ids of the connector test pipeline. + """ + + ACCEPTANCE = "acceptance" + BUILD_NORMALIZATION = "build_normalization" + BUILD_TAR = "build_tar" + BUILD = "build" + CHECK_BASE_IMAGE = "check_base_image" + CHECK_PYTHON_REGISTRY_PUBLISH_CONFIGURATION = "check_python_registry_publish_configuration" + INTEGRATION = "integration" + AIRBYTE_LIB_VALIDATION = "airbyte_lib_validation" + METADATA_VALIDATION = "metadata_validation" + QA_CHECKS = "qa_checks" + UNIT = "unit" + VERSION_FOLLOW_CHECK = "version_follow_check" + VERSION_INC_CHECK = "version_inc_check" + TEST_ORCHESTRATOR = "test_orchestrator" + DEPLOY_ORCHESTRATOR = "deploy_orchestrator" + + def __str__(self) -> str: + return self.value diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 3553cafa8ac1..dff4f9b2a736 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -4,11 +4,13 @@ """Module declaring context related classes.""" +from __future__ import annotations + from datetime import datetime from types import TracebackType -from typing import Iterable, Optional +from typing import TYPE_CHECKING -import yaml +import yaml # type: ignore from anyio import Path from asyncer import asyncify from dagger import Directory, Platform, Secret @@ -17,11 +19,16 @@ from pipelines.consts import BUILD_PLATFORMS from pipelines.dagger.actions import secrets from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles +from pipelines.helpers.execution.run_steps import RunStepOptions from pipelines.helpers.github import update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook from pipelines.helpers.utils import METADATA_FILE_NAME from pipelines.models.contexts.pipeline_context import PipelineContext +if TYPE_CHECKING: + from pathlib import Path as NativePath + from typing import Dict, FrozenSet, List, Optional, Sequence + class ConnectorContext(PipelineContext): """The connector context is used to store configuration for a specific connector pipeline run.""" @@ -33,25 +40,23 @@ def __init__( pipeline_name: str, connector: ConnectorWithModifiedFiles, is_local: bool, - git_branch: bool, - git_revision: bool, + git_branch: str, + git_revision: str, report_output_prefix: str, use_remote_secrets: bool = True, ci_report_bucket: Optional[str] = None, ci_gcs_credentials: Optional[str] = None, ci_git_user: Optional[str] = None, ci_github_access_token: Optional[str] = None, - connector_acceptance_test_image: Optional[str] = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE, + connector_acceptance_test_image: str = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, slack_webhook: Optional[str] = None, reporting_slack_channel: Optional[str] = None, - pull_request: PullRequest = None, + pull_request: Optional[PullRequest.PullRequest] = None, should_save_report: bool = True, - fail_fast: bool = False, - fast_tests_only: bool = False, code_tests_only: bool = False, use_local_cdk: bool = False, use_host_gradle_dist_tar: bool = False, @@ -61,8 +66,9 @@ def __init__( s3_build_cache_access_key_id: Optional[str] = None, s3_build_cache_secret_key: Optional[str] = None, concurrent_cat: Optional[bool] = False, - targeted_platforms: Optional[Iterable[Platform]] = BUILD_PLATFORMS, - ): + run_step_options: RunStepOptions = RunStepOptions(), + targeted_platforms: Sequence[Platform] = BUILD_PLATFORMS, + ) -> None: """Initialize a connector context. Args: @@ -80,8 +86,6 @@ def __init__( slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None. reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None. pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. - fail_fast (bool, optional): Whether to fail fast. Defaults to False. - fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False. enable_report_auto_open (bool, optional): Open HTML report in browser window. Defaults to True. @@ -97,13 +101,10 @@ def __init__( self.connector = connector self.use_remote_secrets = use_remote_secrets self.connector_acceptance_test_image = connector_acceptance_test_image - self.report_output_prefix = report_output_prefix - self._secrets_dir = None - self._updated_secrets_dir = None - self.cdk_version = None + self._secrets_dir: Optional[Directory] = None + self._updated_secrets_dir: Optional[Directory] = None + self.cdk_version: Optional[str] = None self.should_save_report = should_save_report - self.fail_fast = fail_fast - self.fast_tests_only = fast_tests_only self.code_tests_only = code_tests_only self.use_local_cdk = use_local_cdk self.use_host_gradle_dist_tar = use_host_gradle_dist_tar @@ -113,6 +114,7 @@ def __init__( self.s3_build_cache_access_key_id = s3_build_cache_access_key_id self.s3_build_cache_secret_key = s3_build_cache_secret_key self.concurrent_cat = concurrent_cat + self._connector_secrets: Optional[Dict[str, Secret]] = None self.targeted_platforms = targeted_platforms super().__init__( @@ -120,6 +122,7 @@ def __init__( is_local=is_local, git_branch=git_branch, git_revision=git_revision, + report_output_prefix=report_output_prefix, gha_workflow_run_url=gha_workflow_run_url, dagger_logs_url=dagger_logs_url, pipeline_start_timestamp=pipeline_start_timestamp, @@ -131,6 +134,7 @@ def __init__( ci_gcs_credentials=ci_gcs_credentials, ci_git_user=ci_git_user, ci_github_access_token=ci_github_access_token, + run_step_options=run_step_options, enable_report_auto_open=enable_report_auto_open, ) @@ -147,31 +151,31 @@ def s3_build_cache_secret_key_secret(self) -> Optional[Secret]: return None @property - def modified_files(self): + def modified_files(self) -> FrozenSet[NativePath]: return self.connector.modified_files @property - def secrets_dir(self) -> Directory: # noqa D102 + def secrets_dir(self) -> Optional[Directory]: return self._secrets_dir @secrets_dir.setter - def secrets_dir(self, secrets_dir: Directory): # noqa D102 + def secrets_dir(self, secrets_dir: Directory) -> None: self._secrets_dir = secrets_dir @property - def updated_secrets_dir(self) -> Directory: # noqa D102 + def updated_secrets_dir(self) -> Optional[Directory]: return self._updated_secrets_dir @updated_secrets_dir.setter - def updated_secrets_dir(self, updated_secrets_dir: Directory): # noqa D102 + def updated_secrets_dir(self, updated_secrets_dir: Directory) -> None: self._updated_secrets_dir = updated_secrets_dir @property - def connector_acceptance_test_source_dir(self) -> Directory: # noqa D102 + def connector_acceptance_test_source_dir(self) -> Directory: return self.get_repo_dir("airbyte-integrations/bases/connector-acceptance-test") @property - def should_save_updated_secrets(self) -> bool: # noqa D102 + def should_save_updated_secrets(self) -> bool: return self.use_remote_secrets and self.updated_secrets_dir is not None @property @@ -210,7 +214,12 @@ def docker_hub_password_secret(self) -> Optional[Secret]: return None return self.dagger_client.set_secret("docker_hub_password", self.docker_hub_password) - async def get_connector_dir(self, exclude=None, include=None) -> Directory: + async def get_connector_secrets(self) -> Dict[str, Secret]: + if self._connector_secrets is None: + self._connector_secrets = await secrets.get_connector_secrets(self) + return self._connector_secrets + + async def get_connector_dir(self, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: """Get the connector under test source code directory. Args: @@ -259,7 +268,8 @@ async def __aexit__( await asyncify(update_commit_status_check)(**self.github_commit_status) if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) # type: ignore # Supress the exception if any return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 76c50648eb2e..5c5b97ca5621 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -3,7 +3,7 @@ # import asyncclick as click -from connector_ops.utils import console +from connector_ops.utils import console # type: ignore from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from rich.table import Table from rich.text import Text @@ -13,7 +13,7 @@ @click.pass_context async def list_connectors( ctx: click.Context, -): +) -> bool: selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) table = Table(title=f"{len(selected_connectors)} selected connectors") table.add_column("Modified") @@ -26,15 +26,15 @@ async def list_connectors( for connector in selected_connectors: modified = "X" if connector.modified_files else "" connector_name = Text(connector.technical_name) - language = Text(connector.language.value) if connector.language else "N/A" + language: Text = Text(connector.language.value) if connector.language else Text("N/A") try: - support_level = Text(connector.support_level) + support_level: Text = Text(connector.support_level) except Exception: - support_level = "N/A" + support_level = Text("N/A") try: - version = Text(connector.version) + version: Text = Text(connector.version) except Exception: - version = "N/A" + version = Text("N/A") folder = Text(str(connector.code_directory)) table.add_row(modified, connector_name, language, support_level, version, folder) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py index 291d6a52a281..cb1f6d357d3a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py @@ -4,21 +4,28 @@ import textwrap from copy import deepcopy -from typing import Optional +from typing import TYPE_CHECKING -from base_images import version_registry -from connector_ops.utils import ConnectorLanguage +from base_images import version_registry # type: ignore +from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Directory from jinja2 import Template from pipelines.airbyte_ci.connectors.bump_version.pipeline import AddChangelogEntry, BumpDockerImageTagInMetadata, get_bumped_version from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext -from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report from pipelines.helpers import git from pipelines.helpers.connectors import metadata_change_helpers from pipelines.models.steps import Step, StepResult, StepStatus +if TYPE_CHECKING: + from typing import Optional + + from anyio import Semaphore + class UpgradeBaseImageMetadata(Step): + context: ConnectorContext + title = "Upgrade the base image to the latest version in metadata.yaml" def __init__( @@ -26,12 +33,12 @@ def __init__( context: ConnectorContext, repo_dir: Directory, set_if_not_exists: bool = True, - ): + ) -> None: super().__init__(context) self.repo_dir = repo_dir self.set_if_not_exists = set_if_not_exists - async def get_latest_base_image_address(self) -> Optional[str]: + async def get_latest_base_image_address(self) -> "Optional[str]": try: version_registry_for_language = await version_registry.get_registry_for_language( self.dagger_client, self.context.connector.language, (self.context.docker_hub_username, self.context.docker_hub_password) @@ -54,10 +61,10 @@ async def _run(self) -> StepResult: latest_base_image_address = await self.get_latest_base_image_address() if latest_base_image_address is None: return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Could not find a base image for this connector language.", - output_artifact=self.repo_dir, + output=self.repo_dir, ) metadata_path = self.context.connector.metadata_file_path @@ -66,62 +73,66 @@ async def _run(self) -> StepResult: if current_base_image_address is None and not self.set_if_not_exists: return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Connector does not have a base image metadata field.", - output_artifact=self.repo_dir, + output=self.repo_dir, ) if current_base_image_address == latest_base_image_address: return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Connector already uses latest base image", - output_artifact=self.repo_dir, + output=self.repo_dir, ) updated_metadata = self.update_base_image_in_metadata(current_metadata, latest_base_image_address) updated_repo_dir = metadata_change_helpers.get_repo_dir_with_updated_metadata(self.repo_dir, metadata_path, updated_metadata) return StepResult( - self, - StepStatus.SUCCESS, + step=self, + status=StepStatus.SUCCESS, stdout=f"Updated base image to {latest_base_image_address} in {metadata_path}", - output_artifact=updated_repo_dir, + output=updated_repo_dir, ) class DeleteConnectorFile(Step): + context: ConnectorContext + def __init__( self, context: ConnectorContext, file_to_delete: str, - ): + ) -> None: super().__init__(context) self.file_to_delete = file_to_delete @property - def title(self): + def title(self) -> str: return f"Delete {self.file_to_delete}" async def _run(self) -> StepResult: file_to_delete_path = self.context.connector.code_directory / self.file_to_delete if not file_to_delete_path.exists(): return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout=f"Connector does not have a {self.file_to_delete}", ) # As this is a deletion of a file, this has to happen on the host fs # Deleting the file in a Directory container would not work because the directory.export method would not export the deleted file from the Directory back to host. file_to_delete_path.unlink() return StepResult( - self, - StepStatus.SUCCESS, + step=self, + status=StepStatus.SUCCESS, stdout=f"Deleted {file_to_delete_path}", ) class AddBuildInstructionsToReadme(Step): + context: ConnectorContext + title = "Add build instructions to README.md" def __init__(self, context: PipelineContext, repo_dir: Directory) -> None: @@ -132,30 +143,30 @@ async def _run(self) -> StepResult: readme_path = self.context.connector.code_directory / "README.md" if not readme_path.exists(): return StepResult( - self, - StepStatus.SKIPPED, + step=self, + status=StepStatus.SKIPPED, stdout="Connector does not have a documentation file.", - output_artifact=self.repo_dir, + output=self.repo_dir, ) current_readme = await (await self.context.get_connector_dir(include=["README.md"])).file("README.md").contents() try: updated_readme = self.add_build_instructions(current_readme) except Exception as e: return StepResult( - self, - StepStatus.FAILURE, + step=self, + status=StepStatus.FAILURE, stdout=str(e), - output_artifact=self.repo_dir, + output=self.repo_dir, ) - updated_repo_dir = await self.repo_dir.with_new_file(str(readme_path), updated_readme) + updated_repo_dir = await self.repo_dir.with_new_file(str(readme_path), contents=updated_readme) return StepResult( - self, - StepStatus.SUCCESS, + step=self, + status=StepStatus.SUCCESS, stdout=f"Added build instructions to {readme_path}", - output_artifact=updated_repo_dir, + output=updated_repo_dir, ) - def add_build_instructions(self, og_doc_content) -> str: + def add_build_instructions(self, og_doc_content: str) -> str: build_instructions_template = Template( textwrap.dedent( @@ -252,7 +263,7 @@ async def post_connector_install(connector_container: Container) -> Container: return new_doc -async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, semaphore, set_if_not_exists: bool) -> ConnectorReport: +async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, semaphore: "Semaphore", set_if_not_exists: bool) -> Report: """Run a pipeline to upgrade for a single connector to use our base image.""" async with semaphore: steps_results = [] @@ -265,13 +276,16 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s ) update_base_image_in_metadata_result = await update_base_image_in_metadata.run() steps_results.append(update_base_image_in_metadata_result) - final_repo_dir = update_base_image_in_metadata_result.output_artifact + final_repo_dir = update_base_image_in_metadata_result.output await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) - context.report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS") - return context.report + report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS") + context.report = report + return report -async def run_connector_migration_to_base_image_pipeline(context: ConnectorContext, semaphore, pull_request_number: str): +async def run_connector_migration_to_base_image_pipeline( + context: ConnectorContext, semaphore: "Semaphore", pull_request_number: str +) -> Report: async with semaphore: steps_results = [] async with context: @@ -310,7 +324,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte new_version = get_bumped_version(context.connector.version, "patch") bump_version_in_metadata = BumpDockerImageTagInMetadata( context, - update_base_image_in_metadata_result.output_artifact, + update_base_image_in_metadata_result.output, new_version, ) bump_version_in_metadata_result = await bump_version_in_metadata.run() @@ -319,7 +333,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte # ADD CHANGELOG ENTRY add_changelog_entry = AddChangelogEntry( context, - bump_version_in_metadata_result.output_artifact, + bump_version_in_metadata_result.output, new_version, "Base image migration: remove Dockerfile and use the python-connector-base image", pull_request_number, @@ -330,14 +344,14 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte # UPDATE DOC add_build_instructions_to_doc = AddBuildInstructionsToReadme( context, - add_changelog_entry_result.output_artifact, + add_changelog_entry_result.output, ) add_build_instructions_to_doc_results = await add_build_instructions_to_doc.run() steps_results.append(add_build_instructions_to_doc_results) # EXPORT MODIFIED FILES BACK TO HOST - final_repo_dir = add_build_instructions_to_doc_results.output_artifact + final_repo_dir = add_build_instructions_to_doc_results.output await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) - - context.report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS") - return context.report + report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py index e2c60d818eda..b4055029a74f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py @@ -3,23 +3,28 @@ # """This module groups the functions to run full pipelines for connector testing.""" +from __future__ import annotations import sys from pathlib import Path -from typing import Callable, List, Optional +from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union import anyio import dagger -from connector_ops.utils import ConnectorLanguage +from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Config from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.steps.no_op import NoOpStep -from pipelines.consts import DOCKER_CLI_IMAGE, DOCKER_HOST_NAME, DOCKER_HOST_PORT, ContextState +from pipelines.consts import ContextState from pipelines.dagger.actions.system import docker from pipelines.helpers.utils import create_and_open_file from pipelines.models.reports import Report from pipelines.models.steps import StepResult, StepStatus +if TYPE_CHECKING: + from pipelines.models.contexts.pipeline_context import PipelineContext + GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI" GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests" @@ -30,7 +35,7 @@ } -async def context_to_step_result(context: ConnectorContext) -> StepResult: +async def context_to_step_result(context: PipelineContext) -> StepResult: if context.state == ContextState.SUCCESSFUL: return await NoOpStep(context, StepStatus.SUCCESS).run() @@ -45,14 +50,16 @@ async def context_to_step_result(context: ConnectorContext) -> StepResult: # HACK: This is to avoid wrapping the whole pipeline in a dagger pipeline to avoid instability just prior to launch # TODO (ben): Refactor run_connectors_pipelines to wrap the whole pipeline in a dagger pipeline once Steps are refactored -async def run_report_complete_pipeline(dagger_client: dagger.Client, contexts: List[ConnectorContext]) -> List[ConnectorContext]: +async def run_report_complete_pipeline( + dagger_client: dagger.Client, contexts: List[ConnectorContext] | List[PublishConnectorContext] | List[PipelineContext] +) -> None: """Create and Save a report representing the run of the encompassing pipeline. This is to denote when the pipeline is complete, useful for long running pipelines like nightlies. """ if not contexts: - return [] + return # Repurpose the first context to be the pipeline upload context to preserve timestamps first_connector_context = contexts[0] @@ -70,28 +77,23 @@ async def run_report_complete_pipeline(dagger_client: dagger.Client, contexts: L filename="complete", ) - return await report.save() + await report.save() async def run_connectors_pipelines( - contexts: List[ConnectorContext], + contexts: Union[List[ConnectorContext], List[PublishConnectorContext]], connector_pipeline: Callable, pipeline_name: str, concurrency: int, dagger_logs_path: Optional[Path], execute_timeout: Optional[int], - *args, -) -> List[ConnectorContext]: + *args: Any, +) -> List[ConnectorContext] | List[PublishConnectorContext]: """Run a connector pipeline for all the connector contexts.""" default_connectors_semaphore = anyio.Semaphore(concurrency) dagger_logs_output = sys.stderr if not dagger_logs_path else create_and_open_file(dagger_logs_path) async with dagger.Connection(Config(log_output=dagger_logs_output, execute_timeout=execute_timeout)) as dagger_client: - - # HACK: This is to get a long running dockerd service to be shared across all the connectors pipelines - # Using the "normal" service binding leads to restart of dockerd during pipeline run that can cause corrupted docker state - # See https://github.com/airbytehq/airbyte/issues/27233 - docker_hub_username = contexts[0].docker_hub_username docker_hub_password = contexts[0].docker_hub_password @@ -102,27 +104,21 @@ async def run_connectors_pipelines( else: dockerd_service = docker.with_global_dockerd_service(dagger_client) - async with anyio.create_task_group() as tg_main: - tg_main.start_soon(dockerd_service.sync) - await ( # Wait for the docker service to be ready - dagger_client.container() - .from_(DOCKER_CLI_IMAGE) - .with_env_variable("DOCKER_HOST", f"tcp://{DOCKER_HOST_NAME}:{DOCKER_HOST_PORT}") - .with_service_binding(DOCKER_HOST_NAME, dockerd_service) - .with_exec(["docker", "info"]) - ) - async with anyio.create_task_group() as tg_connectors: - for context in contexts: - context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") - context.dockerd_service = dockerd_service - tg_connectors.start_soon( - connector_pipeline, - context, - CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), - *args, - ) - # When the connectors pipelines are done, we can stop the dockerd service - tg_main.cancel_scope.cancel() + await dockerd_service.start() + + async with anyio.create_task_group() as tg_connectors: + for context in contexts: + context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") + context.dockerd_service = dockerd_service + tg_connectors.start_soon( + connector_pipeline, + context, + CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), + *args, + ) + + # When the connectors pipelines are done, we can stop the dockerd service + await dockerd_service.stop() await run_report_complete_pipeline(dagger_client, contexts) return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 9b525d787c3e..fc34936e248b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -2,18 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import asyncclick as click from pipelines import main_logger from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.publish.pipeline import reorder_contexts, run_connector_publish_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import ContextState +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState from pipelines.helpers.utils import fail_if_missing_docker_hub_creds @click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.") +@click_ci_requirements_option() @click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool) @click.option( "--spec-cache-gcs-credentials", @@ -56,6 +59,26 @@ envvar="SLACK_CHANNEL", default="#connector-publish-updates", ) +@click.option( + "--python-registry-token", + help="Access token for python registry", + type=click.STRING, + envvar="PYTHON_REGISTRY_TOKEN", +) +@click.option( + "--python-registry-url", + help="Which python registry url to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + envvar="PYTHON_REGISTRY_URL", +) +@click.option( + "--python-registry-check-url", + help="Which url to check whether a certain version is published already. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/pypi/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, + envvar="PYTHON_REGISTRY_CHECK_URL", +) @click.pass_context async def publish( ctx: click.Context, @@ -66,7 +89,10 @@ async def publish( metadata_service_gcs_credentials: str, slack_webhook: str, slack_channel: str, -): + python_registry_token: str, + python_registry_url: str, + python_registry_check_url: str, +) -> bool: ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name ctx.obj["metadata_service_bucket_name"] = metadata_service_bucket_name @@ -106,15 +132,17 @@ async def publish( s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), use_local_cdk=ctx.obj.get("use_local_cdk"), + python_registry_token=python_registry_token, + python_registry_url=python_registry_url, + python_registry_check_url=python_registry_check_url, ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] ) - main_logger.warn("Concurrency is forced to 1. For stability reasons we disable parallel publish pipelines.") ctx.obj["concurrency"] = 1 - publish_connector_contexts = await run_connectors_pipelines( + ran_publish_connector_contexts = await run_connectors_pipelines( publish_connector_contexts, run_connector_publish_pipeline, "Publishing connectors", @@ -122,4 +150,4 @@ async def publish( ctx.obj["dagger_logs_path"], ctx.obj["execute_timeout"], ) - return all(context.state is ContextState.SUCCESSFUL for context in publish_connector_contexts) + return all(context.state is ContextState.SUCCESSFUL for context in ran_publish_connector_contexts) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 633c19e17b3f..57473eee215b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -6,6 +6,7 @@ from typing import Optional +import asyncclick as click from dagger import Secret from github import PullRequest from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -16,6 +17,9 @@ class PublishConnectorContext(ConnectorContext): + docker_hub_username_secret: Secret + docker_hub_password_secret: Secret + def __init__( self, connector: ConnectorWithModifiedFiles, @@ -31,23 +35,29 @@ def __init__( ci_report_bucket: str, report_output_prefix: str, is_local: bool, - git_branch: bool, - git_revision: bool, + git_branch: str, + git_revision: str, + python_registry_url: str, + python_registry_check_url: str, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, - ci_gcs_credentials: str = None, - pull_request: PullRequest = None, + ci_gcs_credentials: Optional[str] = None, + pull_request: Optional[PullRequest.PullRequest] = None, s3_build_cache_access_key_id: Optional[str] = None, s3_build_cache_secret_key: Optional[str] = None, - use_local_cdk: Optional[bool] = False, - ): + use_local_cdk: bool = False, + python_registry_token: Optional[str] = None, + ) -> None: self.pre_release = pre_release self.spec_cache_bucket_name = spec_cache_bucket_name self.metadata_bucket_name = metadata_bucket_name self.spec_cache_gcs_credentials = sanitize_gcs_credentials(spec_cache_gcs_credentials) self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials) + self.python_registry_token = python_registry_token + self.python_registry_url = python_registry_url + self.python_registry_check_url = python_registry_check_url pipeline_name = f"Publish {connector.technical_name}" pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name @@ -86,15 +96,20 @@ def spec_cache_gcs_credentials_secret(self) -> Secret: return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) @property - def docker_image_tag(self): + def pre_release_suffix(self) -> str: + return self.git_revision[:10] + + @property + def docker_image_tag(self) -> str: # get the docker image tag from the parent class metadata_tag = super().docker_image_tag if self.pre_release: - return f"{metadata_tag}-dev.{self.git_revision[:10]}" + return f"{metadata_tag}-dev.{self.pre_release_suffix}" else: return metadata_tag def create_slack_message(self) -> str: + docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" message = f"*Publish <{docker_hub_url}|{self.docker_image}>*\n" if self.is_ci: @@ -115,7 +130,8 @@ def create_slack_message(self) -> str: message += "🔴" message += f" {self.state.value['description']}\n" if self.state is ContextState.SUCCESSFUL: + assert self.report is not None, "Report should be set when state is successful" message += f"⏲️ Run duration: {format_duration(self.report.run_duration)}\n" if self.state is ContextState.FAILURE: - message += "\ncc. " # @dev-connector-ops + message += "\ncc. " return message diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 2a5908bdb150..88c917211946 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -7,15 +7,17 @@ from typing import List, Tuple import anyio -from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification +from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification # type: ignore from dagger import Container, ExecError, File, ImageLayerCompression, QueryError from pipelines import consts from pipelines.airbyte_ci.connectors.build_image import steps from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker +from pipelines.helpers.pip import is_package_published from pipelines.models.steps import Step, StepResult, StepStatus from pydantic import ValidationError @@ -25,6 +27,7 @@ class InvalidSpecOutputError(Exception): class CheckConnectorImageDoesNotExist(Step): + context: PublishConnectorContext title = "Check if the connector docker image does not exist on the registry." async def _run(self) -> StepResult: @@ -40,22 +43,45 @@ async def _run(self) -> StepResult: crane_ls_stdout = await crane_ls.stdout() except ExecError as e: if "NAME_UNKNOWN" in e.stderr: - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"The docker repository {docker_repository} does not exist.") + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"The docker repository {docker_repository} does not exist.") else: - return StepResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout) else: # The docker repo exists and ls was successful existing_tags = crane_ls_stdout.split("\n") docker_tag_already_exists = docker_tag in existing_tags if docker_tag_already_exists: - return StepResult(self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image} already exists.") - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") + return StepResult(step=self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image} already exists.") + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") + + +class CheckPythonRegistryPackageDoesNotExist(Step): + context: PythonRegistryPublishContext + title = "Check if the connector is published on python registry" + + async def _run(self) -> StepResult: + is_published = is_package_published( + self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry_check_url + ) + if is_published: + return StepResult( + step=self, + status=StepStatus.SKIPPED, + stderr=f"{self.context.package_metadata.name} already exists in version {self.context.package_metadata.version}.", + ) + else: + return StepResult( + step=self, + status=StepStatus.SUCCESS, + stdout=f"{self.context.package_metadata.name} does not exist in version {self.context.package_metadata.version}.", + ) class PushConnectorImageToRegistry(Step): + context: PublishConnectorContext title = "Push connector image to registry" @property - def latest_docker_image_name(self): + def latest_docker_image_name(self) -> str: return f"{self.context.docker_repository}:latest" async def _run(self, built_containers_per_platform: List[Container], attempts: int = 3) -> StepResult: @@ -71,17 +97,18 @@ async def _run(self, built_containers_per_platform: List[Container], attempts: i platform_variants=built_containers_per_platform[1:], forced_compression=ImageLayerCompression.Gzip, ) - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"Published {image_ref}") + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"Published {image_ref}") except QueryError as e: if attempts > 0: self.context.logger.error(str(e)) self.context.logger.warn(f"Failed to publish {self.context.docker_image}. Retrying. {attempts} attempts left.") await anyio.sleep(5) return await self._run(built_containers_per_platform, attempts - 1) - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) class PullConnectorImageFromRegistry(Step): + context: PublishConnectorContext title = "Pull connector image from registry" async def check_if_image_only_has_gzip_layers(self) -> bool: @@ -91,6 +118,7 @@ async def check_if_image_only_has_gzip_layers(self) -> bool: We want to make sure that the image we are about to release is compatible with all docker versions. We use crane to inspect the manifest of the image and check if it only has gzip layers. """ + has_only_gzip_layers = True for platform in consts.BUILD_PLATFORMS: inspect = docker.with_crane(self.context).with_exec( ["manifest", "--platform", f"{str(platform)}", f"docker.io/{self.context.docker_image}"] @@ -102,10 +130,11 @@ async def check_if_image_only_has_gzip_layers(self) -> bool: try: for layer in json.loads(inspect_stdout)["layers"]: if not layer["mediaType"].endswith("gzip"): - return False - return True + has_only_gzip_layers = False + break except (KeyError, json.JSONDecodeError) as e: raise Exception(f"Failed to parse manifest for {self.context.docker_image}: {inspect_stdout}") from e + return has_only_gzip_layers async def _run(self, attempt: int = 3) -> StepResult: try: @@ -116,16 +145,16 @@ async def _run(self, attempt: int = 3) -> StepResult: await anyio.sleep(10) return await self._run(attempt - 1) else: - return StepResult(self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image}") + return StepResult(step=self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image}") if not await self.check_if_image_only_has_gzip_layers(): return StepResult( - self, + step=self, status=StepStatus.FAILURE, stderr=f"Image {self.context.docker_image} does not only have gzip compressed layers. Please rebuild the connector with Docker < 21.", ) else: return StepResult( - self, + step=self, status=StepStatus.SUCCESS, stdout=f"Pulled {self.context.docker_image} and validated it has gzip only compressed layers and we can run spec on it.", ) @@ -133,24 +162,25 @@ async def _run(self, attempt: int = 3) -> StepResult: if attempt > 0: await anyio.sleep(10) return await self._run(attempt - 1) - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) class UploadSpecToCache(Step): + context: PublishConnectorContext title = "Upload connector spec to spec cache bucket" default_spec_file_name = "spec.json" cloud_spec_file_name = "spec.cloud.json" @property - def spec_key_prefix(self): + def spec_key_prefix(self) -> str: return "specs/" + self.context.docker_image.replace(":", "/") @property - def cloud_spec_key(self): + def cloud_spec_key(self) -> str: return f"{self.spec_key_prefix}/{self.cloud_spec_file_name}" @property - def oss_spec_key(self): + def oss_spec_key(self) -> str: return f"{self.spec_key_prefix}/{self.default_spec_file_name}" def _parse_spec_output(self, spec_output: str) -> str: @@ -176,7 +206,7 @@ async def _get_connector_spec(self, connector: Container, deployment_mode: str) spec_output = await connector.with_env_variable("DEPLOYMENT_MODE", deployment_mode).with_exec(["spec"]).stdout() return self._parse_spec_output(spec_output) - async def _get_spec_as_file(self, spec: str, name="spec_to_cache.json") -> File: + async def _get_spec_as_file(self, spec: str, name: str = "spec_to_cache.json") -> File: return (await self.context.get_connector_dir()).with_new_file(name, contents=spec).file(name) async def _run(self, built_connector: Container) -> StepResult: @@ -184,7 +214,7 @@ async def _run(self, built_connector: Container) -> StepResult: oss_spec: str = await self._get_connector_spec(built_connector, "OSS") cloud_spec: str = await self._get_connector_spec(built_connector, "CLOUD") except InvalidSpecOutputError as e: - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, await self._get_spec_as_file(oss_spec))] @@ -201,8 +231,8 @@ async def _run(self, built_connector: Container) -> StepResult: flags=['--cache-control="no-cache"'], ) if exit_code != 0: - return StepResult(self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr) - return StepResult(self, status=StepStatus.SUCCESS, stdout="Uploaded connector spec to spec cache bucket.") + return StepResult(step=self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr) + return StepResult(step=self, status=StepStatus.SUCCESS, stdout="Uploaded connector spec to spec cache bucket.") # Pipeline @@ -253,6 +283,11 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() results.append(check_connector_image_results) + python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) + results.extend(python_registry_steps) + if terminate_early: + return create_connector_report(results) + # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. # We also need to upload the spec to the spec cache bucket. if check_connector_image_results.status is StepStatus.SKIPPED: @@ -278,7 +313,7 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: if build_connector_results.status is not StepStatus.SUCCESS: return create_connector_report(results) - built_connector_platform_variants = list(build_connector_results.output_artifact.values()) + built_connector_platform_variants = list(build_connector_results.output.values()) push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants) results.append(push_connector_image_results) @@ -302,8 +337,45 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) + connector_report = create_connector_report(results) + return connector_report + - return create_connector_report(results) +async def _run_python_registry_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]: + """ + Run the python registry publish pipeline for a single connector. + Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. + """ + results: List[StepResult] = [] + # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to a python registry. + python_registry_context = await PythonRegistryPublishContext.from_publish_connector_context(context) + if not python_registry_context: + return results, False + + if not context.python_registry_token or not context.python_registry_url: + # If the python registry token or url are not set, we can't publish to the python registry - stop the pipeline. + return [ + StepResult( + step=PublishToPythonRegistry(python_registry_context), + status=StepStatus.FAILURE, + stderr="Pypi publishing is enabled, but python registry token or url are not set.", + ) + ], True + + check_python_registry_package_exists_results = await CheckPythonRegistryPackageDoesNotExist(python_registry_context).run() + results.append(check_python_registry_package_exists_results) + if check_python_registry_package_exists_results.status is StepStatus.SKIPPED: + context.logger.info("The connector version is already published on python registry.") + elif check_python_registry_package_exists_results.status is StepStatus.SUCCESS: + context.logger.info("The connector version is not published on python registry. Let's build and publish it.") + publish_to_python_registry_results = await PublishToPythonRegistry(python_registry_context).run() + results.append(publish_to_python_registry_results) + if publish_to_python_registry_results.status is StepStatus.FAILURE: + return results, True + elif check_python_registry_package_exists_results.status is StepStatus.FAILURE: + return results, True + + return results, False def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 43f834909207..594b9573ee57 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -1,16 +1,20 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import json import webbrowser from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Dict -from anyio import Path -from connector_ops.utils import console +from connector_ops.utils import console # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape from pipelines.consts import GCS_PUBLIC_DOMAIN from pipelines.helpers.utils import format_duration +from pipelines.models.artifacts import Artifact from pipelines.models.reports import Report from pipelines.models.steps import StepStatus from rich.console import Group @@ -19,26 +23,40 @@ from rich.table import Table from rich.text import Text +if TYPE_CHECKING: + from typing import List + + from pipelines.airbyte_ci.connectors.context import ConnectorContext + from rich.tree import RenderableType + @dataclass(frozen=True) class ConnectorReport(Report): """A dataclass to build connector test reports to share pipelines executions results with the user.""" + pipeline_context: ConnectorContext + @property - def report_output_prefix(self) -> str: # noqa D102 + def report_output_prefix(self) -> str: return f"{self.pipeline_context.report_output_prefix}/{self.pipeline_context.connector.technical_name}/{self.pipeline_context.connector.version}" @property - def html_report_file_name(self) -> str: # noqa D102 + def html_report_file_name(self) -> str: return self.filename + ".html" + def file_remote_storage_key(self, file_name: str) -> str: + return f"{self.report_output_prefix}/{file_name}" + @property - def html_report_remote_storage_key(self) -> str: # noqa D102 - return f"{self.report_output_prefix}/{self.html_report_file_name}" + def html_report_remote_storage_key(self) -> str: + return self.file_remote_storage_key(self.html_report_file_name) + + def file_url(self, file_name: str) -> str: + return f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{self.file_remote_storage_key(file_name)}" @property - def html_report_url(self) -> str: # noqa D102 - return f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{self.html_report_remote_storage_key}" + def html_report_url(self) -> str: + return self.file_url(self.html_report_file_name) def to_json(self) -> str: """Create a JSON representation of the connector test report. @@ -46,6 +64,8 @@ def to_json(self) -> str: Returns: str: The JSON representation of the report. """ + assert self.pipeline_context.pipeline_start_timestamp is not None, "The pipeline start timestamp must be set to save reports." + return json.dumps( { "connector_technical_name": self.pipeline_context.connector.technical_name, @@ -53,9 +73,9 @@ def to_json(self) -> str: "run_timestamp": self.created_at.isoformat(), "run_duration": self.run_duration.total_seconds(), "success": self.success, - "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], - "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], - "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], # type: ignore + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], # type: ignore + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], # type: ignore "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, "pipeline_end_timestamp": round(self.created_at.timestamp()), @@ -69,7 +89,7 @@ def to_json(self) -> str: } ) - async def to_html(self) -> str: + def to_html(self) -> str: env = Environment( loader=PackageLoader("pipelines.airbyte_ci.connectors.test.steps"), autoescape=select_autoescape(), @@ -79,7 +99,18 @@ async def to_html(self) -> str: template = env.get_template("test_report.html.j2") template.globals["StepStatus"] = StepStatus template.globals["format_duration"] = format_duration - local_icon_path = await Path(f"{self.pipeline_context.connector.code_directory}/icon.svg").resolve() + local_icon_path = Path(f"{self.pipeline_context.connector.code_directory}/icon.svg").resolve() + step_result_to_artifact_links: Dict[str, List[Dict]] = {} + for step_result in self.steps_results: + for artifact in step_result.artifacts: + if artifact.gcs_url: + url = artifact.gcs_url + elif artifact.local_path: + url = artifact.local_path.resolve().as_uri() + else: + continue + step_result_to_artifact_links.setdefault(step_result.step.title, []).append({"name": artifact.name, "url": url}) + template_context = { "connector_name": self.pipeline_context.connector.technical_name, "step_results": self.steps_results, @@ -92,6 +123,8 @@ async def to_html(self) -> str: "git_revision": self.pipeline_context.git_revision, "commit_url": None, "icon_url": local_icon_path.as_uri(), + "report": self, + "step_result_to_artifact_links": MappingProxyType(step_result_to_artifact_links), } if self.pipeline_context.is_ci: @@ -104,20 +137,34 @@ async def to_html(self) -> str: ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" return template.render(template_context) + async def save_html_report(self) -> None: + """Save the report as HTML, upload it to GCS if the pipeline is running in CI""" + + html_report_path = self.report_dir_path / self.html_report_file_name + report_dir = self.pipeline_context.dagger_client.host().directory(str(self.report_dir_path)) + local_html_report_file = report_dir.with_new_file(self.html_report_file_name, self.to_html()).file(self.html_report_file_name) + html_report_artifact = Artifact(name="HTML Report", content_type="text/html", content=local_html_report_file) + await html_report_artifact.save_to_local_path(html_report_path) + absolute_path = html_report_path.absolute() + self.pipeline_context.logger.info(f"Report saved locally at {absolute_path}") + if self.remote_storage_enabled and self.pipeline_context.ci_gcs_credentials_secret and self.pipeline_context.ci_report_bucket: + gcs_url = await html_report_artifact.upload_to_gcs( + dagger_client=self.pipeline_context.dagger_client, + bucket=self.pipeline_context.ci_report_bucket, + key=self.html_report_remote_storage_key, + gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, + ) + self.pipeline_context.logger.info(f"HTML report uploaded to {gcs_url}") + + elif self.pipeline_context.enable_report_auto_open: + self.pipeline_context.logger.info("Opening HTML report in browser.") + webbrowser.open(absolute_path.as_uri()) + async def save(self) -> None: - local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) - absolute_path = await local_html_path.resolve() - if self.pipeline_context.enable_report_auto_open: - self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") - if self.pipeline_context.enable_report_auto_open: - self.pipeline_context.logger.info("Opening HTML report in browser.") - webbrowser.open(absolute_path.as_uri()) - if self.remote_storage_enabled: - await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html") - self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}") await super().save() + await self.save_html_report() - def print(self): + def print(self) -> None: """Print the test report to the console in a nice way.""" connector_name = self.pipeline_context.connector.technical_name main_panel_title = Text(f"{connector_name.upper()} - {self.name}") @@ -136,7 +183,7 @@ def print(self): step_results_table.add_row(step, result, format_duration(step_result.step.run_duration)) details_instructions = Text("ℹ️ You can find more details with step executions logs in the saved HTML report.") - to_render = [step_results_table, details_instructions] + to_render: List[RenderableType] = [step_results_table, details_instructions] if self.pipeline_context.dagger_cloud_url: self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index b858ca839e00..41c1f629ff39 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -2,20 +2,32 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys +from typing import Dict, List import asyncclick as click from pipelines import main_logger +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState +from pipelines.helpers.execution import argument_parsing +from pipelines.helpers.execution.run_steps import RunStepOptions from pipelines.helpers.github import update_global_commit_status_check_for_tests from pipelines.helpers.utils import fail_if_missing_docker_hub_creds +from pipelines.models.steps import STEP_PARAMS -@click.command(cls=DaggerPipelineCommand, help="Test all the selected connectors.") +@click.command( + cls=DaggerPipelineCommand, + help="Test all the selected connectors.", + context_settings=dict( + ignore_unknown_options=True, + ), +) +@click_ci_requirements_option() @click.option( "--code-tests-only", is_flag=True, @@ -30,13 +42,6 @@ type=bool, is_flag=True, ) -@click.option( - "--fast-tests-only", - help="When enabled, slow tests are skipped.", - default=False, - type=bool, - is_flag=True, -) @click.option( "--concurrent-cat", help="When enabled, the CAT tests will run concurrently. Be careful about rate limits", @@ -44,24 +49,44 @@ type=bool, is_flag=True, ) +@click.option( + "--skip-step", + "-x", + "skip_steps", + multiple=True, + type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), + help="Skip a step by name. Can be used multiple times to skip multiple steps.", +) +@click.option( + "--only-step", + "-k", + "only_steps", + multiple=True, + type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), + help="Only run specific step by name. Can be used multiple times to keep multiple steps.", +) +@click.argument( + "extra_params", nargs=-1, type=click.UNPROCESSED, callback=argument_parsing.build_extra_params_mapping(CONNECTOR_TEST_STEP_ID) +) @click.pass_context async def test( ctx: click.Context, code_tests_only: bool, fail_fast: bool, - fast_tests_only: bool, concurrent_cat: bool, + skip_steps: List[str], + only_steps: List[str], + extra_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS], ) -> bool: """Runs a test pipeline for the selected connectors. Args: ctx (click.Context): The click context. """ + if only_steps and skip_steps: + raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.") if ctx.obj["is_ci"]: fail_if_missing_docker_hub_creds(ctx) - if ctx.obj["is_ci"] and ctx.obj["pull_request"] and ctx.obj["pull_request"].draft: - main_logger.info("Skipping connectors tests for draft pull request.") - sys.exit(0) if ctx.obj["selected_connectors_with_modified_files"]: update_global_commit_status_check_for_tests(ctx.obj, "pending") @@ -70,6 +95,12 @@ async def test( update_global_commit_status_check_for_tests(ctx.obj, "success") return True + run_step_options = RunStepOptions( + fail_fast=fail_fast, + skip_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in skip_steps], + keep_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in only_steps], + step_params=extra_params, + ) connectors_tests_contexts = [ ConnectorContext( pipeline_name=f"Testing connector {connector.technical_name}", @@ -86,8 +117,6 @@ async def test( ci_context=ctx.obj.get("ci_context"), pull_request=ctx.obj.get("pull_request"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - fail_fast=fail_fast, - fast_tests_only=fast_tests_only, code_tests_only=code_tests_only, use_local_cdk=ctx.obj.get("use_local_cdk"), s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), @@ -95,10 +124,12 @@ async def test( docker_hub_username=ctx.obj.get("docker_hub_username"), docker_hub_password=ctx.obj.get("docker_hub_password"), concurrent_cat=concurrent_cat, + run_step_options=run_step_options, targeted_platforms=[LOCAL_BUILD_PLATFORM], ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] + try: await run_connectors_pipelines( [connector_context for connector_context in connectors_tests_contexts], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py index df5a4d28888c..98af38978d70 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -3,101 +3,74 @@ # """This module groups factory like functions to dispatch tests steps according to the connector under test language.""" -import itertools -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING import anyio -import asyncer -from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage +from connector_ops.utils import ConnectorLanguage # type: ignore +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.connectors.test.steps import java_connectors, python_connectors -from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck -from pipelines.airbyte_ci.metadata.pipeline import MetadataValidation -from pipelines.models.steps import StepResult +from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks, VersionIncrementCheck +from pipelines.helpers.execution.run_steps import StepToRun, run_steps -LANGUAGE_MAPPING = { - "run_all_tests": { - ConnectorLanguage.PYTHON: python_connectors.run_all_tests, - ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, - ConnectorLanguage.JAVA: java_connectors.run_all_tests, - } -} +if TYPE_CHECKING: + from pipelines.helpers.execution.run_steps import STEP_TREE -async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]: - """Run the metadata validation on a connector. - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the metadata validation steps. - """ - return [await MetadataValidation(context).run()] - - -async def run_version_checks(context: ConnectorContext) -> List[StepResult]: - """Run the version checks on a connector. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the version checks steps. - """ - return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()] - - -async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: - """Run the QA checks on a connector. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the QA checks steps. - """ - return [await QaChecks(context).run()] +LANGUAGE_MAPPING = { + "get_test_steps": { + ConnectorLanguage.PYTHON: python_connectors.get_test_steps, + ConnectorLanguage.LOW_CODE: python_connectors.get_test_steps, + ConnectorLanguage.JAVA: java_connectors.get_test_steps, + }, +} -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all the tests steps according to the connector language. +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """Get all the tests steps according to the connector language. Args: context (ConnectorContext): The current connector context. Returns: - List[StepResult]: The results of the tests steps. + STEP_TREE: The list of tests steps. """ - if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language): - return await _run_all_tests(context) + if _get_test_steps := LANGUAGE_MAPPING["get_test_steps"].get(context.connector.language): + return _get_test_steps(context) else: context.logger.warning(f"No tests defined for connector language {context.connector.language}!") return [] async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: - """Run a test pipeline for a single connector. + """ + Compute the steps to run for a connector test pipeline. + """ + all_steps_to_run: STEP_TREE = [] - A visual DAG can be found on the README.md file of the pipelines modules. + all_steps_to_run += get_test_steps(context) - Args: - context (ConnectorContext): The initialized connector context. + if not context.code_tests_only: + static_analysis_steps_to_run = [ + [ + StepToRun(id=CONNECTOR_TEST_STEP_ID.VERSION_INC_CHECK, step=VersionIncrementCheck(context)), + StepToRun(id=CONNECTOR_TEST_STEP_ID.QA_CHECKS, step=QaChecks(context)), + ] + ] + all_steps_to_run += static_analysis_steps_to_run - Returns: - ConnectorReport: The test reports holding tests results. - """ async with semaphore: async with context: - async with asyncer.create_task_group() as task_group: - tasks = [task_group.soonify(run_all_tests)(context)] - if not context.code_tests_only: - tasks += [ - task_group.soonify(run_metadata_validation)(context), - task_group.soonify(run_version_checks)(context), - task_group.soonify(run_qa_checks)(context), - ] - results = list(itertools.chain(*(task.value for task in tasks))) - context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") - - return context.report + result_dict = await run_steps( + runnables=all_steps_to_run, + options=context.run_step_options, + ) + + results = list(result_dict.values()) + report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") + context.report = report + + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 8d384d6bbd60..177c7dc6a281 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -8,31 +8,35 @@ import os from abc import ABC, abstractmethod from functools import cached_property +from pathlib import Path from typing import ClassVar, List, Optional -import requests +import requests # type: ignore import semver -import yaml -from connector_ops.utils import Connector +import yaml # type: ignore from dagger import Container, Directory from pipelines import hacks -from pipelines.consts import CIContext +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.steps.docker import SimpleDockerStep +from pipelines.consts import INTERNAL_TOOL_PATHS, CIContext from pipelines.dagger.actions import secrets -from pipelines.dagger.containers import internal_tools from pipelines.helpers.utils import METADATA_FILE_NAME -from pipelines.models.contexts.pipeline_context import PipelineContext -from pipelines.models.steps import Step, StepResult, StepStatus +from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus class VersionCheck(Step, ABC): """A step to validate the connector version was bumped if files were modified""" + context: ConnectorContext GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors" failure_message: ClassVar - should_run = True @property - def github_master_metadata_url(self): + def should_run(self) -> bool: + return True + + @property + def github_master_metadata_url(self) -> str: return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" @cached_property @@ -58,11 +62,11 @@ def current_connector_version(self) -> semver.Version: @property def success_result(self) -> StepResult: - return StepResult(self, status=StepStatus.SUCCESS) + return StepResult(step=self, status=StepStatus.SUCCESS) @property def failure_result(self) -> StepResult: - return StepResult(self, status=StepStatus.FAILURE, stderr=self.failure_message) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=self.failure_message) @abstractmethod def validate(self) -> StepResult: @@ -70,16 +74,17 @@ def validate(self) -> StepResult: async def _run(self) -> StepResult: if not self.should_run: - return StepResult(self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.") + return StepResult(step=self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.") if self.context.ci_context == CIContext.MASTER: - return StepResult(self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.") + return StepResult(step=self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.") try: return self.validate() except (requests.HTTPError, ValueError, TypeError) as e: - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) class VersionIncrementCheck(VersionCheck): + context: ConnectorContext title = "Connector version increment check" BYPASS_CHECK_FOR = [ @@ -114,77 +119,77 @@ def validate(self) -> StepResult: return self.success_result -class VersionFollowsSemverCheck(VersionCheck): - title = "Connector version semver check" - - @property - def failure_message(self) -> str: - return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" - - def validate(self) -> StepResult: - try: - if not self.current_connector_version >= self.master_connector_version: - return self.failure_result - except ValueError: - return self.failure_result - return self.success_result - - -class QaChecks(Step): - """A step to run QA checks for a connector.""" - - title = "QA checks" - - async def _run(self) -> StepResult: - """Run QA checks on a connector. - - The QA checks are defined in this module: - https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/qa_checks.py - - Args: - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. - Returns: - StepResult: Failure or success of the QA checks with stdout and stderr. - """ - connector_ops = await internal_tools.with_connector_ops(self.context) - include = [ - str(self.context.connector.code_directory), - str(self.context.connector.documentation_file_path), - str(self.context.connector.migration_guide_file_path), - str(self.context.connector.icon_path), - ] - if ( - self.context.connector.technical_name.endswith("strict-encrypt") - or self.context.connector.technical_name == "source-file-secure" - ): - original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) - include += [ - str(original_connector.code_directory), - str(original_connector.documentation_file_path), - str(original_connector.icon_path), - str(original_connector.migration_guide_file_path), - ] - - filtered_repo = self.context.get_repo_dir( - include=include, - ) - - qa_checks = ( - connector_ops.with_mounted_directory("/airbyte", filtered_repo) - .with_workdir("/airbyte") - .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) +class QaChecks(SimpleDockerStep): + """A step to run QA checks for a connectors. + More details in https://github.com/airbytehq/airbyte/blob/main/airbyte-ci/connectors/connectors_qa/README.md + """ + + def __init__(self, context: ConnectorContext) -> None: + code_directory = context.connector.code_directory + documentation_file_path = context.connector.documentation_file_path + migration_guide_file_path = context.connector.migration_guide_file_path + icon_path = context.connector.icon_path + technical_name = context.connector.technical_name + + # When the connector is strict-encrypt, we should run QA checks on the main one as it's the one whose artifacts gets released + if context.connector.technical_name.endswith("-strict-encrypt"): + technical_name = technical_name.replace("-strict-encrypt", "") + code_directory = Path(str(code_directory).replace("-strict-encrypt", "")) + if documentation_file_path: + documentation_file_path = Path(str(documentation_file_path).replace("-strict-encrypt", "")) + if migration_guide_file_path: + migration_guide_file_path = Path(str(migration_guide_file_path).replace("-strict-encrypt", "")) + if icon_path: + icon_path = Path(str(icon_path).replace("-strict-encrypt", "")) + + super().__init__( + title=f"Run QA checks for {technical_name}", + context=context, + paths_to_mount=[ + MountPath(code_directory), + # These paths are optional + # But their absence might make the QA check fail + MountPath(documentation_file_path, optional=True), + MountPath(migration_guide_file_path, optional=True), + MountPath(icon_path, optional=True), + ], + internal_tools=[ + MountPath(INTERNAL_TOOL_PATHS.CONNECTORS_QA.value), + ], + secrets={ + k: v + for k, v in { + "DOCKER_HUB_USERNAME": context.docker_hub_username_secret, + "DOCKER_HUB_PASSWORD": context.docker_hub_password_secret, + }.items() + if v + }, + command=["connectors-qa", "run", f"--name={technical_name}"], ) - return await self.get_step_result(qa_checks) - class AcceptanceTests(Step): """A step to run acceptance tests for a connector if it has an acceptance test config file.""" + context: ConnectorContext title = "Acceptance tests" CONTAINER_TEST_INPUT_DIRECTORY = "/test_input" CONTAINER_SECRETS_DIRECTORY = "/test_input/secrets" skipped_exit_code = 5 + accept_extra_params = True + + @property + def default_params(self) -> STEP_PARAMS: + """Default pytest options. + + Returns: + dict: The default pytest options. + """ + return super().default_params | { + "-ra": [], # Show extra test summary info in the report for all but the passed tests + "--disable-warnings": [], # Disable warnings in the pytest report + "--durations": ["3"], # Show the 3 slowest tests in the report + } @property def base_cat_command(self) -> List[str]: @@ -192,23 +197,21 @@ def base_cat_command(self) -> List[str]: "python", "-m", "pytest", - "--disable-warnings", - "--durations=3", # Show the 3 slowest tests in the report - "-ra", # Show extra test summary info in the report for all but the passed tests "-p", # Load the connector_acceptance_test plugin "connector_acceptance_test.plugin", "--acceptance-test-config", self.CONTAINER_TEST_INPUT_DIRECTORY, ] + if self.concurrent_test_run: command += ["--numprocesses=auto"] # Using pytest-xdist to run tests in parallel, auto means using all available cores return command - def __init__(self, context: PipelineContext, concurrent_test_run: Optional[bool] = False) -> None: + def __init__(self, context: ConnectorContext, concurrent_test_run: Optional[bool] = False) -> None: """Create a step to run acceptance tests for a connector if it has an acceptance test config file. Args: - context (PipelineContext): The current test context, providing a connector object, a dagger client and a repository directory. + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. concurrent_test_run (Optional[bool], optional): Whether to run acceptance tests in parallel. Defaults to False. """ super().__init__(context) @@ -224,7 +227,7 @@ async def get_cat_command(self, connector_dir: Directory) -> List[str]: if "integration_tests" in await connector_dir.entries(): if "acceptance.py" in await connector_dir.directory("integration_tests").entries(): cat_command += ["-p", "integration_tests.acceptance"] - return cat_command + return cat_command + self.params_as_cli_options async def _run(self, connector_under_test_container: Container) -> StepResult: """Run the acceptance test suite on a connector dev image. Build the connector acceptance test image if the tag is :dev. @@ -237,7 +240,7 @@ async def _run(self, connector_under_test_container: Container) -> StepResult: """ if not self.context.connector.acceptance_test_config: - return StepResult(self, StepStatus.SKIPPED) + return StepResult(step=self, status=StepStatus.SKIPPED) connector_dir = await self.context.get_connector_dir() cat_container = await self._build_connector_acceptance_test(connector_under_test_container, connector_dir) cat_command = await self.get_cat_command(connector_dir) @@ -252,7 +255,7 @@ async def _run(self, connector_under_test_container: Container) -> StepResult: break return step_result - async def get_cache_buster(self) -> str: + def get_cache_buster(self) -> str: """ This bursts the CAT cached results everyday and on new version or image size change. It's cool because in case of a partially failing nightly build the connectors that already ran CAT won't re-run CAT. @@ -283,11 +286,11 @@ async def _build_connector_acceptance_test(self, connector_under_test_container: cat_container = ( cat_container.with_env_variable("RUN_IN_AIRBYTE_CI", "1") .with_exec(["mkdir", "/dagger_share"], skip_entrypoint=True) - .with_env_variable("CACHEBUSTER", await self.get_cache_buster()) - .with_new_file("/tmp/container_id.txt", str(connector_container_id)) + .with_env_variable("CACHEBUSTER", self.get_cache_buster()) + .with_new_file("/tmp/container_id.txt", contents=str(connector_container_id)) .with_workdir("/test_input") .with_mounted_directory("/test_input", test_input) - .with_(await secrets.mounted_connector_secrets(self.context, "/test_input/secrets")) + .with_(await secrets.mounted_connector_secrets(self.context, self.CONTAINER_SECRETS_DIRECTORY)) ) if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: self.context.logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") @@ -298,29 +301,3 @@ async def _build_connector_acceptance_test(self, connector_under_test_container: ) return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) - - -class CheckBaseImageIsUsed(Step): - title = "Check our base image is used" - - async def _run(self, *args, **kwargs) -> StepResult: - is_certified = self.context.connector.metadata.get("supportLevel") == "certified" - if not is_certified: - return self.skip("Connector is not certified, it does not require the use of our base image.") - - is_using_base_image = self.context.connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None - migration_hint = f"Please run 'airbyte-ci connectors --name={self.context.connector.technical_name} migrate_to_base_image ' and commit the changes." - if not is_using_base_image: - return StepResult( - self, - StepStatus.FAILURE, - stdout=f"Connector is certified but does not use our base image. {migration_hint}", - ) - has_dockerfile = "Dockerfile" in await (await self.context.get_connector_dir(include="Dockerfile")).entries() - if has_dockerfile: - return StepResult( - self, - StepStatus.FAILURE, - stdout=f"Connector is certified but is still using a Dockerfile. {migration_hint}", - ) - return StepResult(self, StepStatus.SUCCESS, stdout="Connector is certified and uses our base image.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py index 5e8babe25223..a4259b8f67c8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py @@ -3,43 +3,60 @@ # """This module groups steps made to run tests for a specific Java connector given a test context.""" +from __future__ import annotations -from typing import List, Optional +from typing import TYPE_CHECKING import anyio -import asyncer -from dagger import Directory, File, QueryError +from dagger import File, QueryError from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import ( BuildConnectorDistributionTar, BuildConnectorImages, dist_tar_directory_path, ) from pipelines.airbyte_ci.connectors.build_image.steps.normalization import BuildOrPullNormalization +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests from pipelines.airbyte_ci.steps.gradle import GradleTask from pipelines.consts import LOCAL_BUILD_PLATFORM -from pipelines.dagger.actions import secrets from pipelines.dagger.actions.system import docker +from pipelines.helpers.execution.run_steps import StepToRun from pipelines.helpers.utils import export_container_to_tarball -from pipelines.models.steps import StepResult, StepStatus +from pipelines.models.steps import STEP_PARAMS, StepResult, StepStatus + +if TYPE_CHECKING: + from typing import Callable, Dict, List, Optional + + from pipelines.helpers.execution.run_steps import RESULTS_DICT, STEP_TREE class IntegrationTests(GradleTask): """A step to run integrations tests for Java connectors using the integrationTestJava Gradle task.""" title = "Java Connector Integration Tests" - gradle_task_name = "integrationTestJava -x buildConnectorImage -x assemble" + gradle_task_name = "integrationTestJava" mount_connector_secrets = True bind_to_docker_host = True - - async def _load_normalization_image(self, normalization_tar_file: File): + with_test_artifacts = True + + @property + def default_params(self) -> STEP_PARAMS: + return super().default_params | { + # Exclude the assemble task to avoid a circular dependency on airbyte-ci. + # The integrationTestJava gradle task depends on assemble, which in turns + # depends on buildConnectorImage to build the connector's docker image. + # At this point, the docker image has already been built. + "-x": ["assemble"], + } + + async def _load_normalization_image(self, normalization_tar_file: File) -> None: normalization_image_tag = f"{self.context.connector.normalization_repository}:dev" self.context.logger.info("Load the normalization image to the docker host.") await docker.load_image_to_docker_host(self.context, normalization_tar_file, normalization_image_tag) self.context.logger.info("Successfully loaded the normalization image to the docker host.") - async def _load_connector_image(self, connector_tar_file: File): + async def _load_connector_image(self, connector_tar_file: File) -> None: connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev" self.context.logger.info("Load the connector image to the docker host") await docker.load_image_to_docker_host(self.context, connector_tar_file, connector_image_tag) @@ -52,7 +69,7 @@ async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[ tg.start_soon(self._load_normalization_image, normalization_tar_file) tg.start_soon(self._load_connector_image, connector_tar_file) except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e)) # Run the gradle integration test task now that the required docker images have been loaded. return await super()._run() @@ -63,68 +80,95 @@ class UnitTests(GradleTask): title = "Java Connector Unit Tests" gradle_task_name = "test" bind_to_docker_host = True + with_test_artifacts = True -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all tests for a Java connectors. - - - Build the normalization image if the connector supports it. - - Run unit tests with Gradle. - - Build connector image with Gradle. - - Run integration and acceptance test in parallel using the built connector and normalization images. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of all the tests steps. +def _create_integration_step_args_factory(context: ConnectorContext) -> Callable: + """ + Create a function that can process the args for the integration step. """ - context.connector_secrets = await secrets.get_connector_secrets(context) - step_results = [] - - build_distribution_tar_result = await BuildConnectorDistributionTar(context).run() - step_results.append(build_distribution_tar_result) - if build_distribution_tar_result.status is StepStatus.FAILURE: - return step_results - dist_tar_dir = build_distribution_tar_result.output_artifact.directory(dist_tar_directory_path(context)) + async def _create_integration_step_args(results: RESULTS_DICT) -> Dict[str, Optional[File]]: - async def run_docker_build_dependent_steps(dist_tar_dir: Directory) -> List[StepResult]: - step_results = [] - build_connector_image_results = await BuildConnectorImages(context).run(dist_tar_dir) - step_results.append(build_connector_image_results) - if build_connector_image_results.status is StepStatus.FAILURE: - return step_results + connector_container = results["build"].output[LOCAL_BUILD_PLATFORM] + connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container, LOCAL_BUILD_PLATFORM) if context.connector.supports_normalization: - normalization_image = f"{context.connector.normalization_repository}:dev" - context.logger.info(f"This connector supports normalization: will build {normalization_image}.") - build_normalization_results = await BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM).run() - normalization_container = build_normalization_results.output_artifact + tar_file_name = f"{context.connector.normalization_repository}_{context.git_revision}.tar" + build_normalization_results = results["build_normalization"] + + normalization_container = build_normalization_results.output normalization_tar_file, _ = await export_container_to_tarball( - context, - normalization_container, - LOCAL_BUILD_PLATFORM, - tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar", + context, normalization_container, LOCAL_BUILD_PLATFORM, tar_file_name=tar_file_name ) - step_results.append(build_normalization_results) else: normalization_tar_file = None - connector_container = build_connector_image_results.output_artifact[LOCAL_BUILD_PLATFORM] - connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container, LOCAL_BUILD_PLATFORM) + return {"connector_tar_file": connector_image_tar_file, "normalization_tar_file": normalization_tar_file} - async with asyncer.create_task_group() as docker_build_dependent_group: - soon_integration_tests_results = docker_build_dependent_group.soonify(IntegrationTests(context).run)( - connector_tar_file=connector_image_tar_file, normalization_tar_file=normalization_tar_file - ) - soon_cat_results = docker_build_dependent_group.soonify(AcceptanceTests(context, True).run)(connector_container) + return _create_integration_step_args + + +def _get_normalization_steps(context: ConnectorContext) -> List[StepToRun]: + normalization_image = f"{context.connector.normalization_repository}:dev" + context.logger.info(f"This connector supports normalization: will build {normalization_image}.") + normalization_steps = [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.BUILD_NORMALIZATION, + step=BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ) + ] + + return normalization_steps - step_results += [soon_cat_results.value, soon_integration_tests_results.value] - return step_results - async with asyncer.create_task_group() as test_task_group: - soon_unit_tests_result = test_task_group.soonify(UnitTests(context).run)() - soon_docker_build_dependent_steps_results = test_task_group.soonify(run_docker_build_dependent_steps)(dist_tar_dir) +def _get_acceptance_test_steps(context: ConnectorContext) -> List[StepToRun]: + """ + Generate the steps to run the acceptance tests for a Java connector. + """ + # Run tests in parallel + return [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=_create_integration_step_args_factory(context), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, True), + args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + ] + + +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """ + Get all the tests steps for a Java connector. + """ - return step_results + [soon_unit_tests_result.value] + soon_docker_build_dependent_steps_results.value + steps: STEP_TREE = [ + [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD_TAR, step=BuildConnectorDistributionTar(context))], + [StepToRun(id=CONNECTOR_TEST_STEP_ID.UNIT, step=UnitTests(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD_TAR])], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.BUILD, + step=BuildConnectorImages(context), + args=lambda results: { + "dist_dir": results[CONNECTOR_TEST_STEP_ID.BUILD_TAR].output.directory(dist_tar_directory_path(context)) + }, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD_TAR], + ), + ], + ] + + if context.connector.supports_normalization: + normalization_steps = _get_normalization_steps(context) + steps.append(normalization_steps) + + acceptance_test_steps = _get_acceptance_test_steps(context) + steps.append(acceptance_test_steps) + + return steps diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 769eb8575146..c7cc04cea7f3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -5,29 +5,47 @@ """This module groups steps made to run tests for a specific Python connector given a test context.""" from abc import ABC, abstractmethod -from typing import Callable, Iterable, List, Tuple +from typing import List, Sequence, Tuple -import asyncer +import dpath.util import pipelines.dagger.actions.python.common import pipelines.dagger.actions.system.docker from dagger import Container, File +from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed +from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.dagger.actions import secrets -from pipelines.models.steps import Step, StepResult, StepStatus +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun +from pipelines.models.steps import STEP_PARAMS, Step, StepResult class PytestStep(Step, ABC): """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" + context: ConnectorContext + PYTEST_INI_FILE_NAME = "pytest.ini" PYPROJECT_FILE_NAME = "pyproject.toml" common_test_dependencies: List[str] = [] skipped_exit_code = 5 bind_to_docker_host = False + accept_extra_params = True + + @property + def default_params(self) -> STEP_PARAMS: + """Default pytest options. + + Returns: + dict: The default pytest options. + """ + return super().default_params | { + "-s": [], # Disable capturing stdout/stderr in pytest + } @property @abstractmethod @@ -35,20 +53,11 @@ def test_directory_name(self) -> str: raise NotImplementedError("test_directory_name must be implemented in the child class.") @property - def extra_dependencies_names(self) -> Iterable[str]: + def extra_dependencies_names(self) -> Sequence[str]: if self.context.connector.is_using_poetry: return ("dev",) return ("dev", "tests") - @property - def additional_pytest_options(self) -> List[str]: - """Theses options are added to the pytest command. - - Returns: - List[str]: The additional pytest options. - """ - return [] - async def _run(self, connector_under_test: Container) -> StepResult: """Run all pytest tests declared in the test directory of the connector code. @@ -80,7 +89,7 @@ def get_pytest_command(self, test_config_file_name: str) -> List[str]: Returns: List[str]: The pytest command to run. """ - cmd = ["pytest", "-s", self.test_directory_name, "-c", test_config_file_name] + self.additional_pytest_options + cmd = ["pytest", self.test_directory_name, "-c", test_config_file_name] + self.params_as_cli_options if self.context.connector.is_using_poetry: return ["poetry", "run"] + cmd return cmd @@ -127,15 +136,15 @@ async def install_testing_environment( built_connector_container: Container, test_config_file_name: str, test_config_file: File, - extra_dependencies_names: Iterable[str], - ) -> Callable: + extra_dependencies_names: Sequence[str], + ) -> Container: """Install the connector with the extra dependencies in /test_environment. Args: - extra_dependencies_names (Iterable[str]): Extra dependencies to install. + extra_dependencies_names (List[str]): Extra dependencies to install. Returns: - Callable: The decorator to use with the with_ method of a container. + Container: The container with the test environment installed. """ secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets") @@ -171,57 +180,101 @@ class UnitTests(PytestStep): MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS = 90 @property - def additional_pytest_options(self) -> List[str]: + def default_params(self) -> STEP_PARAMS: """Make sure the coverage computation is run for the unit tests. - Fail if the coverage is under 90% for certified connectors. Returns: - List[str]: The additional pytest options to run coverage reports. + dict: The default pytest options. """ - coverage_options = ["--cov", self.context.connector.technical_name.replace("-", "_")] + coverage_options = {"--cov": [self.context.connector.technical_name.replace("-", "_")]} if self.context.connector.support_level == "certified": - coverage_options += ["--cov-fail-under", str(self.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS)] + coverage_options["--cov-fail-under"] = [str(self.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS)] + return super().default_params | coverage_options - return super().additional_pytest_options + coverage_options +class AirbyteLibValidation(Step): + """A step to validate the connector will work with airbyte-lib, using the airbyte-lib validation helper.""" -class IntegrationTests(PytestStep): - """A step to run the connector integration tests with Pytest.""" + title = "AirbyteLib validation tests" - title = "Integration tests" - test_directory_name = "integration_tests" - bind_to_docker_host = True + context: ConnectorContext + + async def _run(self, connector_under_test: Container) -> StepResult: + """Run all pytest tests declared in the test directory of the connector code. + Args: + connector_under_test (Container): The connector under test container. + Returns: + StepResult: Failure or success of the unit tests with stdout and stdout. + """ + if dpath.util.get(self.context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: + return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") + test_environment = await self.install_testing_environment(with_poetry(self.context)) + test_execution = test_environment.with_( + hacks.never_fail_exec(["airbyte-lib-validate-source", "--connector-dir", ".", "--validate-install-only"]) + ) -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all tests for a Python connector. + return await self.get_step_result(test_execution) - Args: - context (ConnectorContext): The current connector context. + async def install_testing_environment( + self, + built_connector_container: Container, + ) -> Container: + """Add airbyte-lib and secrets to the test environment.""" + context: ConnectorContext = self.context - Returns: - List[StepResult]: The results of all the steps that ran or were skipped. - """ - step_results = [] - build_connector_image_results = await BuildConnectorImages(context).run() - if build_connector_image_results.status is StepStatus.FAILURE: - return [build_connector_image_results] - step_results.append(build_connector_image_results) + container_with_test_deps = await pipelines.dagger.actions.python.common.with_python_package( + self.context, built_connector_container.with_entrypoint([]), str(context.connector.code_directory) + ) + return container_with_test_deps.with_exec( + [ + "pip", + "install", + "airbyte-lib", + ] + ) - connector_container = build_connector_image_results.output_artifact[LOCAL_BUILD_PLATFORM] - context.connector_secrets = await secrets.get_connector_secrets(context) +class IntegrationTests(PytestStep): + """A step to run the connector integration tests with Pytest.""" - unit_test_results = await UnitTests(context).run(connector_container) + title = "Integration tests" + test_directory_name = "integration_tests" + bind_to_docker_host = True - if unit_test_results.status is StepStatus.FAILURE: - return step_results + [unit_test_results] - step_results.append(unit_test_results) - async with asyncer.create_task_group() as task_group: - tasks = [ - task_group.soonify(IntegrationTests(context).run)(connector_container), - task_group.soonify(AcceptanceTests(context, context.concurrent_cat).run)(connector_container), - task_group.soonify(CheckBaseImageIsUsed(context).run)(), - ] - return step_results + [task.value for task in tasks] +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """ + Get all the tests steps for a Python connector. + """ + return [ + [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD, step=BuildConnectorImages(context))], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.UNIT, + step=UnitTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ) + ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.AIRBYTE_LIB_VALIDATION, + step=AirbyteLibValidation(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, context.concurrent_cat), + args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + ], + ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 index 5ac9282ac5bd..d0027605d0bf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 @@ -82,7 +82,7 @@ transition: max-height .25s ease-in-out; } .toggle:checked + .lbl-toggle + .collapsible-content { - max-height: 100vh; + max-height: 70vh; } .toggle:checked + .lbl-toggle { border-bottom-right-radius: 0; @@ -110,6 +110,14 @@ } +

    {{ connector_name }} test report

      @@ -159,13 +167,21 @@ {% endif %}
      + {% if step_result_to_artifact_links[step_result.step.title] %} +

      Artifacts

      +
        + {% for artifact in step_result_to_artifact_links[step_result.step.title] %} +
      • {{ artifact.name }}
        • + {% endfor %} +
      + {% endif %}
      {% if step_result.stdout %} - Standard output: + Standard output(): 
      {{ step_result.stdout }}
      {% endif %} {% if step_result.stderr %} - Standard error: + Standard error(): 
      {{ step_result.stderr }}
      {% endif %}
      diff --git a/octavia-cli/octavia_cli/_import/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/__init__.py similarity index 100% rename from octavia-cli/octavia_cli/_import/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py new file mode 100644 index 000000000000..68c031cac62e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + + +@click.command(cls=DaggerPipelineCommand, short_help="Upgrade CDK version") +@click.argument("target-cdk-version", type=str, default="latest") +@click.pass_context +async def bump_version( + ctx: click.Context, + target_cdk_version: str, +) -> bool: + """Upgrade CDK version""" + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Upgrade CDK version of connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + enable_report_auto_open=False, + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + await run_connectors_pipelines( + connectors_contexts, + run_connector_cdk_upgrade_pipeline, + "Upgrade CDK version pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + target_cdk_version, + ) + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py new file mode 100644 index 000000000000..8af4baab6a75 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import os +import re +from typing import TYPE_CHECKING + +from connector_ops.utils import ConnectorLanguage # type: ignore +from dagger import Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.helpers import git +from pipelines.helpers.connectors import cdk_helpers +from pipelines.models.steps import Step, StepResult, StepStatus + +if TYPE_CHECKING: + from typing import Optional + + from anyio import Semaphore + + +class SetCDKVersion(Step): + context: ConnectorContext + title = "Set CDK Version" + + def __init__( + self, + context: ConnectorContext, + new_version: str, + ) -> None: + super().__init__(context) + self.new_version = new_version + + async def _run(self) -> StepResult: + context = self.context + + try: + og_connector_dir = await context.get_connector_dir() + if self.context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: + updated_connector_dir = await self.upgrade_cdk_version_for_python_connector(og_connector_dir) + elif self.context.connector.language is ConnectorLanguage.JAVA: + updated_connector_dir = await self.upgrade_cdk_version_for_java_connector(og_connector_dir) + else: + return StepResult( + step=self, + status=StepStatus.FAILURE, + stderr=f"No CDK for connector {self.context.connector.technical_name} of written in {self.context.connector.language}", + ) + + if updated_connector_dir is None: + return self.skip(self.skip_reason) + diff = og_connector_dir.diff(updated_connector_dir) + exported_successfully = await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory)) + if not exported_successfully: + return StepResult( + step=self, + status=StepStatus.FAILURE, + stdout="Could not export diff to local git repo.", + ) + return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"Updated CDK version to {self.new_version}", output=diff) + except ValueError as e: + return StepResult( + step=self, + status=StepStatus.FAILURE, + stderr=f"Could not set CDK version: {e}", + exc_info=e, + ) + + async def upgrade_cdk_version_for_java_connector(self, og_connector_dir: Directory) -> Directory: + if "build.gradle" not in await og_connector_dir.entries(): + raise ValueError(f"Java connector {self.context.connector.technical_name} does not have a build.gradle file.") + + build_gradle = og_connector_dir.file("build.gradle") + build_gradle_content = await build_gradle.contents() + + old_cdk_version_required = re.search(r"cdkVersionRequired *= *'(?P[0-9]*\.[0-9]*\.[0-9]*)?'", build_gradle_content) + # If there is no airbyte-cdk dependency, add the version + if old_cdk_version_required is None: + raise ValueError("Could not find airbyte-cdk dependency in build.gradle") + + if self.new_version == "latest": + new_version = await cdk_helpers.get_latest_java_cdk_version(self.context.get_repo_dir()) + else: + new_version = self.new_version + + updated_build_gradle = build_gradle_content.replace(old_cdk_version_required.group("version"), new_version) + + use_local_cdk = re.search(r"useLocalCdk *=.*", updated_build_gradle) + if use_local_cdk is not None: + updated_build_gradle = updated_build_gradle.replace(use_local_cdk.group(), "useLocalCdk = false") + + return og_connector_dir.with_new_file("build.gradle", updated_build_gradle) + + async def upgrade_cdk_version_for_python_connector(self, og_connector_dir: Directory) -> Optional[Directory]: + context = self.context + og_connector_dir = await context.get_connector_dir() + if "setup.py" not in await og_connector_dir.entries(): + self.skip_reason = f"Python connector {self.context.connector.technical_name} does not have a setup.py file." + return None + setup_py = og_connector_dir.file("setup.py") + setup_py_content = await setup_py.contents() + + airbyte_cdk_dependency = re.search( + r"airbyte-cdk(?P\[[a-zA-Z0-9-]*\])?(?P[<>=!~]+[0-9]*(?:\.[0-9]*)?(?:\.[0-9]*)?)?", setup_py_content + ) + # If there is no airbyte-cdk dependency, add the version + if airbyte_cdk_dependency is None: + raise ValueError("Could not find airbyte-cdk dependency in setup.py") + + if self.new_version == "latest": + new_version = cdk_helpers.get_latest_python_cdk_version() + else: + new_version = self.new_version + + new_version_str = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={new_version}" + updated_setup_py = setup_py_content.replace(airbyte_cdk_dependency.group(), new_version_str) + + return og_connector_dir.with_new_file("setup.py", updated_setup_py) + + +async def run_connector_cdk_upgrade_pipeline( + context: ConnectorContext, + semaphore: Semaphore, + target_version: str, +) -> Report: + """Run a pipeline to upgrade the CDK version for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + Report: The reports holding the CDK version set results. + """ + async with semaphore: + steps_results = [] + async with context: + set_cdk_version = SetCDKVersion( + context, + target_version, + ) + set_cdk_version_result = await set_cdk_version.run() + steps_results.append(set_cdk_version_result) + report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION CDK UPGRADE RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index ec917de61575..a1f2d3cc613f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -9,11 +9,12 @@ import logging import sys +from typing import Dict, List import asyncclick as click from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter from pipelines.airbyte_ci.format.format_command import FormatCommand -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import StepStatus @@ -24,10 +25,11 @@ help="Commands related to formatting.", ) @click.option("--quiet", "-q", help="Hide details of the formatter execution.", default=False, is_flag=True) +@click_ci_requirements_option() @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def format_code(pipeline_context: ClickPipelineContext): +async def format_code(pipeline_context: ClickPipelineContext) -> None: pass @@ -35,7 +37,7 @@ async def format_code(pipeline_context: ClickPipelineContext): help="Run code format checks and fail if any checks fail.", chain=True, ) -async def check(): +async def check() -> None: pass @@ -43,17 +45,23 @@ async def check(): help="Run code format checks and fix any failures.", chain=True, ) -async def fix(): +async def fix() -> None: pass # Check and fix commands only differ in the export_formatted_code parameter value: check does not export, fix does. -FORMATTERS_CHECK_COMMANDS = { - formatter: FormatCommand(formatter, export_formatted_code=False, **config) for formatter, config in FORMATTERS_CONFIGURATIONS.items() +FORMATTERS_CHECK_COMMANDS: Dict[Formatter, FormatCommand] = { + config.formatter: FormatCommand( + config.formatter, config.file_filter, config.get_format_container_fn, config.format_commands, export_formatted_code=False + ) + for config in FORMATTERS_CONFIGURATIONS } -FORMATTERS_FIX_COMMANDS = { - formatter: FormatCommand(formatter, export_formatted_code=True, **config) for formatter, config in FORMATTERS_CONFIGURATIONS.items() +FORMATTERS_FIX_COMMANDS: Dict[Formatter, FormatCommand] = { + config.formatter: FormatCommand( + config.formatter, config.file_filter, config.get_format_container_fn, config.format_commands, export_formatted_code=True + ) + for config in FORMATTERS_CONFIGURATIONS } # Register language specific check commands @@ -67,17 +75,18 @@ async def fix(): @check.command(name="all", help="Run all format checks and fail if any checks fail.") @click.pass_context -async def all_checks(ctx: click.Context): +async def all_checks(ctx: click.Context) -> None: """ Run all format checks and fail if any checks fail. """ # We disable logging and exit on failure because its this the current command that takes care of reporting. - all_commands = [command.set_enable_logging(False).set_exit_on_failure(False) for command in FORMATTERS_CHECK_COMMANDS.values()] + all_commands: List[click.Command] = [ + command.set_enable_logging(False).set_exit_on_failure(False) for command in FORMATTERS_CHECK_COMMANDS.values() + ] command_results = await invoke_commands_concurrently(ctx, all_commands) failure = any([r.status is StepStatus.FAILURE for r in command_results]) - parent_command = ctx.parent.command - logger = logging.getLogger(parent_command.name) + logger = logging.getLogger(check.commands["all"].name) log_options = LogOptions( quiet=ctx.obj["quiet"], help_message="Run `airbyte-ci format fix all` to fix the code format.", @@ -89,23 +98,21 @@ async def all_checks(ctx: click.Context): @fix.command(name="all", help="Fix all format failures. Exits with status 1 if any file was modified.") @click.pass_context -async def all_fix(ctx: click.Context): +async def all_fix(ctx: click.Context) -> None: """Run code format checks and fix any failures.""" - parent_command = ctx.parent.command - - logger = logging.getLogger(parent_command.name) + logger = logging.getLogger(fix.commands["all"].name) # We have to run license command sequentially because it modifies the same set of files as other commands. # If we ran it concurrently with language commands, we face race condition issues. # We also want to run it before language specific formatter as they might reformat the license header. - sequential_commands = [ + sequential_commands: List[click.Command] = [ FORMATTERS_FIX_COMMANDS[Formatter.LICENSE].set_enable_logging(False).set_exit_on_failure(False), ] command_results = await invoke_commands_sequentially(ctx, sequential_commands) # We can run language commands concurrently because they modify different set of files. # We disable logging and exit on failure because its this the current command that takes care of reporting. - concurrent_commands = [ + concurrent_commands: List[click.Command] = [ FORMATTERS_FIX_COMMANDS[Formatter.JAVA].set_enable_logging(False).set_exit_on_failure(False), FORMATTERS_FIX_COMMANDS[Formatter.PYTHON].set_enable_logging(False).set_exit_on_failure(False), FORMATTERS_FIX_COMMANDS[Formatter.JS].set_enable_logging(False).set_exit_on_failure(False), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py index 5a0372568694..0ee6e5441b0b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py @@ -2,6 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from dataclasses import dataclass +from typing import Callable, List + from pipelines.airbyte_ci.format.consts import CACHE_MOUNT_PATH, LICENSE_FILE_NAME, Formatter from pipelines.airbyte_ci.format.containers import ( format_java_container, @@ -10,32 +13,41 @@ format_python_container, ) -FORMATTERS_CONFIGURATIONS = { + +@dataclass +class FormatConfiguration: + """A class to store the configuration of a formatter.""" + + formatter: Formatter + file_filter: List[str] + get_format_container_fn: Callable + format_commands: List[str] + + +FORMATTERS_CONFIGURATIONS: List[FormatConfiguration] = [ # Run spotless on all java and gradle files. - Formatter.JAVA: { - "get_format_container_fn": format_java_container, - "file_filter": ["**/*.java", "**/*.gradle"], - "format_commands": ["mvn -f spotless-maven-pom.xml spotless:apply clean"], - }, + FormatConfiguration( + Formatter.JAVA, ["**/*.java", "**/*.gradle"], format_java_container, ["mvn -f spotless-maven-pom.xml spotless:apply clean"] + ), # Run prettier on all json and yaml files. - Formatter.JS: { - "get_format_container_fn": format_js_container, - "file_filter": ["**/*.json", "**/*.yaml", "**/*.yml"], - "format_commands": [f"prettier --write . --list-different --cache --cache-location={CACHE_MOUNT_PATH}/.prettier_cache"], - }, + FormatConfiguration( + Formatter.JS, + ["**/*.json", "**/*.yaml", "**/*.yml"], + format_js_container, + [f"prettier --write . --list-different --cache --cache-location={CACHE_MOUNT_PATH}/.prettier_cache"], + ), # Add license header to java and python files. The license header is stored in LICENSE_SHORT file. - Formatter.LICENSE: { - "get_format_container_fn": format_license_container, - "file_filter": ["**/*.java", "**/*.py"], - "format_commands": [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {LICENSE_FILE_NAME} ."], - }, + FormatConfiguration( + Formatter.LICENSE, + ["**/*.java", "**/*.py"], + format_license_container, + [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {LICENSE_FILE_NAME} ."], + ), # Run isort and black on all python files. - Formatter.PYTHON: { - "get_format_container_fn": format_python_container, - "file_filter": ["**/*.py"], - "format_commands": [ - "poetry run isort --settings-file pyproject.toml .", - "poetry run black --config pyproject.toml .", - ], - }, -} + FormatConfiguration( + Formatter.PYTHON, + ["**/*.py"], + format_python_container, + ["poetry run isort --settings-file pyproject.toml .", "poetry run black --config pyproject.toml ."], + ), +] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py index 618b1bcc9ce4..37d321f0f398 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -8,47 +8,38 @@ LICENSE_FILE_NAME = "LICENSE_SHORT" +# TODO create .airbyte_ci_ignore files? DEFAULT_FORMAT_IGNORE_LIST = [ + "**/__init__.py", # These files has never been formatted and we don't want to start now (for now) see https://github.com/airbytehq/airbyte/issues/33296 "**/__pycache__", - '"**/.pytest_cache', - "**/.venv", - "**/venv", - "**/.gradle", - "**/node_modules", - "**/.tox", "**/.eggs", + "**/.git", + "**/.gradle", "**/.mypy_cache", + "**/.pytest_cache", + "**/.tox", "**/.venv", "**/*.egg-info", "**/build", - "**/dbt-project-template", + "**/charts", # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + "**/dbt_test_config", + "**/dbt-project-template-clickhouse", + "**/dbt-project-template-duckdb", "**/dbt-project-template-mssql", "**/dbt-project-template-mysql", "**/dbt-project-template-oracle", - "**/dbt-project-template-clickhouse", "**/dbt-project-template-snowflake", "**/dbt-project-template-tidb", - "**/dbt-project-template-duckdb", - "**/dbt_test_config", + "**/dbt-project-template", + "**/node_modules", + "**/pnpm-lock.yaml", # This file is generated and should not be formatted "**/normalization_test_output", - # '**/tools', - "**/secrets", - "**/charts", # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - "**/resources/seed/*_catalog.json", # Do not remove - this is also necessary to prevent diffs in our github workflows - "**/resources/seed/*_registry.json", # Do not remove - this is also necessary to prevent diffs in our github workflows - "**/resources/seed/specs_secrets_mask.yaml", # Downloaded externally. - "**/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py", "**/source-amplitude/unit_tests/api_data/zipped.json", # Zipped file presents as non-UTF-8 making spotless sad - "**/airbyte-connector-builder-server/connector_builder/generated", # autogenerated code doesn't need to be formatted - "**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # These are deliberately invalid and unformattable. - "**/__init__.py", - "**/declarative_component_schema.py", - "**/source-stock-ticker-api-tutorial/source.py", "**/tools/git_hooks/tests/test_spec_linter.py", - "**/tools/schema_generator/schema_generator/infer_schemas.py", - "**/.git", + "airbyte-cdk/python/airbyte_cdk/sources/declarative/models/**", # These files are generated and should not be formatted + "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/**", # These files are generated and should not be formatted + "**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # This is a test directory with invalid and sometimes unformatted code "airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code", # This is a test directory with badly formatted code - "airbyte-ci/connectors/pipelines/pipeline_reports", # This is a directory with generated reports that should not be formatted ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 7935c919e9a5..9528bc0f6eb9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -17,7 +17,7 @@ def build_container( dir_to_format: dagger.Directory, warmup_dir: Optional[dagger.Directory] = None, install_commands: Optional[List[str]] = None, - env_vars: Optional[Dict[str, Any]] = {}, + env_vars: Dict[str, Any] = {}, cache_volume: Optional[dagger.CacheVolume] = None, ) -> dagger.Container: """Build a container for formatting code. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py index fced3faceaf2..5787c2c67132 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py @@ -3,11 +3,9 @@ # from __future__ import annotations -import io import logging import sys -import tempfile -from typing import Any, Callable, List, Tuple +from typing import Callable, List, Tuple import asyncclick as click import dagger @@ -15,6 +13,7 @@ from pipelines.airbyte_ci.format.actions import list_files_in_directory from pipelines.airbyte_ci.format.configuration import Formatter from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST, REPO_MOUNT_PATH, WARM_UP_INCLUSIONS +from pipelines.consts import GIT_IMAGE from pipelines.helpers import sentry_utils from pipelines.helpers.cli import LogOptions, log_command_results from pipelines.helpers.utils import sh_dash_c @@ -35,10 +34,8 @@ def __init__( get_format_container_fn: Callable, format_commands: List[str], export_formatted_code: bool, - *args, enable_logging: bool = True, exit_on_failure: bool = True, - **kwargs, ) -> None: """Initialize a FormatCommand. @@ -51,7 +48,7 @@ def __init__( enable_logging (bool, optional): Make the command log its output. Defaults to True. exit_on_failure (bool, optional): Exit the process with status code 1 if the command fails. Defaults to True. """ - super().__init__(formatter.value, *args, **kwargs) + super().__init__(formatter.value) self.formatter = formatter self.file_filter = file_filter self.get_format_container_fn = get_format_container_fn @@ -76,19 +73,41 @@ def get_help_message(self) -> str: return message def get_dir_to_format(self, dagger_client: dagger.Client) -> dagger.Directory: - """Get the directory to format according to the file_filter. + """Get a directory with all the source code to format according to the file_filter. + We mount the files to format in a git container and remove all gitignored files. + It ensures we're not formatting files that are gitignored. Args: - dagger_client (dagger.Client): The dagger client to use to get the directory + dagger_client (dagger.Client): The dagger client to use to get the directory. Returns: - dagger.Directory: The directory to format + Directory: The directory with the files to format that are not gitignored. """ - return dagger_client.host().directory(self.LOCAL_REPO_PATH, include=self.file_filter, exclude=DEFAULT_FORMAT_IGNORE_LIST) + # Load a directory from the host with all the files to format according to the file_filter and the .gitignore files + dir_to_format = dagger_client.host().directory( + self.LOCAL_REPO_PATH, include=self.file_filter + ["**/.gitignore"], exclude=DEFAULT_FORMAT_IGNORE_LIST + ) + + return ( + dagger_client.container() + .from_(GIT_IMAGE) + .with_workdir(REPO_MOUNT_PATH) + .with_mounted_directory(REPO_MOUNT_PATH, dir_to_format) + # All with_exec commands below will re-run if the to_format directory changes + .with_exec(["init"]) + # Remove all gitignored files + .with_exec(["clean", "-dfqX"]) + # Delete all .gitignore files + .with_exec(sh_dash_c(['find . -type f -name ".gitignore" -exec rm {} \;']), skip_entrypoint=True) + # Delete .git + .with_exec(["rm", "-rf", ".git"], skip_entrypoint=True) + .directory(REPO_MOUNT_PATH) + .with_timestamps(0) + ) @pass_pipeline_context @sentry_utils.with_command_context - async def invoke(self, ctx: click.Context, click_pipeline_context: ClickPipelineContext) -> Any: + async def invoke(self, ctx: click.Context, click_pipeline_context: ClickPipelineContext) -> CommandResult: """Run the command. If _exit_on_failure is True, exit the process with status code 1 if the command fails. Args: @@ -98,21 +117,14 @@ async def invoke(self, ctx: click.Context, click_pipeline_context: ClickPipeline Returns: Any: The result of running the command """ - dagger_logs_file_descriptor, dagger_logs_temp_file_path = tempfile.mkstemp( - dir="/tmp", prefix=f"format_{self.formatter.value}_dagger_logs_", suffix=".log" - ) - # Create a FileIO object from the file descriptor - dagger_logs = io.FileIO(dagger_logs_file_descriptor, "w+") - self.logger.info(f"Running {self.formatter.value} formatter. Logging dagger output to {dagger_logs_temp_file_path}") - dagger_client = await click_pipeline_context.get_dagger_client( - pipeline_name=f"Format {self.formatter.value}", log_output=dagger_logs - ) + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Format {self.formatter.value}") dir_to_format = self.get_dir_to_format(dagger_client) + container = self.get_format_container_fn(dagger_client, dir_to_format) command_result = await self.get_format_command_result(dagger_client, container, dir_to_format) - if (formatted_code_dir := command_result.output_artifact) and self.export_formatted_code: + if (formatted_code_dir := command_result.output) and self.export_formatted_code: await formatted_code_dir.export(self.LOCAL_REPO_PATH) if self._enable_logging: @@ -191,7 +203,7 @@ async def get_format_command_result( if await dir_with_modified_files.entries(): modified_files = await list_files_in_directory(dagger_client, dir_with_modified_files) self.logger.debug(f"Modified files: {modified_files}") - return CommandResult(self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr, output_artifact=dir_with_modified_files) - return CommandResult(self, status=StepStatus.SUCCESS, stdout=stdout, stderr=stderr) + return CommandResult(command=self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr, output=dir_with_modified_files) + return CommandResult(command=self, status=StepStatus.SUCCESS, stdout=stdout, stderr=stderr) except dagger.ExecError as e: - return CommandResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) + return CommandResult(command=self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py index a7b92022b641..ca856d9bbb67 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py @@ -3,31 +3,36 @@ # import asyncclick as click -from pipelines.airbyte_ci.metadata.pipeline import run_metadata_orchestrator_deploy_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand # MAIN GROUP @click.group(help="Commands related to the metadata service.") +@click_ci_requirements_option() @click.pass_context -def metadata(ctx: click.Context): +def metadata(ctx: click.Context) -> None: pass @metadata.group(help="Commands related to deploying components of the metadata service.") @click.pass_context -def deploy(ctx: click.Context): +def deploy(ctx: click.Context) -> None: pass @deploy.command(cls=DaggerPipelineCommand, name="orchestrator", help="Deploy the metadata service orchestrator to production") @click.pass_context -async def deploy_orchestrator(ctx: click.Context) -> bool: +async def deploy_orchestrator(ctx: click.Context) -> None: + # Import locally to speed up CLI. + from pipelines.airbyte_ci.metadata.pipeline import run_metadata_orchestrator_deploy_pipeline + await run_metadata_orchestrator_deploy_pipeline( ctx.obj["is_local"], ctx.obj["git_branch"], ctx.obj["git_revision"], + ctx.obj["report_output_prefix"], ctx.obj.get("gha_workflow_run_url"), ctx.obj.get("dagger_logs_url"), ctx.obj.get("pipeline_start_timestamp"), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py index a1a5843bdfc6..2bd32b1fcaae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py @@ -6,13 +6,14 @@ from typing import Optional import dagger +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.airbyte_ci.steps.poetry import PoetryRunStep from pipelines.consts import DOCS_DIRECTORY_ROOT_PATH, INTERNAL_TOOL_PATHS from pipelines.dagger.actions.python.common import with_pip_packages from pipelines.dagger.containers.python import with_python_base -from pipelines.helpers.steps import run_steps +from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun, run_steps from pipelines.helpers.utils import DAGGER_CONFIG, get_secret_host_variable from pipelines.models.reports import Report from pipelines.models.steps import MountPath, Step, StepResult @@ -21,7 +22,7 @@ class MetadataValidation(SimpleDockerStep): - def __init__(self, context: ConnectorContext): + def __init__(self, context: ConnectorContext) -> None: super().__init__( title=f"Validate metadata for {context.connector.technical_name}", context=context, @@ -63,7 +64,7 @@ def __init__( docker_hub_password_secret: dagger.Secret, pre_release: bool = False, pre_release_tag: Optional[str] = None, - ): + ) -> None: title = f"Upload metadata for {context.connector.technical_name} v{context.connector.version}" command_to_run = [ "metadata_service", @@ -73,7 +74,7 @@ def __init__( metadata_bucket_name, ] - if pre_release: + if pre_release and pre_release_tag: command_to_run += ["--prerelease", pre_release_tag] super().__init__( @@ -122,7 +123,7 @@ async def _run(self) -> StepResult: # mount metadata_service/lib and metadata_service/orchestrator parent_dir = self.context.get_repo_dir("airbyte-ci/connectors/metadata_service") python_base = with_python_base(self.context, "3.9") - python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.2.6", "pydantic==1.10.6", "poetry2setup==1.1.0"]) + python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.5.14", "poetry2setup==1.1.0"]) dagster_cloud_api_token_secret: dagger.Secret = get_secret_host_variable( self.context.dagger_client, "DAGSTER_CLOUD_METADATA_API_TOKEN" ) @@ -130,7 +131,7 @@ async def _run(self) -> StepResult: container_to_run = ( python_with_dependencies.with_mounted_directory("/src", parent_dir) .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret) - .with_workdir(f"/src/orchestrator") + .with_workdir("/src/orchestrator") .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"]) .with_exec(self.deploy_dagster_command) ) @@ -138,17 +139,15 @@ async def _run(self) -> StepResult: class TestOrchestrator(PoetryRunStep): - def __init__(self, context: PipelineContext): + def __init__(self, context: PipelineContext) -> None: super().__init__( context=context, title="Test Metadata Orchestrator", parent_dir_path="airbyte-ci/connectors/metadata_service", module_path="orchestrator", + poetry_run_args=["pytest"], ) - async def _run(self) -> StepResult: - return await super()._run(["pytest"]) - # PIPELINES @@ -157,29 +156,50 @@ async def run_metadata_orchestrator_deploy_pipeline( is_local: bool, git_branch: str, git_revision: str, + report_output_prefix: str, gha_workflow_run_url: Optional[str], dagger_logs_url: Optional[str], pipeline_start_timestamp: Optional[int], ci_context: Optional[str], ) -> bool: + success: bool = False + metadata_pipeline_context = PipelineContext( pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", is_local=is_local, git_branch=git_branch, git_revision=git_revision, + report_output_prefix=report_output_prefix, gha_workflow_run_url=gha_workflow_run_url, dagger_logs_url=dagger_logs_url, pipeline_start_timestamp=pipeline_start_timestamp, ci_context=ci_context, ) - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) async with metadata_pipeline_context: - steps = [TestOrchestrator(context=metadata_pipeline_context), DeployOrchestrator(context=metadata_pipeline_context)] + steps: STEP_TREE = [ + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.TEST_ORCHESTRATOR, + step=TestOrchestrator(context=metadata_pipeline_context), + ) + ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.DEPLOY_ORCHESTRATOR, + step=DeployOrchestrator(context=metadata_pipeline_context), + depends_on=[CONNECTOR_TEST_STEP_ID.TEST_ORCHESTRATOR], + ) + ], + ] steps_results = await run_steps(steps) - metadata_pipeline_context.report = Report( - pipeline_context=metadata_pipeline_context, steps_results=steps_results, name="METADATA ORCHESTRATOR DEPLOY RESULTS" + report = Report( + pipeline_context=metadata_pipeline_context, + steps_results=list(steps_results.values()), + name="METADATA ORCHESTRATOR DEPLOY RESULTS", ) - return metadata_pipeline_context.report.success + metadata_pipeline_context.report = report + success = report.success + return success diff --git a/octavia-cli/octavia_cli/get/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py similarity index 100% rename from octavia-cli/octavia_cli/get/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py new file mode 100644 index 000000000000..72dbe53b170f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +import asyncclick as click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + + +@click.group( + name="poetry", + help="Commands related to running poetry commands.", + cls=LazyGroup, + lazy_subcommands={ + "publish": "pipelines.airbyte_ci.poetry.publish.commands.publish", + }, +) +@click.option( + "--package-path", + help="The path to publish", + type=click.STRING, + required=True, +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def poetry(pipeline_context: ClickPipelineContext) -> None: + pass diff --git a/octavia-cli/octavia_cli/init/example_files/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py similarity index 100% rename from octavia-cli/octavia_cli/init/example_files/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py new file mode 100644 index 000000000000..0eac52947bf1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -0,0 +1,107 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +from typing import Optional + +import asyncclick as click +from packaging import version +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry +from pipelines.cli.confirm_prompt import confirm +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext +from pipelines.models.steps import StepStatus + + +async def _has_metadata_yaml(context: PythonRegistryPublishContext) -> bool: + dir_to_publish = context.get_repo_dir(context.package_path) + return "metadata.yaml" in await dir_to_publish.entries() + + +def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> Optional[str]: + """ + Check if an given version is valid. + """ + if value is None: + return value + try: + version.Version(value) + return value + except version.InvalidVersion: + raise click.BadParameter(f"Version {value} is not a valid version.") + + +@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to a registry.") +@click.option( + "--python-registry-token", + help="Access token", + type=click.STRING, + required=True, + envvar="PYTHON_REGISTRY_TOKEN", +) +@click.option( + "--python-registry-url", + help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + envvar="PYTHON_REGISTRY_URL", +) +@click.option( + "--publish-name", + help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.", + type=click.STRING, +) +@click.option( + "--publish-version", + help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", + type=click.STRING, + callback=_validate_python_version, +) +@pass_pipeline_context +@click.pass_context +async def publish( + ctx: click.Context, + click_pipeline_context: ClickPipelineContext, + python_registry_token: str, + python_registry_url: str, + publish_name: Optional[str], + publish_version: Optional[str], +) -> bool: + context = PythonRegistryPublishContext( + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + python_registry_token=python_registry_token, + registry=python_registry_url, + registry_check_url=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, + package_path=ctx.obj["package_path"], + package_name=publish_name, + version=publish_version, + ) + + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to python registry") + context.dagger_client = dagger_client + + if await _has_metadata_yaml(context): + confirm( + "It looks like you are trying to publish a connector. In most cases, the `connectors` command group should be used instead. Do you want to continue?", + abort=True, + ) + + publish_result = await PublishToPythonRegistry(context).run() + + return publish_result.status is StepStatus.SUCCESS diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py index 964ee2c93b3f..71c692c37fae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -22,7 +22,7 @@ def __init__( env_variables: dict[str, str] = {}, working_directory: str = "/", command: Optional[List[str]] = None, - ): + ) -> None: """A simple step that runs a given command in a container. Args: @@ -35,7 +35,7 @@ def __init__( working_directory (str, optional): working directory to run the command in. Defaults to "/". command (Optional[List[str]], optional): The default command to run. Defaults to None. """ - self.title = title + self._title = title super().__init__(context) self.paths_to_mount = paths_to_mount @@ -45,9 +45,13 @@ def __init__( self.env_variables = env_variables self.command = command + @property + def title(self) -> str: + return self._title + def _mount_paths(self, container: dagger.Container) -> dagger.Container: for path_to_mount in self.paths_to_mount: - if path_to_mount.optional and not path_to_mount.path.exists(): + if path_to_mount.optional and not path_to_mount.get_path().exists(): continue path_string = str(path_to_mount) @@ -89,7 +93,7 @@ async def init_container(self) -> dagger.Container: return container - async def _run(self, command=None) -> StepResult: + async def _run(self, command: Optional[List[str]] = None) -> StepResult: command_to_run = command or self.command if not command_to_run: raise ValueError(f"No command given to the {self.title} step") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/git.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/git.py deleted file mode 100644 index 973d87afe4d4..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/git.py +++ /dev/null @@ -1,146 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dagger import Client, Container, Directory, Secret -from pipelines.helpers.github import AIRBYTE_GITHUB_REPO -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.steps import Step, StepResult - - -def with_git(dagger_client, ci_git_user: str = "octavia") -> Container: - return ( - dagger_client.container() - .from_("alpine:latest") - .with_exec( - sh_dash_c( - [ - "apk update", - "apk add git tar wget", - f"git config --global user.email {ci_git_user}@users.noreply.github.com", - f"git config --global user.name {ci_git_user}", - "git config --global --add --bool push.autoSetupRemote true", - ] - ) - ) - .with_workdir("/ghcli") - .with_exec( - sh_dash_c( - [ - "wget https://github.com/cli/cli/releases/download/v2.30.0/gh_2.30.0_linux_amd64.tar.gz -O ghcli.tar.gz", - "tar --strip-components=1 -xf ghcli.tar.gz", - "rm ghcli.tar.gz", - "cp bin/gh /usr/local/bin/gh", - ] - ) - ) - ) - - -class GitPushChanges(Step): - """ - A step to push changes to the remote repository. - """ - - title = "Push changes to the remote repository" - - GITHUB_REPO_URL = f"https://github.com/{AIRBYTE_GITHUB_REPO}.git" - - @property - def ci_git_user(self) -> str: - return self.context.ci_git_user - - @property - def ci_github_access_token(self) -> str: - return self.context.ci_github_access_token - - @property - def dagger_client(self) -> Client: - return self.context.dagger_client - - @property - def git_branch(self) -> str: - return self.context.git_branch - - @property - def authenticated_repo_url(self) -> Secret: - url = self.GITHUB_REPO_URL.replace("https://", f"https://{self.ci_git_user}:{self.ci_github_access_token}@") - return self.dagger_client.set_secret("authenticated_repo_url", url) - - @property - def airbyte_repo(self) -> Directory: - return self.dagger_client.git(self.GITHUB_REPO_URL, keep_git_dir=True).branch(self.git_branch).tree() - - def get_commit_message(self, commit_message: str, skip_ci: bool) -> str: - commit_message = f"🤖 {commit_message}" - return f"{commit_message} [skip ci]" if skip_ci else commit_message - - async def _run( - self, changed_directory: Directory, changed_directory_path: str, commit_message: str, skip_ci: bool = True - ) -> StepResult: - diff = ( - with_git(self.dagger_client, self.context.ci_github_access_token_secret, self.ci_git_user) - .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) - .with_mounted_directory("/airbyte", self.airbyte_repo) - .with_workdir("/airbyte") - .with_exec(["git", "checkout", self.git_branch]) - .with_mounted_directory(f"/airbyte/{changed_directory_path}", changed_directory) - .with_exec(["git", "diff", "--name-only"]) - ) - - if not await diff.stdout(): - return self.skip("No changes to push") - - commit_and_push = ( - diff.with_exec(["sh", "-c", "git remote set-url origin $AUTHENTICATED_REPO_URL"]) - .with_exec(["git", "add", "."]) - .with_exec(["git", "commit", "-m", self.get_commit_message(commit_message, skip_ci)]) - .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) - .with_exec(["git", "push"]) - ) - return await self.get_step_result(commit_and_push) - - -class GitPushEmptyCommit(GitPushChanges): - """ - A step to push an empty commit to the remote repository. - """ - - title = "Push empty commit to the remote repository" - - def __init__(self, dagger_client, ci_git_user, ci_github_access_token, git_branch): - self._dagger_client = dagger_client - self._ci_github_access_token = ci_github_access_token - self._ci_git_user = ci_git_user - self._git_branch = git_branch - self.ci_github_access_token_secret = dagger_client.set_secret("ci_github_access_token", ci_github_access_token) - - @property - def dagger_client(self) -> Client: - return self._dagger_client - - @property - def ci_git_user(self) -> str: - return self._ci_git_user - - @property - def ci_github_access_token(self) -> Secret: - return self._ci_github_access_token - - @property - def git_branch(self) -> str: - return self._git_branch - - async def _run(self, commit_message: str, skip_ci: bool = True) -> StepResult: - push_empty_commit = ( - with_git(self.dagger_client, self.ci_github_access_token_secret, self.ci_git_user) - .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) - .with_mounted_directory("/airbyte", self.airbyte_repo) - .with_workdir("/airbyte") - .with_exec(["git", "checkout", self.git_branch]) - .with_exec(sh_dash_c(["git remote set-url origin $AUTHENTICATED_REPO_URL"])) - .with_exec(["git", "commit", "--allow-empty", "-m", self.get_commit_message(commit_message, skip_ci)]) - .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) - .with_exec(["git", "push"]) - ) - return await self.get_step_result(push_empty_commit) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 0f04429afb34..456e6c3f1aa3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -1,16 +1,18 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from abc import ABC -from typing import ClassVar, List +from datetime import datetime +from typing import Any, ClassVar, List, Optional, Tuple, cast import pipelines.dagger.actions.system.docker -from dagger import CacheSharingMode, CacheVolume +from dagger import CacheSharingMode, CacheVolume, Container, ExecError +from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.hacks import never_fail_exec +from pipelines.helpers.utils import dagger_directory_as_zip_file, sh_dash_c +from pipelines.models.artifacts import Artifact from pipelines.models.steps import Step, StepResult @@ -25,17 +27,20 @@ class GradleTask(Step, ABC): mount_connector_secrets (bool): Whether to mount connector secrets. """ - DEFAULT_GRADLE_TASK_OPTIONS = ("--no-daemon", "--no-watch-fs", "--scan", "--build-cache", "--console=plain") - LOCAL_MAVEN_REPOSITORY_PATH = "/root/.m2" + context: ConnectorContext + GRADLE_DEP_CACHE_PATH = "/root/gradle-cache" GRADLE_HOME_PATH = "/root/.gradle" - + STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain") gradle_task_name: ClassVar[str] bind_to_docker_host: ClassVar[bool] = False mount_connector_secrets: ClassVar[bool] = False + with_test_artifacts: ClassVar[bool] = False + accept_extra_params = True - def __init__(self, context: PipelineContext) -> None: - super().__init__(context) + @property + def gradle_task_options(self) -> Tuple[str, ...]: + return self.STATIC_GRADLE_OPTIONS + (f"-Ds3BuildCachePrefix={self.context.connector.technical_name}",) @property def dependency_cache_volume(self) -> CacheVolume: @@ -56,10 +61,11 @@ def build_include(self) -> List[str]: for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True) ] - def _get_gradle_command(self, task: str, *args) -> str: - return f"./gradlew {' '.join(self.DEFAULT_GRADLE_TASK_OPTIONS + args)} {task}" + def _get_gradle_command(self, task: str, *args: Any, task_options: Optional[List[str]] = None) -> str: + task_options = task_options or [] + return f"./gradlew {' '.join(self.gradle_task_options + args)} {task} {' '.join(task_options)}" - async def _run(self) -> StepResult: + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: include = [ ".root", ".env", @@ -123,11 +129,11 @@ async def _run(self) -> StepResult: ) # Augment the base container with S3 build cache secrets when available. - if self.context.s3_build_cache_access_key_id: + if self.context.s3_build_cache_access_key_id_secret: gradle_container_base = gradle_container_base.with_secret_variable( "S3_BUILD_CACHE_ACCESS_KEY_ID", self.context.s3_build_cache_access_key_id_secret ) - if self.context.s3_build_cache_secret_key: + if self.context.s3_build_cache_secret_key_secret: gradle_container_base = gradle_container_base.with_secret_variable( "S3_BUILD_CACHE_SECRET_KEY", self.context.s3_build_cache_secret_key_secret ) @@ -139,23 +145,21 @@ async def _run(self) -> StepResult: # When running locally, this dependency update is slower and less useful than within a CI runner. Skip it. warm_dependency_cache_args = ["--dry-run"] - # Mount the whole git repo to update the cache volume contents and build the CDK. + # Mount the whole git repo to update the cache volume contents. with_whole_git_repo = ( gradle_container_base # Mount the whole repo. .with_directory("/airbyte", self.context.get_repo_dir(".")) - # Update the cache in place by executing a gradle task which will update all dependencies and build the CDK. + # Update the cache in place by executing a gradle task which will update all dependencies. .with_exec( sh_dash_c( [ - # Ensure that the .m2 directory exists. - f"mkdir -p {self.LOCAL_MAVEN_REPOSITORY_PATH}", + # Defensively delete the gradle home directory to avoid dirtying the cache volume. + f"rm -rf {self.GRADLE_HOME_PATH}", # Load from the cache volume. f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)", # Resolve all dependencies and write their checksums to './gradle/verification-metadata.dryrun.xml'. self._get_gradle_command("help", *warm_dependency_cache_args), - # Build the CDK and publish it to the local maven repository. - self._get_gradle_command(":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"), # Store to the cache volume. f"(rsync -a --stats {self.GRADLE_HOME_PATH}/ {self.GRADLE_DEP_CACHE_PATH} || true)", ] @@ -166,8 +170,8 @@ async def _run(self) -> StepResult: # Mount only the code needed to build the connector. gradle_container = ( gradle_container_base - # Copy the local maven repository and force evaluation of `with_whole_git_repo` container. - .with_directory(self.LOCAL_MAVEN_REPOSITORY_PATH, await with_whole_git_repo.directory(self.LOCAL_MAVEN_REPOSITORY_PATH)) + # Copy the gradle home directory and force evaluation of `with_whole_git_repo` container. + .with_directory(self.GRADLE_HOME_PATH, await with_whole_git_repo.directory(self.GRADLE_HOME_PATH)) # Mount the connector-agnostic whitelisted files in the git repo. .with_mounted_directory("/airbyte", self.context.get_repo_dir(".", include=include)) # Mount the sources for the connector and its dependencies in the git repo. @@ -185,15 +189,97 @@ async def _run(self) -> StepResult: gradle_container = gradle_container.with_exec(["yum", "install", "-y", "docker"]) # Run the gradle task that we actually care about. - connector_task = f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}" - gradle_container = gradle_container.with_exec( - sh_dash_c( - [ - # Warm the gradle cache. - f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)", - # Run the gradle task. - self._get_gradle_command(connector_task, f"-Ds3BuildCachePrefix={self.context.connector.technical_name}"), - ] + connector_gradle_task = f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}" + gradle_command = self._get_gradle_command(connector_gradle_task, task_options=self.params_as_cli_options) + gradle_container = gradle_container.with_(never_fail_exec([gradle_command])) + + # Collect the test artifacts, if applicable. + artifacts = [] + if self.with_test_artifacts: + if test_logs := await self._collect_test_logs(gradle_container): + artifacts.append(test_logs) + if test_results := await self._collect_test_results(gradle_container): + artifacts.append(test_results) + + return await self.get_step_result(gradle_container, artifacts) + + async def get_step_result(self, container: Container, outputs: List[Artifact]) -> StepResult: + step_result = await super().get_step_result(container) + # Decorate with test report, if applicable. + return StepResult( + step=step_result.step, + status=step_result.status, + stdout=step_result.stdout, + stderr=step_result.stderr, + output=step_result.output, + artifacts=outputs, + ) + + async def _collect_test_logs(self, gradle_container: Container) -> Optional[Artifact]: + """ + Exports the java docs from the container into the host filesystem. + The docs in the container are expected to be in build/test-logs, and will end up test-artifact directory by default + One can change the destination directory by setting the outputs + """ + test_logs_dir_name_in_container = "test-logs" + test_logs_dir_name_in_zip = f"test-logs-{datetime.fromtimestamp(cast(float, self.context.pipeline_start_timestamp)).isoformat()}-{self.context.git_branch}-{self.gradle_task_name}".replace( + "/", "_" + ) + if ( + test_logs_dir_name_in_container + not in await gradle_container.directory(f"{self.context.connector.code_directory}/build").entries() + ): + self.context.logger.warn(f"No {test_logs_dir_name_in_container} found directory in the build folder") + return None + try: + zip_file = await ( + dagger_directory_as_zip_file( + self.dagger_client, + await gradle_container.directory(f"{self.context.connector.code_directory}/build/{test_logs_dir_name_in_container}"), + test_logs_dir_name_in_zip, + ) ) + return Artifact( + name=f"{test_logs_dir_name_in_zip}.zip", + content=zip_file, + content_type="application/zip", + to_upload=True, + ) + except ExecError as e: + self.context.logger.error(str(e)) + return None + + async def _collect_test_results(self, gradle_container: Container) -> Optional[Artifact]: + """ + Exports the junit test reports from the container into the host filesystem. + The docs in the container are expected to be in build/test-results, and will end up test-artifact directory by default + Only the XML files generated by junit are downloaded into the host filesystem + One can change the destination directory by setting the outputs + """ + test_results_dir_name_in_container = "test-results" + test_results_dir_name_in_zip = f"test-results-{datetime.fromtimestamp(cast(float, self.context.pipeline_start_timestamp)).isoformat()}-{self.context.git_branch}-{self.gradle_task_name}".replace( + "/", "_" ) - return await self.get_step_result(gradle_container) + if ( + test_results_dir_name_in_container + not in await gradle_container.directory(f"{self.context.connector.code_directory}/build").entries() + ): + self.context.logger.warn(f"No {test_results_dir_name_in_container} found directory in the build folder") + return None + try: + zip_file = await ( + dagger_directory_as_zip_file( + self.dagger_client, + await gradle_container.directory(f"{self.context.connector.code_directory}/build/{test_results_dir_name_in_container}"), + test_results_dir_name_in_zip, + ) + ) + return Artifact( + name=f"{test_results_dir_name_in_zip}.zip", + content=zip_file, + content_type="application/zip", + to_upload=True, + ) + except ExecError as e: + self.context.logger.error(str(e)) + return None diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py index 23e70650f405..86b9712713a3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any + from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus @@ -16,5 +18,5 @@ def __init__(self, context: PipelineContext, step_status: StepStatus) -> None: super().__init__(context) self.step_status = step_status - async def _run(self, *args, **kwargs) -> StepResult: - return StepResult(self, self.step_status) + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: + return StepResult(step=self, status=self.step_status) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py index f7140f1e2ccd..43cb05ead074 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py @@ -2,13 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import List + from pipelines.dagger.actions.python.poetry import with_poetry_module from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult class PoetryRunStep(Step): - def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str): + def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str, poetry_run_args: List[str]) -> None: """A simple step that runs a given command inside a poetry project. Args: @@ -16,14 +18,20 @@ def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, m title (str): name of the step parent_dir_path (str): The path to the parent directory of the poetry project module_path (str): The path to the poetry project + poetry_run_args (List[str]): The arguments to pass to the poetry run command """ - self.title = title + self._title = title super().__init__(context) parent_dir = self.context.get_repo_dir(parent_dir_path) module_path = module_path + self.poetry_run_args = poetry_run_args self.poetry_run_container = with_poetry_module(self.context, parent_dir, module_path).with_entrypoint(["poetry", "run"]) - async def _run(self, poetry_run_args: list) -> StepResult: - poetry_run_exec = self.poetry_run_container.with_exec(poetry_run_args) + @property + def title(self) -> str: + return self._title + + async def _run(self) -> StepResult: + poetry_run_exec = self.poetry_run_container.with_exec(self.poetry_run_args) return await self.get_step_result(poetry_run_exec) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py new file mode 100644 index 000000000000..2bfebec127b5 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -0,0 +1,169 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import configparser +import io +import uuid +from enum import Enum, auto +from typing import Dict, Optional + +import tomli +import tomli_w +from dagger import Container, Directory +from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext +from pipelines.models.steps import Step, StepResult + + +class PackageType(Enum): + POETRY = auto() + PIP = auto() + + +class PublishToPythonRegistry(Step): + context: PythonRegistryPublishContext + title = "Publish package to python registry" + + def _get_base_container(self) -> Container: + return with_poetry(self.context) + + async def _get_package_metadata_from_pyproject_toml(self, package_dir_to_publish: Directory) -> Optional[PythonPackageMetadata]: + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + try: + return PythonPackageMetadata(contents["tool"]["poetry"]["name"], contents["tool"]["poetry"]["version"]) + except KeyError: + return None + + async def _get_package_type(self, package_dir_to_publish: Directory) -> Optional[PackageType]: + files = await package_dir_to_publish.entries() + has_pyproject_toml = PYPROJECT_TOML_FILE_PATH in files + has_setup_py = SETUP_PY_FILE_PATH in files + if has_pyproject_toml: + return PackageType.POETRY + elif has_setup_py: + return PackageType.PIP + else: + return None + + async def _run(self) -> StepResult: + package_dir_to_publish = await self.context.get_repo_dir(self.context.package_path) + package_type = await self._get_package_type(package_dir_to_publish) + + if not package_type: + return self.skip("Connector does not have a pyproject.toml file or setup.py file, skipping.") + + result = await self._ensure_package_name_and_version(package_dir_to_publish, package_type) + if result: + return result + + self.logger.info( + f"Uploading package {self.context.package_metadata.name} version {self.context.package_metadata.version} to {self.context.registry}..." + ) + + return await self._publish(package_dir_to_publish, package_type) + + async def _ensure_package_name_and_version(self, package_dir_to_publish: Directory, package_type: PackageType) -> Optional[StepResult]: + """ + Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set. + Setup.py packages need to set package name and version as parameter. + + Returns None if package name and version are set, otherwise a StepResult with a skip message. + """ + if self.context.package_metadata.name and self.context.package_metadata.version: + return None + + if package_type is not PackageType.POETRY: + return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") + + inferred_package_metadata = await self._get_package_metadata_from_pyproject_toml(package_dir_to_publish) + + if not inferred_package_metadata: + return self.skip( + "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." + ) + + if not self.context.package_metadata.name: + self.context.package_metadata.name = inferred_package_metadata.name + if not self.context.package_metadata.version: + self.context.package_metadata.version = inferred_package_metadata.version + + return None + + async def _publish(self, package_dir_to_publish: Directory, package_type: PackageType) -> StepResult: + if package_type is PackageType.PIP: + return await self._pip_publish(package_dir_to_publish) + else: + return await self._poetry_publish(package_dir_to_publish) + + async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult: + python_registry_token = self.context.dagger_client.set_secret("python_registry_token", self.context.python_registry_token) + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + # make sure package name and version are set to the configured one + contents["tool"]["poetry"]["name"] = self.context.package_metadata.name + contents["tool"]["poetry"]["version"] = self.context.package_metadata.version + # enforce consistent author + contents["tool"]["poetry"]["authors"] = ["Airbyte "] + poetry_publish = ( + self._get_base_container() + .with_secret_variable("PYTHON_REGISTRY_TOKEN", python_registry_token) + .with_directory("package", package_dir_to_publish) + .with_workdir("package") + .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) + # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry) + # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) + .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"])) + .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) + ) + + return await self.get_step_result(poetry_publish) + + async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult: + files = await package_dir_to_publish.entries() + pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") + pypi_password = self.context.dagger_client.set_secret("pypi_password", self.context.python_registry_token) + metadata: Dict[str, str] = { + "name": str(self.context.package_metadata.name), + "version": str(self.context.package_metadata.version), + # Enforce consistent author + "author": "Airbyte", + "author_email": "contact@airbyte.io", + } + if "README.md" in files: + metadata["long_description"] = await package_dir_to_publish.file("README.md").contents() + metadata["long_description_content_type"] = "text/markdown" + + config = configparser.ConfigParser() + config["metadata"] = metadata + + setup_cfg_io = io.StringIO() + config.write(setup_cfg_io) + setup_cfg = setup_cfg_io.getvalue() + + twine_upload = ( + self._get_base_container() + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y twine"])) + .with_directory("package", package_dir_to_publish) + .with_workdir("package") + # clear out setup.py metadata so setup.cfg is used + .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", SETUP_PY_FILE_PATH]) + .with_new_file("setup.cfg", contents=setup_cfg) + .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) + .with_exec(["python", SETUP_PY_FILE_PATH, "sdist", "bdist_wheel"]) + # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry) + # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_secret_variable("TWINE_USERNAME", pypi_username) + .with_secret_variable("TWINE_PASSWORD", pypi_password) + .with_exec(["twine", "upload", "--verbose", "--repository-url", self.context.registry, "dist/*"]) + ) + + return await self.get_step_result(twine_upload) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py index c941b3045795..882d61c8f7c7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py @@ -1,3 +1,20 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from pathlib import Path + +INTERNAL_POETRY_PACKAGES = [ + "airbyte-lib", + "airbyte-ci/connectors/pipelines", + "airbyte-ci/connectors/base_images", + "airbyte-ci/connectors/common_utils", + "airbyte-ci/connectors/connector_ops", + "airbyte-ci/connectors/connectors_qa", + "airbyte-ci/connectors/ci_credentials", + "airbyte-ci/connectors/metadata_service/lib", + "airbyte-ci/connectors/metadata_service/orchestrator", + "airbyte-integrations/bases/connector-acceptance-test" +] + +INTERNAL_POETRY_PACKAGES_PATH = [Path(package) for package in INTERNAL_POETRY_PACKAGES] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 1fe614d7a841..a79e93640862 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -1,83 +1,129 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations -import logging +from typing import TYPE_CHECKING import asyncclick as click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.consts import DOCKER_VERSION -from pipelines.helpers.utils import sh_dash_c +import asyncer +from pipelines.airbyte_ci.test import INTERNAL_POETRY_PACKAGES, INTERNAL_POETRY_PACKAGES_PATH, pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.git import get_modified_files +from pipelines.helpers.utils import transform_strs_to_paths from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import StepStatus + +if TYPE_CHECKING: + from pathlib import Path + from typing import List, Set, Tuple + + +async def find_modified_internal_packages(pipeline_context: ClickPipelineContext) -> Set[Path]: + """Finds the modified internal packages according to the modified files on the branch/commit. + + Args: + pipeline_context (ClickPipelineContext): The context object. + + Returns: + Set[Path]: The set of modified internal packages. + """ + modified_files = transform_strs_to_paths( + await get_modified_files( + pipeline_context.params["git_branch"], + pipeline_context.params["git_revision"], + pipeline_context.params["diffed_branch"], + pipeline_context.params["is_local"], + pipeline_context.params["ci_context"], + ) + ) + modified_packages = set() + for modified_file in modified_files: + for internal_package in INTERNAL_POETRY_PACKAGES_PATH: + if modified_file.is_relative_to(internal_package): + modified_packages.add(internal_package) + return modified_packages + + +async def get_packages_to_run(pipeline_context: ClickPipelineContext) -> Set[Path]: + """Gets the packages to run the poe tasks on. + + Args: + pipeline_context (ClickPipelineContext): The context object. + + Raises: + click.ClickException: If no packages are specified to run the poe tasks on. + + Returns: + Set[Path]: The set of packages to run the poe tasks on. + """ + if not pipeline_context.params["poetry_package_paths"] and not pipeline_context.params["modified"]: + raise click.ClickException("You must specify at least one package to test.") + + poetry_package_paths = set() + if pipeline_context.params["modified"]: + poetry_package_paths = await find_modified_internal_packages(pipeline_context) + + return poetry_package_paths.union(set(pipeline_context.params["poetry_package_paths"])) + + +def crash_on_any_failure(poetry_package_poe_tasks_results: List[Tuple[Path, asyncer.SoonValue]]) -> None: + """Fail the command if any of the poe tasks failed. + + Args: + poetry_package_poe_tasks_results (List[Tuple[Path, asyncer.SoonValue]]): The results of the poe tasks. + + Raises: + click.ClickException: If any of the poe tasks failed. + """ + failed_packages = set() + for poetry_package_paths, package_result in poetry_package_poe_tasks_results: + poe_command_results = package_result.value + if any([result.status is StepStatus.FAILURE for result in poe_command_results]): + failed_packages.add(poetry_package_paths) + if failed_packages: + raise click.ClickException( + f"The following packages failed to run poe tasks: {', '.join([str(package_path) for package_path in failed_packages])}" + ) + return None @click.command() -@click.argument("poetry_package_path") -@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +@click.option("--modified", default=False, is_flag=True, help="Run on modified internal packages.") +@click.option( + "--poetry-package-path", + "-p", + "poetry_package_paths", + help="The path to the poetry package to test.", + type=click.Choice(INTERNAL_POETRY_PACKAGES), + multiple=True, +) +@click_ci_requirements_option() @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def test(pipeline_context: ClickPipelineContext): +# TODO this command should be renamed ci and go under the poetry command group +# e.g. airbyte-ci poetry ci --poetry-package-path airbyte-ci/connectors/pipelines +async def test(pipeline_context: ClickPipelineContext) -> None: """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. """ - poetry_package_path = pipeline_context.params["poetry_package_path"] - test_directory = pipeline_context.params["test_directory"] - - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [ - ".git", # This is needed as some package tests rely on being in a git repo - ".github", - "docs", - "airbyte-integrations", - "airbyte-ci", - "airbyte-cdk", - "pyproject.toml", - "LICENSE_SHORT", - "poetry.lock", - "spotless-maven-pom.xml", - "tools/gradle/codestyle/java-google-style.xml", - ] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - - pipeline_name = f"Unit tests for {poetry_package_path}" - dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name) - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] + poetry_package_paths = await get_packages_to_run(pipeline_context) + click.echo(f"Running poe tasks of the following packages: {', '.join([str(package_path) for package_path in poetry_package_paths])}") + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Internal poetry packages CI") + + poetry_package_poe_tasks_results: List[Tuple[Path, asyncer.SoonValue]] = [] + async with asyncer.create_task_group() as poetry_packages_task_group: + for poetry_package_path in poetry_package_paths: + poetry_package_poe_tasks_results.append( + ( + poetry_package_path, + poetry_packages_task_group.soonify(pipeline.run_poe_tasks_for_package)( + dagger_client, poetry_package_path, pipeline_context.params + ), + ) ) - ) - .with_env_variable("VERSION", DOCKER_VERSION) - .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_exec(["poetry", "config", "virtualenvs.create", "false"]) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_env_variable("CI", str(pipeline_context.params["is_ci"])) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - await pytest_container + crash_on_any_failure(poetry_package_poe_tasks_results) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py new file mode 100644 index 000000000000..fd4ed95de5ab --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +from typing import Dict, Set + +from pydantic import BaseModel, Field, validator + + +class AirbyteCiPackageConfiguration(BaseModel): + poe_tasks: Set[str] = Field(..., description="List of unique poe tasks to run") + required_environment_variables: Set[str] = Field( + set(), description="List of unique required environment variables to pass to the container running the poe task" + ) + extra_poetry_groups: Set[str] = Field(set(), description="List of unique extra poetry groups to install") + side_car_docker_engine: bool = Field( + False, description="Flag indicating the use of a sidecar Docker engine during the poe task executions" + ) + mount_docker_socket: bool = Field( + False, + description="Flag indicating the mount of the host docker socket to the container running the poe task, useful when the package under test is using dagger", + ) + + @validator("required_environment_variables") + def check_required_environment_variables_are_set(cls, value: Set) -> Set: + for required_env_var in value: + if required_env_var not in os.environ: + raise ValueError(f"Environment variable {required_env_var} is not set.") + return value + + +def deserialize_airbyte_ci_config(pyproject_toml: Dict) -> AirbyteCiPackageConfiguration: + try: + airbyte_ci_config = pyproject_toml["tool"]["airbyte_ci"] + except KeyError: + raise ValueError("Missing tool.airbyte_ci configuration in pyproject.toml") + return AirbyteCiPackageConfiguration.parse_obj(airbyte_ci_config) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py new file mode 100644 index 000000000000..f8ada153f44c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py @@ -0,0 +1,289 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING + +import asyncer +import dagger +import toml +from pipelines.airbyte_ci.test.models import deserialize_airbyte_ci_config +from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_VERSION, POETRY_CACHE_VOLUME_NAME, PYPROJECT_TOML_FILE_PATH +from pipelines.dagger.actions.system import docker +from pipelines.helpers.github import update_commit_status_check +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.steps import PoeTaskResult, StepStatus + +if TYPE_CHECKING: + from logging import Logger + from pathlib import Path + from typing import Dict, List + + from pipelines.airbyte_ci.test.models import AirbyteCiPackageConfiguration + +# The following directories are always mounted because a lot of tests rely on them +DIRECTORIES_TO_ALWAYS_MOUNT = [ + ".git", # This is needed as some package tests rely on being in a git repo + ".github", + "docs", + "airbyte-integrations", + "airbyte-ci", + "airbyte-cdk", + PYPROJECT_TOML_FILE_PATH, + "LICENSE_SHORT", + "poetry.lock", + "spotless-maven-pom.xml", + "tools/gradle/codestyle/java-google-style.xml", +] + +DEFAULT_EXCLUDE = ["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"] + + +async def get_filtered_airbyte_repo_dir(dagger_client: dagger.Client, poetry_package_path: Path) -> dagger.Directory: + """Get a filtered airbyte repo directory with the directories to always mount and the poetry package path. + + Args: + dagger_client (dagger.Client): Dagger client. + poetry_package_path (Path): Path to the poetry package in the airbyte repo. + + Returns: + dagger.Directory: The filtered airbyte repo directory. + """ + directories_to_mount = list(set([str(poetry_package_path), *DIRECTORIES_TO_ALWAYS_MOUNT])) + return dagger_client.host().directory( + ".", + exclude=DEFAULT_EXCLUDE, + include=directories_to_mount, + ) + + +async def get_poetry_package_dir(airbyte_repo_dir: dagger.Directory, poetry_package_path: Path) -> dagger.Directory: + """Get the poetry package directory from the airbyte repo directory. + + Args: + airbyte_repo_dir (dagger.Directory): The airbyte repo directory. + poetry_package_path (Path): Path to the poetry package in the airbyte repo. + + Raises: + FileNotFoundError: If the pyproject.toml file is not found in the poetry package directory. + FileNotFoundError: If the poetry package directory is not found in the airbyte repo directory. + + Returns: + dagger.Directory: The poetry package directory. + """ + try: + package_directory = await airbyte_repo_dir.directory(str(poetry_package_path)) + if PYPROJECT_TOML_FILE_PATH not in await package_directory.entries(): + raise FileNotFoundError(f"Could not find pyproject.toml in {poetry_package_path}, are you sure this is a poetry package?") + except dagger.DaggerError: + raise FileNotFoundError(f"Could not find {poetry_package_path} in the repository, are you sure this path is correct?") + return package_directory + + +async def get_airbyte_ci_package_config(poetry_package_dir: dagger.Directory) -> AirbyteCiPackageConfiguration: + """Get the airbyte ci package configuration from the pyproject.toml file in the poetry package directory. + + Args: + poetry_package_dir (dagger.Directory): The poetry package directory. + + Returns: + AirbyteCiPackageConfiguration: The airbyte ci package configuration. + """ + raw_pyproject_toml = await poetry_package_dir.file(PYPROJECT_TOML_FILE_PATH).contents() + pyproject_toml = toml.loads(raw_pyproject_toml) + return deserialize_airbyte_ci_config(pyproject_toml) + + +def get_poetry_base_container(dagger_client: dagger.Client) -> dagger.Container: + """Get a base container with system dependencies to run poe tasks of poetry package: + - git: required for packages using GitPython + - poetry + - poethepoet + - docker: required for packages using docker in their tests + + Args: + dagger_client (dagger.Client): The dagger client. + + Returns: + dagger.Container: The base container. + """ + poetry_cache_volume: dagger.CacheVolume = dagger_client.cache_volume(POETRY_CACHE_VOLUME_NAME) + poetry_cache_path = "/root/.cache/poetry" + return ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_env_variable("POETRY_CACHE_DIR", poetry_cache_path) + .with_mounted_cache(poetry_cache_path, poetry_cache_volume) + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + "pipx install poethepoet", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + ) + + +def prepare_container_for_poe_tasks( + dagger_client: dagger.Client, + airbyte_repo_dir: dagger.Directory, + airbyte_ci_package_config: AirbyteCiPackageConfiguration, + poetry_package_path: Path, + is_ci: bool, +) -> dagger.Container: + """Prepare a container to run poe tasks for a poetry package. + + Args: + dagger_client (dagger.Client): The dagger client. + airbyte_repo_dir (dagger.Directory): The airbyte repo directory. + airbyte_ci_package_config (AirbyteCiPackageConfiguration): The airbyte ci package configuration. + poetry_package_path (Path): The path to the poetry package in the airbyte repo. + is_ci (bool): Whether the container is running in a CI environment. + + Returns: + dagger.Container: The container to run poe tasks for the poetry package. + """ + + # BE CAREFUL ABOUT THE ORDER OF THESE INSTRUCTIONS + # PLEASE REMIND THAT DAGGER OPERATION ARE CACHED LIKE IN DOCKERFILE: + # ANY CHANGE IN THE INPUTS OF AN OPERATION WILL INVALIDATE THE DOWNSTREAM OPERATIONS CACHE + + # Start from the base container + container = get_poetry_base_container(dagger_client) + + # Set the CI environment variable + if is_ci: + container = container.with_env_variable("CI", "true") + + # Bind to dockerd service if needed + if airbyte_ci_package_config.side_car_docker_engine: + dockerd_service = docker.with_global_dockerd_service(dagger_client) + container = ( + container.with_env_variable("DOCKER_HOST", f"tcp://{DOCKER_HOST_NAME}:{DOCKER_HOST_PORT}") + .with_env_variable("DOCKER_HOST_NAME", DOCKER_HOST_NAME) + .with_service_binding(DOCKER_HOST_NAME, dockerd_service) + ) + + # Mount the docker socket if needed + if airbyte_ci_package_config.mount_docker_socket: + container = container.with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + + # Set the required environment variables according to the package configuration + for required_env_var in airbyte_ci_package_config.required_environment_variables: + # We consider any environment variable as a secret for safety reasons + secret_env_var = dagger_client.set_secret(required_env_var, os.environ[required_env_var]) + container = container.with_secret_variable(required_env_var, secret_env_var) + + # Mount the airbyte repo directory + container = container.with_mounted_directory("/airbyte", airbyte_repo_dir) + + # Set working directory to the poetry package directory + container = container.with_workdir(f"/airbyte/{poetry_package_path}") + + # Install the poetry package + container = container.with_exec(["poetry", "install"] + [f"--with={group}" for group in airbyte_ci_package_config.extra_poetry_groups]) + return container + + +async def run_poe_task(container: dagger.Container, poe_task: str) -> PoeTaskResult: + """Run the poe task in the container and return a PoeTaskResult. + + Args: + container (dagger.Container): The container to run the poe task in. + poe_task (str): The poe task to run. + + Returns: + PoeTaskResult: The result of the command execution. + """ + try: + executed_container = await container.pipeline(f"Run poe {poe_task}").with_exec(["poe", poe_task]) + return PoeTaskResult( + task_name=poe_task, + status=StepStatus.SUCCESS, + stdout=await executed_container.stdout(), + stderr=await executed_container.stderr(), + ) + except dagger.ExecError as e: + return PoeTaskResult(task_name=poe_task, status=StepStatus.FAILURE, exc_info=e) + + +async def run_and_log_poe_task_results( + pipeline_context_params: Dict, package_name: str, container: dagger.Container, poe_task: str, logger: Logger +) -> PoeTaskResult: + """Run the poe task in the container and log the result. + + Args: + pipeline_context_params (Dict): The pipeline context parameters. + package_name (str): The name of the package to run the poe task for. + container (dagger.Container): The container to run the poe task in. + poe_task (str): The poe task to run. + logger (Logger): The logger to log the result. + + Returns: + PoeTaskResult: The result of the command execution. + """ + + commit_status_check_params = { + "sha": pipeline_context_params["git_revision"], + "description": f"{poe_task} execution for {package_name}", + "context": f"{package_name} - {poe_task}", + "target_url": f"{pipeline_context_params['gha_workflow_run_url']}", + "should_send": pipeline_context_params["is_ci"], + "logger": logger, + } + + logger.info(f"Running poe task: {poe_task}") + # Send pending status check + update_commit_status_check(**{**commit_status_check_params, "state": "pending"}) + result = await run_poe_task(container, poe_task) + result.log(logger) + # Send the final status check + update_commit_status_check(**{**commit_status_check_params, "state": result.status.get_github_state()}) + + return result + + +async def run_poe_tasks_for_package( + dagger_client: dagger.Client, poetry_package_path: Path, pipeline_context_params: Dict +) -> List[PoeTaskResult]: + """Concurrently Run the poe tasks declared in pyproject.toml for a poetry package. + + Args: + dagger_client (dagger.Client): The dagger client. + poetry_package_path (Path): The path to the poetry package in the airbyte repo. + pipeline_context_params (Dict): The pipeline context parameters. + Returns: + List[PoeTaskResult]: The results of the poe tasks. + """ + dagger_client = dagger_client.pipeline(f"Run poe tasks for {poetry_package_path}") + airbyte_repo_dir = await get_filtered_airbyte_repo_dir(dagger_client, poetry_package_path) + package_dir = await get_poetry_package_dir(airbyte_repo_dir, poetry_package_path) + package_config = await get_airbyte_ci_package_config(package_dir) + container = prepare_container_for_poe_tasks( + dagger_client, airbyte_repo_dir, package_config, poetry_package_path, pipeline_context_params["is_ci"] + ) + logger = logging.getLogger(str(poetry_package_path)) + + if not package_config.poe_tasks: + logger.warning("No poe tasks to run.") + return [] + + poe_task_results = [] + async with asyncer.create_task_group() as poe_tasks_task_group: + for task in package_config.poe_tasks: + poe_task_results.append( + poe_tasks_task_group.soonify(run_and_log_poe_task_results)( + pipeline_context_params, str(poetry_package_path), container, task, logger + ) + ) + return [result.value for result in poe_task_results] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py index 78c5e22e7db4..c633f59db1d2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py @@ -12,7 +12,7 @@ @click.command() @click.option("--version", default="latest", type=str, help="The version to update to.") -async def update(version: str): +async def update(version: str) -> None: """Updates airbyte-ci to the latest version.""" is_dev = is_dev_command() if is_dev: diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 3ce62bca74d6..0cc95dcb056c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -4,79 +4,46 @@ """This module is the CLI entrypoint to the airbyte-ci commands.""" +from __future__ import annotations + +# HACK! IMPORTANT! This import and function call must be the first import in this file +# This is needed to ensure that the working directory is the root of the airbyte repo +# ruff: noqa: E402 +from pipelines.cli.ensure_repo_root import set_working_directory_to_root + +set_working_directory_to_root() + import logging import multiprocessing import os import sys -from pathlib import Path from typing import Optional import asyncclick as click -import docker -import git +import docker # type: ignore from github import PullRequest from pipelines import main_logger from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag -from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import ( + CI_REQUIREMENTS_OPTION_NAME, + click_append_to_context_object, + click_ci_requirements_option, + click_ignore_unused_kwargs, + click_merge_args_into_context_obj, +) from pipelines.cli.confirm_prompt import pre_confirm_all_flag from pipelines.cli.lazy_group import LazyGroup from pipelines.cli.telemetry import click_track_command -from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, CIContext +from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, LOCAL_BUILD_PLATFORM, CIContext +from pipelines.dagger.actions.connector.hooks import get_dagger_sdk_version from pipelines.helpers import github from pipelines.helpers.git import get_current_git_branch, get_current_git_revision from pipelines.helpers.utils import get_current_epoch_time -def _validate_airbyte_repo(repo: git.Repo) -> bool: - """Check if any of the remotes are the airbyte repo.""" - expected_repo_name = "airbytehq/airbyte" - for remote in repo.remotes: - if expected_repo_name in remote.url: - return True - - warning_message = f""" - ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ - - It looks like you are not running this command from the airbyte repo ({expected_repo_name}). - - If this command is run from outside the airbyte repo, it will not work properly. - - Please run this command your local airbyte project. - - ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ - """ - - logging.warning(warning_message) - - return False - - -def get_airbyte_repo() -> git.Repo: - """Get the airbyte repo.""" - repo = git.Repo(search_parent_directories=True) - _validate_airbyte_repo(repo) - return repo - - -def get_airbyte_repo_path_with_fallback() -> Path: - """Get the path to the airbyte repo.""" - try: - return get_airbyte_repo().working_tree_dir - except git.exc.InvalidGitRepositoryError: - logging.warning("Could not find the airbyte repo, falling back to the current working directory.") - path = Path.cwd() - logging.warning(f"Using {path} as the airbyte repo path.") - return path - - -def set_working_directory_to_root() -> None: - """Set the working directory to the root of the airbyte repo.""" - working_dir = get_airbyte_repo_path_with_fallback() - logging.info(f"Setting working directory to {working_dir}") - os.chdir(working_dir) - - -def log_git_info(ctx: click.Context): +def log_context_info(ctx: click.Context) -> None: + main_logger.info(f"Running airbyte-ci version {__installed_version__}") + main_logger.info(f"Running dagger version {get_dagger_sdk_version()}") main_logger.info("Running airbyte-ci in CI mode.") main_logger.info(f"CI Context: {ctx.obj['ci_context']}") main_logger.info(f"CI Report Bucket Name: {ctx.obj['ci_report_bucket_name']}") @@ -86,6 +53,7 @@ def log_git_info(ctx: click.Context): main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") main_logger.info(f"Pull Request Number: {ctx.obj['pull_request_number']}") main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") + main_logger.info(f"Local build platform: {LOCAL_BUILD_PLATFORM}") def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: @@ -96,7 +64,7 @@ def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" -def _get_pull_request(ctx: click.Context) -> PullRequest or None: +def _get_pull_request(ctx: click.Context) -> Optional[PullRequest.PullRequest]: pull_request_number = ctx.obj["pull_request_number"] ci_github_access_token = ctx.obj["ci_github_access_token"] @@ -107,7 +75,7 @@ def _get_pull_request(ctx: click.Context) -> PullRequest or None: return github.get_pull_request(pull_request_number, ci_github_access_token) -def check_local_docker_configuration(): +def check_local_docker_configuration() -> None: try: docker_client = docker.from_env() except Exception as e: @@ -122,6 +90,9 @@ def check_local_docker_configuration(): def is_dagger_run_enabled_by_default() -> bool: + if CI_REQUIREMENTS_OPTION_NAME in sys.argv: + return False + dagger_run_by_default = [ ["connectors", "test"], ["connectors", "build"], @@ -136,7 +107,7 @@ def is_dagger_run_enabled_by_default() -> bool: return False -def check_dagger_wrap(): +def check_dagger_wrap() -> bool: """ Check if the command is already wrapped by dagger run. This is useful to avoid infinite recursion when calling dagger run from dagger run. @@ -161,6 +132,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "poetry": "pipelines.airbyte_ci.poetry.commands.poetry", "format": "pipelines.airbyte_ci.format.commands.format_code", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", @@ -183,7 +155,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: type=str, ) @click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str) -@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice(CIContext)) +@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice([c for c in CIContext])) @click.option("--pipeline-start-timestamp", default=get_current_epoch_time, envvar="CI_PIPELINE_START_TIMESTAMP", type=int) @click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int) @click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str) @@ -201,6 +173,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: @click.option("--s3-build-cache-access-key-id", envvar="S3_BUILD_CACHE_ACCESS_KEY_ID", type=str) @click.option("--s3-build-cache-secret-key", envvar="S3_BUILD_CACHE_SECRET_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) +@click_ci_requirements_option() @click_track_command @click_merge_args_into_context_obj @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @@ -208,7 +181,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: @click_append_to_context_object("pull_request", _get_pull_request) @click.pass_context @click_ignore_unused_kwargs -async def airbyte_ci(ctx: click.Context): # noqa D103 +async def airbyte_ci(ctx: click.Context) -> None: # noqa D103 # Check that the command being run is not upgrade is_update_command = ctx.invoked_subcommand == "update" if ctx.obj["enable_update_check"] and ctx.obj["is_local"] and not is_update_command: @@ -230,10 +203,8 @@ async def airbyte_ci(ctx: click.Context): # noqa D103 check_local_docker_configuration() if not ctx.obj["is_local"]: - log_git_info(ctx) - + log_context_info(ctx) -set_working_directory_to_root() if __name__ == "__main__": airbyte_ci() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py index eb455e74133a..e1ac37ee68d9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py @@ -3,19 +3,24 @@ # # HELPERS +from __future__ import annotations import importlib import logging import os import sys +from typing import TYPE_CHECKING import asyncclick as click -import requests +import requests # type: ignore from pipelines import main_logger from pipelines.cli.confirm_prompt import confirm from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH from pipelines.external_scripts.airbyte_ci_install import RELEASE_URL, get_airbyte_os_name +if TYPE_CHECKING: + from typing import Callable + __installed_version__ = importlib.metadata.version("pipelines") PROD_COMMAND = "airbyte-ci" @@ -23,7 +28,7 @@ AUTO_UPDATE_AGREE_KEY = "yes_auto_update" -def pre_confirm_auto_update_flag(f): +def pre_confirm_auto_update_flag(f: Callable) -> Callable: """Decorator to add a --yes-auto-update flag to a command.""" return click.option( "--yes-auto-update", AUTO_UPDATE_AGREE_KEY, is_flag=True, default=False, help="Skip prompts and automatically upgrade pipelines" @@ -70,7 +75,7 @@ def is_dev_command() -> bool: def check_for_upgrade( require_update: bool = True, enable_auto_update: bool = True, -): +) -> None: """Check if the installed version of pipelines is up to date.""" current_command = " ".join(sys.argv) latest_version = _get_latest_version() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index c7740a0f0f65..b88f582c6e37 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -5,9 +5,14 @@ import functools import inspect from functools import wraps -from typing import Any, Callable, Type +from typing import Any, Callable, Type, TypeVar import asyncclick as click +from pipelines.models.ci_requirements import CIRequirements + +_AnyCallable = Callable[..., Any] +FC = TypeVar("FC", bound="_AnyCallable | click.core.Command") +CI_REQUIREMENTS_OPTION_NAME = "--ci-requirements" def _contains_var_kwarg(f: Callable) -> bool: @@ -15,8 +20,11 @@ def _contains_var_kwarg(f: Callable) -> bool: def _is_kwarg_of(key: str, f: Callable) -> bool: - param = inspect.signature(f).parameters.get(key, False) - return param and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) + param = inspect.signature(f).parameters.get(key) + if not param: + return False + + return bool(param) and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) def click_ignore_unused_kwargs(f: Callable) -> Callable: @@ -31,7 +39,7 @@ def click_ignore_unused_kwargs(f: Callable) -> Callable: return f @functools.wraps(f) - def inner(*args, **kwargs): + def inner(*args: Any, **kwargs: Any) -> Callable: filtered_kwargs = {key: value for key, value in kwargs.items() if _is_kwarg_of(key, f)} return f(*args, **filtered_kwargs) @@ -43,7 +51,7 @@ def click_merge_args_into_context_obj(f: Callable) -> Callable: Decorator to pass click context and args to children commands. """ - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Callable: ctx = click.get_current_context() ctx.ensure_object(dict) click_obj = ctx.obj @@ -61,13 +69,13 @@ def wrapper(*args, **kwargs): return wrapper -def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: +def click_append_to_context_object(key: str, value: Callable) -> Callable: """ Decorator to append a value to the click context object. """ - def decorator(f): - async def wrapper(*args, **kwargs): + def decorator(f: Callable) -> Callable: + async def wrapper(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 ctx = click.get_current_context() ctx.ensure_object(dict) @@ -104,7 +112,7 @@ def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: """ @wraps(f) - def decorated_function(*args: Any, **kwargs: Any) -> Any: + def decorated_function(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 # Check if the kwargs already contain the arguments being passed by the decorator decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} # Create an instance of the class @@ -118,3 +126,27 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any: return f(*args, **kwargs) return decorated_function + + +def click_ci_requirements_option() -> Callable[[FC], FC]: + """Add a --ci-requirements option to the command. + + Returns: + Callable[[FC], FC]: The decorated command. + """ + + def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None: + if value: + ci_requirements = CIRequirements() + click.echo(ci_requirements.to_json()) + ctx.exit() + + return click.decorators.option( + CI_REQUIREMENTS_OPTION_NAME, + is_flag=True, + expose_value=False, + is_eager=True, + flag_value=True, + help="Show the CI requirements and exit. It used to make airbyte-ci client define the CI runners it will run on.", + callback=callback, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py b/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py index 79f7785ef48f..b7504cb452f1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py @@ -2,19 +2,24 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Optional +from __future__ import annotations + +from typing import TYPE_CHECKING import asyncclick as click +if TYPE_CHECKING: + from typing import Any, Callable + PRE_CONFIRM_ALL_KEY = "yes" -def pre_confirm_all_flag(f): +def pre_confirm_all_flag(f: Callable) -> Callable: """Decorator to add a --yes flag to a command.""" return click.option("-y", "--yes", PRE_CONFIRM_ALL_KEY, is_flag=True, default=False, help="Skip prompts and use default values")(f) -def confirm(*args, **kwargs) -> bool: +def confirm(*args: Any, **kwargs: Any) -> bool: """Confirm a prompt with the user, with support for a --yes flag.""" additional_pre_confirm_key = kwargs.pop("additional_pre_confirm_key", None) ctx = click.get_current_context() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py index 2fed8450ebd6..96c8dbb89419 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py @@ -6,9 +6,7 @@ from __future__ import annotations import sys -from glob import glob from pathlib import Path -from typing import Any import asyncclick as click from dagger import DaggerError @@ -21,14 +19,12 @@ class DaggerPipelineCommand(click.Command): @sentry_utils.with_command_context - async def invoke(self, ctx: click.Context) -> Any: + async def invoke(self, ctx: click.Context) -> None: """Wrap parent invoke in a try catch suited to handle pipeline failures. Args: ctx (click.Context): The invocation context. Raises: e: Raise whatever exception that was caught. - Returns: - Any: The invocation return value. """ command_name = self.name main_logger.info(f"Running Dagger Command {command_name}...") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py index 155332f8d0ab..e290d41c4766 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -12,14 +12,16 @@ from pathlib import Path from typing import Optional -import pkg_resources -import requests +import pkg_resources # type: ignore +import requests # type: ignore from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME LOGGER = logging.getLogger(__name__) BIN_DIR = Path.home() / "bin" BIN_DIR.mkdir(exist_ok=True) -DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE = ( +DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE = ( + # The _EXPERIMENTAL_DAGGER_CLOUD_TOKEN is used for telemetry only at the moment. + # It will eventually be renamed to a more specific name in future Dagger versions. "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN", "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k", ) @@ -35,6 +37,7 @@ def get_dagger_path() -> Optional[str]: except subprocess.CalledProcessError: if Path(BIN_DIR / "dagger").exists(): return str(Path(BIN_DIR / "dagger")) + return None def get_current_dagger_sdk_version() -> str: @@ -82,6 +85,7 @@ def check_dagger_cli_install() -> str: LOGGER.info(f"The Dagger CLI is not installed. Installing {expected_dagger_cli_version}...") install_dagger_cli(expected_dagger_cli_version) dagger_path = get_dagger_path() + assert dagger_path is not None, "Dagger CLI installation failed, dagger not found in path" cli_version = get_dagger_cli_version(dagger_path) if cli_version != expected_dagger_cli_version: @@ -93,17 +97,19 @@ def check_dagger_cli_install() -> str: return dagger_path -def mark_dagger_wrap(): +def mark_dagger_wrap() -> None: """ Mark that the dagger wrap has been applied. """ os.environ[DAGGER_WRAP_ENV_VAR_NAME] = "true" -def call_current_command_with_dagger_run(): +def call_current_command_with_dagger_run() -> None: mark_dagger_wrap() - if (os.environ.get("AIRBYTE_ROLE") == "airbyter") or (os.environ.get("CI") == "True"): - os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] + # We're enabling telemetry only for local runs. + # CI runs already have telemetry as DAGGER_CLOUD_TOKEN env var is set on the CI. + if (os.environ.get("AIRBYTE_ROLE") == "airbyter") and not os.environ.get("CI"): + os.environ[DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE[1] exit_code = 0 dagger_path = check_dagger_cli_install() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py new file mode 100644 index 000000000000..5970979d9d71 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import os +from pathlib import Path + +import git + + +def _validate_airbyte_repo(repo: git.Repo) -> bool: + """Check if any of the remotes are the airbyte repo.""" + expected_repo_name = "airbytehq/airbyte" + for remote in repo.remotes: + if expected_repo_name in remote.url: + return True + + warning_message = f""" + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + + It looks like you are not running this command from the airbyte repo ({expected_repo_name}). + + If this command is run from outside the airbyte repo, it will not work properly. + + Please run this command your local airbyte project. + + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + """ + + logging.warning(warning_message) + + return False + + +def get_airbyte_repo() -> git.Repo: + """Get the airbyte repo.""" + repo = git.Repo(search_parent_directories=True) + _validate_airbyte_repo(repo) + return repo + + +def get_airbyte_repo_path_with_fallback() -> Path: + """Get the path to the airbyte repo.""" + try: + repo_path = get_airbyte_repo().working_tree_dir + if repo_path is not None: + return Path(str(get_airbyte_repo().working_tree_dir)) + except git.exc.InvalidGitRepositoryError: + pass + logging.warning("Could not find the airbyte repo, falling back to the current working directory.") + path = Path.cwd() + logging.warning(f"Using {path} as the airbyte repo path.") + return path + + +def set_working_directory_to_root() -> None: + """Set the working directory to the root of the airbyte repo.""" + working_dir = get_airbyte_repo_path_with_fallback() + logging.info(f"Setting working directory to {working_dir}") + os.chdir(working_dir) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py index b7214a26f7f9..def24edb1b6d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py @@ -3,7 +3,7 @@ # Source: https://click.palletsprojects.com/en/8.1.x/complex/ import importlib -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import asyncclick as click @@ -13,7 +13,7 @@ class LazyGroup(click.Group): A click Group that can lazily load subcommands. """ - def __init__(self, *args, lazy_subcommands: Optional[Dict[str, str]] = None, **kwargs): + def __init__(self, *args: Any, lazy_subcommands: Optional[Dict[str, str]] = None, **kwargs: Any) -> None: super().__init__(*args, **kwargs) # lazy_subcommands is a map of the form: # @@ -31,7 +31,7 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma return self._lazy_load(cmd_name) return super().get_command(ctx, cmd_name) - def _lazy_load(self, cmd_name: str) -> click.BaseCommand: + def _lazy_load(self, cmd_name: str) -> click.Command: # lazily loading a command, first get the module name and attribute name import_path = self.lazy_subcommands[cmd_name] modname, cmd_object_name = import_path.rsplit(".", 1) @@ -40,7 +40,7 @@ def _lazy_load(self, cmd_name: str) -> click.BaseCommand: # get the Command object from that module cmd_object = getattr(mod, cmd_object_name) # check the result to make debugging easier - if not isinstance(cmd_object, click.BaseCommand): + if not isinstance(cmd_object, click.Command): print(f"{cmd_object} is of instance {type(cmd_object)}") raise ValueError(f"Lazy loading of {import_path} failed by returning " "a non-command object") return cmd_object diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 7ad4bea8daaa..8ecb606973a1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -1,27 +1,36 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import getpass import hashlib import os import platform import sys +from typing import TYPE_CHECKING -import segment.analytics as analytics +import segment.analytics as analytics # type: ignore from asyncclick import get_current_context +DISABLE_TELEMETRY = os.environ.get("AIRBYTE_CI_DISABLE_TELEMETRY", "false").lower() == "true" + +if TYPE_CHECKING: + from typing import Any, Callable, Dict, Tuple + + from asyncclick import Command + analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" -analytics.send = True +analytics.send = not DISABLE_TELEMETRY analytics.debug = False -def _is_airbyte_user(): +def _is_airbyte_user() -> bool: """Returns True if the user is airbyter, False otherwise.""" return os.getenv("AIRBYTE_ROLE") == "airbyter" -def _get_anonymous_system_id(): +def _get_anonymous_system_id() -> str: """Returns a unique anonymous hashid of the current system info.""" # Collect machine-specific information machine_info = platform.node() @@ -35,20 +44,20 @@ def _get_anonymous_system_id(): return unique_id -def click_track_command(f): +def click_track_command(f: Callable) -> Callable: """ Decorator to track CLI commands with segment.io """ - def wrapper(*args, **kwargs): + def wrapper(*args: Tuple, **kwargs: Dict[str, Any]) -> Command: ctx = get_current_context() + top_level_command = ctx.command_path full_cmd = " ".join(sys.argv) # remove anything prior to the command name f.__name__ # to avoid logging inline secrets - santized_cmd = full_cmd[full_cmd.find(top_level_command) :] - + sanitized_cmd = full_cmd[full_cmd.find(top_level_command) :] sys_id = _get_anonymous_system_id() sys_user_name = f"anonymous:{sys_id}" airbyter = _is_airbyte_user() @@ -58,7 +67,7 @@ def wrapper(*args, **kwargs): event = f"airbyte-ci:{f.__name__}" # IMPORTANT! do not log kwargs as they may contain secrets - analytics.track(user_id, event, {"username": sys_user_name, "command": santized_cmd, "airbyter": airbyter}) + analytics.track(user_id, event, {"username": sys_user_name, "command": sanitized_cmd, "airbyter": airbyter}) return f(*args, **kwargs) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 353b7ba657ff..06bec50a19bb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -30,17 +30,18 @@ } LOCAL_MACHINE_TYPE = platform.machine() LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[LOCAL_MACHINE_TYPE] -AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" +AMAZONCORRETTO_IMAGE = "amazoncorretto:21-al2023" NODE_IMAGE = "node:18.18.0-slim" GO_IMAGE = "golang:1.17" PYTHON_3_10_IMAGE = "python:3.10.13-slim" -MAVEN_IMAGE = "maven:3.9.5-amazoncorretto-17-al2023" -DOCKER_VERSION = "24.0.2" +MAVEN_IMAGE = "maven:3.9.6-amazoncorretto-21-al2023" +DOCKER_VERSION = "24" DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli" DOCKER_REGISTRY_MIRROR_URL = os.getenv("DOCKER_REGISTRY_MIRROR_URL") DOCKER_REGISTRY_ADDRESS = "docker.io" DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache" +GIT_IMAGE = "alpine/git:latest" GRADLE_CACHE_PATH = "/root/.gradle/caches" GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1" GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache" @@ -58,7 +59,9 @@ POETRY_CACHE_VOLUME_NAME = "poetry_cache" POETRY_CACHE_PATH = "/root/.cache/pypoetry" STORAGE_DRIVER = "fuse-overlayfs" -TAILSCALE_AUTH_KEY = os.getenv("TAILSCALE_AUTH_KEY") +SETUP_PY_FILE_PATH = "setup.py" +DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://upload.pypi.org/legacy/" +DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL = "https://pypi.org/pypi" class CIContext(str, Enum): @@ -85,6 +88,7 @@ class ContextState(Enum): class INTERNAL_TOOL_PATHS(str, Enum): CI_CREDENTIALS = "airbyte-ci/connectors/ci_credentials" CONNECTOR_OPS = "airbyte-ci/connectors/connector_ops" + CONNECTORS_QA = "airbyte-ci/connectors/connectors_qa" METADATA_SERVICE = "airbyte-ci/connectors/metadata_service/lib" diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py index 54fb8f674eab..cf4abfaec5e7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py @@ -3,15 +3,23 @@ # import importlib.util +from importlib import metadata +from importlib.abc import Loader from dagger import Container -from dagger.engine._version import CLI_VERSION as dagger_engine_version from pipelines.airbyte_ci.connectors.context import ConnectorContext +def get_dagger_sdk_version() -> str: + try: + return metadata.version("dagger-io") + except metadata.PackageNotFoundError: + return "n/a" + + async def finalize_build(context: ConnectorContext, connector_container: Container) -> Container: """Finalize build by adding dagger engine version label and running finalize_build.sh or finalize_build.py if present in the connector directory.""" - connector_container = connector_container.with_label("io.dagger.engine_version", dagger_engine_version) + connector_container = connector_container.with_label("io.dagger.engine_version", get_dagger_sdk_version()) connector_dir_with_finalize_script = await context.get_connector_dir(include=["finalize_build.sh", "finalize_build.py"]) finalize_scripts = await connector_dir_with_finalize_script.entries() if not finalize_scripts: @@ -19,6 +27,8 @@ async def finalize_build(context: ConnectorContext, connector_container: Contain # We don't want finalize scripts to override the entrypoint so we keep it in memory to reset it after finalization original_entrypoint = await connector_container.entrypoint() + if not original_entrypoint: + original_entrypoint = [] has_finalize_bash_script = "finalize_build.sh" in finalize_scripts has_finalize_python_script = "finalize_build.py" in finalize_scripts @@ -31,7 +41,11 @@ async def finalize_build(context: ConnectorContext, connector_container: Contain connector_finalize_module_spec = importlib.util.spec_from_file_location( f"{context.connector.code_directory.name}_finalize", module_path ) + if connector_finalize_module_spec is None: + raise Exception("Connector has a finalize_build.py script but it can't be loaded.") connector_finalize_module = importlib.util.module_from_spec(connector_finalize_module_spec) + if not isinstance(connector_finalize_module_spec.loader, Loader): + raise Exception("Connector has a finalize_build.py script but it can't be loaded.") connector_finalize_module_spec.loader.exec_module(connector_finalize_module) try: connector_container = await connector_finalize_module.finalize_build(context, connector_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py index 04d60ded7437..9fe2806b7e27 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any, Dict + from dagger import Container, Platform from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -43,7 +45,7 @@ "dbt_adapter": "dbt-postgres==1.0.0", "integration_name": "postgres", "normalization_image": "airbyte/normalization:0.4.3", - "supports_in_connector_normalization": False, + "supports_in_connector_normalization": True, "yum_packages": [], }, "destination-redshift": { @@ -63,13 +65,13 @@ "yum_packages": [], }, } -DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { +DESTINATION_NORMALIZATION_BUILD_CONFIGURATION: Dict[str, Dict[str, Any]] = { **BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, **{f"{k}-strict-encrypt": v for k, v in BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION.items()}, } def with_normalization(context: ConnectorContext, build_platform: Platform) -> Container: - return context.dagger_client.container(platform=build_platform).from_( - DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["normalization_image"] - ) + normalization_image_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["normalization_image"] + assert isinstance(normalization_image_name, str) + return context.dagger_client.container(platform=build_platform).from_(normalization_image_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 76717b663127..fff7611c5ec5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -4,7 +4,7 @@ import re from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Sequence from dagger import Container, Directory from pipelines import hacks @@ -89,9 +89,7 @@ async def find_local_dependencies_in_requirements_txt(python_package: Container, # Some package declare themselves as a requirement in requirements.txt, # #Without line != "-e ." the package will be considered a dependency of itself which can cause an infinite loop if line.startswith("-e .") and line != "-e .": - local_dependency_path = Path(line[3:]) - package_source_code_path = Path(package_source_code_path) - local_dependency_path = str((package_source_code_path / local_dependency_path).resolve().relative_to(Path.cwd())) + local_dependency_path = str((Path(package_source_code_path) / Path(line[3:])).resolve().relative_to(Path.cwd())) local_requirements_dependency_paths.append(local_dependency_path) return local_requirements_dependency_paths @@ -135,7 +133,7 @@ async def find_local_python_dependencies( def _install_python_dependencies_from_setup_py( container: Container, - additional_dependency_groups: Optional[List] = None, + additional_dependency_groups: Optional[Sequence[str]] = None, ) -> Container: install_connector_package_cmd = ["pip", "install", "."] container = container.with_exec(install_connector_package_cmd) @@ -157,25 +155,36 @@ def _install_python_dependencies_from_requirements_txt(container: Container) -> def _install_python_dependencies_from_poetry( container: Container, - additional_dependency_groups: Optional[List] = None, + additional_dependency_groups: Optional[Sequence[str]] = None, + install_root_package: bool = True, ) -> Container: pip_install_poetry_cmd = ["pip", "install", "poetry"] poetry_disable_virtual_env_cmd = ["poetry", "config", "virtualenvs.create", "false"] - poetry_install_no_venv_cmd = ["poetry", "install"] + poetry_install_cmd = ["poetry", "install"] + poetry_check_cmd = ["poetry", "check"] + if not install_root_package: + poetry_install_cmd += ["--no-root"] if additional_dependency_groups: for group in additional_dependency_groups: - poetry_install_no_venv_cmd += ["--with", group] - - return container.with_exec(pip_install_poetry_cmd).with_exec(poetry_disable_virtual_env_cmd).with_exec(poetry_install_no_venv_cmd) + poetry_install_cmd += ["--with", group] + else: + poetry_install_cmd += ["--only", "main"] + return ( + container.with_exec(pip_install_poetry_cmd) + .with_exec(poetry_disable_virtual_env_cmd) + .with_exec(poetry_check_cmd) + .with_exec(poetry_install_cmd) + ) async def with_installed_python_package( context: PipelineContext, python_environment: Container, package_source_code_path: str, - additional_dependency_groups: Optional[List] = None, + additional_dependency_groups: Optional[Sequence[str]] = None, exclude: Optional[List] = None, include: Optional[List] = None, + install_root_package: bool = True, ) -> Container: """Install a python package in a python environment container. @@ -183,8 +192,10 @@ async def with_installed_python_package( context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. python_environment (Container): An existing python environment in which the package will be installed. package_source_code_path (str): The local path to the package source code. - additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. + additional_dependency_groups (Optional[Sequence[str]]): extra_requires dependency of setup.py to install. Defaults to None. exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + include (Optional[List]): A list of file or directory to include from the python package source code. + install_root_package (bool): Whether to install the root package. Defaults to True. Returns: Container: A python environment container with the python package installed. @@ -201,7 +212,7 @@ async def with_installed_python_package( if has_pyproject_toml: container = with_poetry_cache(container, context.dagger_client) - container = _install_python_dependencies_from_poetry(container, additional_dependency_groups) + container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package) elif has_setup_py: container = with_pip_cache(container, context.dagger_client) container = _install_python_dependencies_from_setup_py(container, additional_dependency_groups) @@ -249,9 +260,10 @@ async def with_python_connector_installed( context: ConnectorContext, python_container: Container, connector_source_path: str, - additional_dependency_groups: Optional[List] = None, - exclude: Optional[List] = None, - include: Optional[List] = None, + additional_dependency_groups: Optional[Sequence[str]] = None, + exclude: Optional[List[str]] = None, + include: Optional[List[str]] = None, + install_root_package: bool = True, ) -> Container: """Install an airbyte python connectors dependencies.""" @@ -265,6 +277,7 @@ async def with_python_connector_installed( additional_dependency_groups=additional_dependency_groups, exclude=exclude, include=include, + install_root_package=install_root_package, ) container = await apply_python_development_overrides(context, container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py index df555f18b750..c9399a11c699 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import List, Optional -import toml +import toml # type: ignore from dagger import Container, Directory from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package @@ -37,11 +37,9 @@ async def find_local_dependencies_in_pyproject_toml( pyproject_content = toml.loads(pyproject_content_raw) local_dependency_paths = [] - for dep, value in pyproject_content["tool"]["poetry"]["dependencies"].items(): + for value in pyproject_content["tool"]["poetry"]["dependencies"].values(): if isinstance(value, dict) and "path" in value: - local_dependency_path = Path(value["path"]) - pyproject_file_path = Path(pyproject_file_path) - local_dependency_path = str((pyproject_file_path / local_dependency_path).resolve().relative_to(Path.cwd())) + local_dependency_path = str((Path(pyproject_file_path) / Path(value["path"])).resolve().relative_to(Path.cwd())) local_dependency_paths.append(local_dependency_path) # Ensure we parse the child dependencies diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index 9160aadf5a57..2a943ddd3dd0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -6,18 +6,31 @@ from __future__ import annotations import datetime -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING from anyio import Path from dagger import Secret from pipelines.helpers.utils import get_file_contents, get_secret_host_variable if TYPE_CHECKING: + from typing import Callable, Dict + from dagger import Container - from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext + from pipelines.airbyte_ci.connectors.context import ConnectorContext + + +# List of overrides for the secrets masking logic. +# These keywords may have been marked as secrets, perhaps somewhat aggressively. +# Masking them, however, is annoying and pointless. +# This list should be extended (carefully) as needed. +NOT_REALLY_SECRETS = { + "admin", + "airbyte", + "host", +} -async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: +async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container, connector_technical_name: str) -> list[str]: """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. @@ -25,6 +38,9 @@ async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) secrets_to_mask = [] if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): for secret_to_mask in secrets_to_mask_file.splitlines(): + if secret_to_mask in NOT_REALLY_SECRETS or secret_to_mask in connector_technical_name: + # Don't mask secrets which are also common words or connector name. + continue # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked print(f"::add-mask::{secret_to_mask}") @@ -32,7 +48,7 @@ async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) return secrets_to_mask -async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> dict[str, Secret]: +async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Dict[str, Secret]: """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory. Args: @@ -57,7 +73,7 @@ async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = " ) # We don't want to print secrets in the logs when running locally. if context.is_ci: - context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) + context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets, context.connector.technical_name) connector_secrets = {} for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() @@ -68,7 +84,7 @@ async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = " return connector_secrets -async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS"): +async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Container: """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir. Args: @@ -81,6 +97,7 @@ async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GC Raises: ExecError: If the command returns a non-zero exit code. """ + assert context.updated_secrets_dir is not None, "The context's updated_secrets_dir must be set to upload secrets." # temp - fix circular import from pipelines.dagger.containers.internal_tools import with_ci_credentials @@ -94,7 +111,7 @@ async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GC ) -async def load_from_local(context: ConnectorContext) -> dict[str, Secret]: +async def load_from_local(context: ConnectorContext) -> Dict[str, Secret]: """Load the secrets from the local secrets directory for a connector. Args: @@ -103,7 +120,7 @@ async def load_from_local(context: ConnectorContext) -> dict[str, Secret]: Returns: dict[str, Secret]: A dict mapping the secret file name to the dagger Secret object. """ - connector_secrets = {} + connector_secrets: Dict[str, Secret] = {} local_secrets_path = Path(context.connector.code_directory / "secrets") if not await local_secrets_path.is_dir(): context.logger.warning(f"Local secrets directory {local_secrets_path} does not exist, no secrets will be loaded.") @@ -133,47 +150,28 @@ async def get_connector_secrets(context: ConnectorContext) -> dict[str, Secret]: return connector_secrets -async def mounted_connector_secrets(context: PipelineContext, secret_directory_path: str) -> Callable[[Container], Container]: - # By default, mount the secrets properly as dagger secret files. - # - # This will cause the contents of these files to be scrubbed from the logs. This scrubbing comes at the cost of - # unavoidable latency in the log output, see next paragraph for details as to why. This is fine in a CI environment - # however this becomes a nuisance locally: the developer wants the logs to be displayed to them in an as timely - # manner as possible. Since the secrets aren't really secret in that case anyway, we mount them in the container as - # regular files instead. - # - # The buffering behavior that comes into play when logs are scrubbed is both unavoidable and not configurable. - # It's fundamentally unavoidable because dagger needs to match a bunch of regexes (one per secret) and therefore - # needs to buffer at least as many bytes as the longest of all possible matches. Still, this isn't that long in - # practice in our case. The real problem is that the buffering is not configurable: dagger relies on a golang - # library called transform [1] to perform the regexp matching on a stream and this library hard-codes a buffer - # size of 4096 bytes for each regex [2]. - # - # Remove the special local case whenever dagger implements scrubbing differently [3,4]. - # - # [1] https://golang.org/x/text/transform - # [2] https://cs.opensource.google/go/x/text/+/refs/tags/v0.13.0:transform/transform.go;l=130 - # [3] https://github.com/dagger/dagger/blob/v0.6.4/cmd/shim/main.go#L294 - # [4] https://github.com/airbytehq/airbyte/issues/30394 - # - if context.is_local: - # Special case for local development. - # Query dagger for the contents of the secrets and mount these strings as files in the container. - contents = {} - for secret_file_name, secret in context.connector_secrets.items(): - contents[secret_file_name] = await secret.plaintext() - - def with_secrets_mounted_as_regular_files(container: Container) -> Container: - container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True) - for secret_file_name, secret_content_str in contents.items(): - container = container.with_new_file(f"{secret_directory_path}/{secret_file_name}", secret_content_str, permissions=0o600) - return container - - return with_secrets_mounted_as_regular_files +async def mounted_connector_secrets(context: ConnectorContext, secret_directory_path: str) -> Callable[[Container], Container]: + """Returns an argument for a dagger container's with_ method which mounts all connector secrets in it. + + Args: + context (ConnectorContext): The context providing a connector object and its secrets. + secret_directory_path (str): Container directory where the secrets will be mounted, as files. + + Returns: + fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. + """ + connector_secrets = await context.get_connector_secrets() + java_log_scrub_pattern_secret = context.java_log_scrub_pattern_secret def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container: + if java_log_scrub_pattern_secret: + # This LOG_SCRUB_PATTERN environment variable is used by our log4j test configuration + # to scrub secrets from the log messages. Although we already ensure that github scrubs them + # from its runner logs, this is required to prevent the secrets from leaking into gradle scans, + # test reports or any other build artifacts generated by a java connector test. + container = container.with_secret_variable("LOG_SCRUB_PATTERN", java_log_scrub_pattern_secret) container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True) - for secret_file_name, secret in context.connector_secrets.items(): + for secret_file_name, secret in connector_secrets.items(): container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret) return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py index 2f7fe5543b3c..a22291682e13 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py @@ -4,9 +4,9 @@ import json import uuid -from typing import Callable, Optional +from typing import Callable, Dict, List, Optional, Union -from dagger import Client, Container, File, Secret +from dagger import Client, Container, File, Secret, Service from pipelines import consts from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import ( @@ -17,10 +17,8 @@ DOCKER_TMP_VOLUME_NAME, DOCKER_VAR_LIB_VOLUME_NAME, STORAGE_DRIVER, - TAILSCALE_AUTH_KEY, ) from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.pipeline_context import PipelineContext def get_base_dockerd_container(dagger_client: Client) -> Container: @@ -74,7 +72,7 @@ def get_daemon_config_json(registry_mirror_url: Optional[str] = None) -> str: Returns: str: The json representation of the docker daemon config. """ - daemon_config = { + daemon_config: Dict[str, Union[List[str], str]] = { "storage-driver": STORAGE_DRIVER, } if registry_mirror_url: @@ -85,15 +83,15 @@ def get_daemon_config_json(registry_mirror_url: Optional[str] = None) -> str: def docker_login( dockerd_container: Container, - docker_registry_username_secret: Optional[Secret], - docker_registry_password_secret: Optional[Secret], + docker_registry_username_secret: Secret, + docker_registry_password_secret: Secret, ) -> Container: """Login to a docker registry if the username and password secrets are provided. Args: dockerd_container (Container): The dockerd_container container to login to the registry. - docker_registry_username_secret (Optional[Secret]): The docker registry username secret. - docker_registry_password_secret (Optional[Secret]): The docker registry password secret. + docker_registry_username_secret (Secret): The docker registry username secret. + docker_registry_password_secret (Secret): The docker registry password secret. docker_registry_address (Optional[str]): The docker registry address to login to. Defaults to "docker.io" (DockerHub). Returns: Container: The container with the docker login command executed if the username and password secrets are provided. Noop otherwise. @@ -118,10 +116,10 @@ def with_global_dockerd_service( dagger_client: Client, docker_hub_username_secret: Optional[Secret] = None, docker_hub_password_secret: Optional[Secret] = None, -) -> Container: +) -> Service: """Create a container with a docker daemon running. We expose its 2375 port to use it as a docker host for docker-in-docker use cases. - It is optionally bound to a tailscale VPN if the TAILSCALE_AUTH_KEY env var is set. + It is optionally connected to a DockerHub mirror if the DOCKER_REGISTRY_MIRROR_URL env var is set. Args: dagger_client (Client): The dagger client used to create the container. docker_hub_username_secret (Optional[Secret]): The DockerHub username secret. @@ -131,7 +129,7 @@ def with_global_dockerd_service( """ dockerd_container = get_base_dockerd_container(dagger_client) - if TAILSCALE_AUTH_KEY is not None: + if DOCKER_REGISTRY_MIRROR_URL is not None: # Ping the registry mirror host to make sure it's reachable through VPN # We set a cache buster here to guarantee the curl command is always executed. dockerd_container = dockerd_container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec( @@ -141,12 +139,13 @@ def with_global_dockerd_service( else: daemon_config_json = get_daemon_config_json() - dockerd_container = dockerd_container.with_new_file("/etc/docker/daemon.json", daemon_config_json) - # Docker login happens late because there's a cache buster in the docker login command. - dockerd_container = docker_login(dockerd_container, docker_hub_username_secret, docker_hub_password_secret) + dockerd_container = dockerd_container.with_new_file("/etc/docker/daemon.json", contents=daemon_config_json) + if docker_hub_username_secret and docker_hub_password_secret: + # Docker login happens late because there's a cache buster in the docker login command. + dockerd_container = docker_login(dockerd_container, docker_hub_username_secret, docker_hub_password_secret) return dockerd_container.with_exec( ["dockerd", "--log-level=error", f"--host=tcp://0.0.0.0:{DOCKER_HOST_PORT}", "--tls=false"], insecure_root_capabilities=True - ) + ).as_service() def with_bound_docker_host( @@ -161,6 +160,7 @@ def with_bound_docker_host( Returns: Container: The container bound to the docker host. """ + assert context.dockerd_service is not None return ( container.with_env_variable("DOCKER_HOST", f"tcp://{DOCKER_HOST_NAME}:{DOCKER_HOST_PORT}") .with_service_binding(DOCKER_HOST_NAME, context.dockerd_service) @@ -188,7 +188,7 @@ def with_docker_cli(context: ConnectorContext) -> Container: return with_bound_docker_host(context, docker_cli) -async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str): +async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str) -> str: """Load a docker image tar archive to the docker host. Args: @@ -210,7 +210,7 @@ async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, i def with_crane( - context: PipelineContext, + context: ConnectorContext, ) -> Container: """Crane is a tool to analyze and manipulate container images. We can use it to extract the image manifest and the list of layers or list the existing tags on an image repository. diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index d143314ce91b..6061f321969e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -68,7 +68,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ) -def with_integration_base_java_and_normalization(context: PipelineContext, build_platform: Platform) -> Container: +def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container: yum_packages_to_install = [ "python3", "python3-devel", @@ -81,7 +81,9 @@ def with_integration_base_java_and_normalization(context: PipelineContext, build yum_packages_to_install += additional_yum_packages dbt_adapter_package = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dbt_adapter"] + assert isinstance(dbt_adapter_package, str) normalization_integration_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["integration_name"] + assert isinstance(normalization_integration_name, str) pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py index e69de29bb2d1..f70ecfc3a89e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py index dbb9b31d51d7..f938896d95a4 100755 --- a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py @@ -8,7 +8,7 @@ import sys -def check_command_exists(command, not_found_message): +def check_command_exists(command: str, not_found_message: str) -> None: """ Check if a command exists in the system path. """ @@ -19,7 +19,7 @@ def check_command_exists(command, not_found_message): sys.exit(1) -def main(): +def main() -> None: # Check if Python 3.10 is on the path check_command_exists( "python3.10", diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py index 2dab769f8b33..d1267c9840e9 100755 --- a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py @@ -4,19 +4,25 @@ # Meaning, no external dependencies are allowed as we don't want users to have to run anything # other than this script to install the tool. +from __future__ import annotations + import os import shutil import ssl import sys import tempfile import urllib.request +from typing import TYPE_CHECKING # !IMPORTANT! This constant is inline here instead of being imported from pipelines/consts.py # because we don't want to introduce any dependencies on other files in the repository. RELEASE_URL = os.getenv("RELEASE_URL", "https://connectors.airbyte.com/files/airbyte-ci/releases") +if TYPE_CHECKING: + from typing import Optional + -def _get_custom_certificate_path(): +def _get_custom_certificate_path() -> Optional[str]: """ Returns the path to the custom certificate file if certifi is installed, otherwise None. @@ -41,10 +47,10 @@ def _get_custom_certificate_path(): return certifi.where() except ImportError: - return + return None -def get_ssl_context(): +def get_ssl_context() -> ssl.SSLContext: """ Returns an ssl.SSLContext object with the custom certificate file if certifi is installed, otherwise returns the default ssl.SSLContext object. @@ -56,22 +62,22 @@ def get_ssl_context(): return ssl.create_default_context(cafile=certifi_path) -def get_airbyte_os_name(): +def get_airbyte_os_name() -> Optional[str]: """ Returns 'ubuntu' if the system is Linux or 'macos' if the system is macOS. """ OS = os.uname().sysname if OS == "Linux": - print(f"Linux based system detected.") + print("Linux based system detected.") return "ubuntu" elif OS == "Darwin": - print(f"macOS based system detected.") + print("macOS based system detected.") return "macos" else: return None -def main(version="latest"): +def main(version: str = "latest") -> None: # Determine the operating system os_name = get_airbyte_os_name() if os_name is None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 4a65628eaea4..e11cc5664b23 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -47,7 +47,7 @@ async def cache_latest_cdk(context: ConnectorContext) -> None: ) -def never_fail_exec(command: List[str]) -> Callable: +def never_fail_exec(command: List[str]) -> Callable[[Container], Container]: """ Wrap a command execution with some bash sugar to always exit with a 0 exit code but write the actual exit code to a file. @@ -66,7 +66,7 @@ def never_fail_exec(command: List[str]) -> Callable: Callable: _description_ """ - def never_fail_exec_inner(container: Container): + def never_fail_exec_inner(container: Container) -> Container: return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True) return never_fail_exec_inner diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index e4040ce62bdb..4f601b7e83dc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from logging import Logger -from typing import Any, List +from typing import Any, List, Optional import asyncclick as click import asyncer @@ -47,10 +47,12 @@ @dataclass class LogOptions: quiet: bool = True - help_message: str = None + help_message: Optional[str] = None -def log_command_results(ctx: click.Context, command_results: List[CommandResult], logger: Logger, options: LogOptions = LogOptions()): +def log_command_results( + ctx: click.Context, command_results: List[CommandResult], logger: Logger, options: LogOptions = LogOptions() +) -> None: """ Log the output of the subcommands run by `run_all_subcommands`. """ @@ -95,4 +97,4 @@ def get_all_sibling_commands(ctx: click.Context) -> List[click.Command]: """ Get all sibling commands of the current command. """ - return [c for c in ctx.parent.command.commands.values() if c.name != ctx.command.name] + return [c for c in ctx.parent.command.commands.values() if c.name != ctx.command.name] # type: ignore diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py new file mode 100644 index 000000000000..726fdd98bf23 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import re + +import requests # type: ignore +from dagger import Directory + + +def get_latest_python_cdk_version() -> str: + """ + Get the latest version of airbyte-cdk from pypi + """ + cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" + response = requests.get(cdk_pypi_url) + response.raise_for_status() + package_info = response.json() + return package_info["info"]["version"] + + +async def get_latest_java_cdk_version(repo_dir: Directory) -> str: + version_file_content = await repo_dir.file("airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties").contents() + match = re.search(r"version *= *(?P[0-9]*\.[0-9]*\.[0-9]*)", version_file_content) + if match: + return match.group("version") + raise ValueError("Could not find version in version.properties") diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py index c70bd4e19a0d..7fe4d1be191d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py @@ -4,7 +4,7 @@ from pathlib import Path -import yaml +import yaml # type: ignore from dagger import Directory # Helpers @@ -14,8 +14,16 @@ async def get_current_metadata(repo_dir: Directory, metadata_path: Path) -> dict return yaml.safe_load(await repo_dir.file(str(metadata_path)).contents()) +async def get_current_metadata_str(repo_dir: Directory, metadata_path: Path) -> str: + return await repo_dir.file(str(metadata_path)).contents() + + def get_repo_dir_with_updated_metadata(repo_dir: Directory, metadata_path: Path, updated_metadata: dict) -> Directory: - return repo_dir.with_new_file(str(metadata_path), yaml.safe_dump(updated_metadata)) + return repo_dir.with_new_file(str(metadata_path), contents=yaml.safe_dump(updated_metadata)) + + +def get_repo_dir_with_updated_metadata_str(repo_dir: Directory, metadata_path: Path, updated_metadata_str: str) -> Directory: + return repo_dir.with_new_file(str(metadata_path), contents=updated_metadata_str) def get_current_version(current_metadata: dict) -> str: diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py index 2cf33dd925ca..b58258ef3465 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py @@ -6,8 +6,7 @@ from pathlib import Path from typing import FrozenSet, Set, Union -from anyio import Path -from connector_ops.utils import Connector +from connector_ops.utils import Connector # type: ignore from pipelines import main_logger from pipelines.helpers.utils import IGNORED_FILE_EXTENSIONS, METADATA_FILE_NAME @@ -64,7 +63,7 @@ def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Conne @dataclass(frozen=True) class ConnectorWithModifiedFiles(Connector): - modified_files: Set[Path] = field(default_factory=frozenset) + modified_files: FrozenSet[Path] = field(default_factory=frozenset) @property def has_metadata_change(self) -> bool: diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py new file mode 100644 index 000000000000..af32aa52b213 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py @@ -0,0 +1,66 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +import asyncclick as click + +if TYPE_CHECKING: + from enum import Enum + from typing import Callable, Dict, Tuple, Type + + from pipelines.models.steps import STEP_PARAMS + +# Pattern for extra param options: --.= +EXTRA_PARAM_PATTERN_FOR_OPTION = re.compile(r"^--([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_-][a-zA-Z0-9_-]*)=([^=]+)$") +# Pattern for extra param flag: --. +EXTRA_PARAM_PATTERN_FOR_FLAG = re.compile(r"^--([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_-][a-zA-Z0-9_-]*)$") +EXTRA_PARAM_PATTERN_ERROR_MESSAGE = "The extra flags must be structured as --. for flags or --.= for options. You can use - or -- for option/flag names." + + +def build_extra_params_mapping(SupportedStepIds: Type[Enum]) -> Callable: + def callback(ctx: click.Context, argument: click.core.Argument, raw_extra_params: Tuple[str]) -> Dict[str, STEP_PARAMS]: + """Build a mapping of step id to extra params. + Validate the extra params and raise a ValueError if they are invalid. + Validation rules: + - The extra params must be structured as --.= for options or --. for flags. + - The step id must be one of the existing step ids. + + + Args: + ctx (click.Context): The click context. + argument (click.core.Argument): The click argument. + raw_extra_params (Tuple[str]): The extra params provided by the user. + Raises: + ValueError: Raised if the extra params format is invalid. + ValueError: Raised if the step id in the extra params is not one of the unique steps to run. + + Returns: + Dict[Literal, STEP_PARAMS]: The mapping of step id to extra params. + """ + extra_params_mapping: Dict[str, STEP_PARAMS] = {} + for param in raw_extra_params: + is_flag = "=" not in param + pattern = EXTRA_PARAM_PATTERN_FOR_FLAG if is_flag else EXTRA_PARAM_PATTERN_FOR_OPTION + matches = pattern.match(param) + if not matches: + raise ValueError(f"Invalid parameter {param}. {EXTRA_PARAM_PATTERN_ERROR_MESSAGE}") + if is_flag: + step_name, param_name = matches.groups() + param_value = None + else: + step_name, param_name, param_value = matches.groups() + try: + step_id = SupportedStepIds(step_name).value + except ValueError: + raise ValueError(f"Invalid step name {step_name}, it must be one of {[step_id.value for step_id in SupportedStepIds]}") + + extra_params_mapping.setdefault(step_id, {}).setdefault(param_name, []) + # param_value is None if the param is a flag + if param_value is not None: + extra_params_mapping[step_id][param_name].append(param_value) + return extra_params_mapping + + return callback diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py new file mode 100644 index 000000000000..9517e1f8d40b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py @@ -0,0 +1,328 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""The actions package is made to declare reusable pipeline components.""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union + +import anyio +import asyncer +from pipelines import main_logger +from pipelines.models.steps import StepStatus + +if TYPE_CHECKING: + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID + from pipelines.models.steps import STEP_PARAMS, Step, StepResult + + RESULTS_DICT = Dict[str, StepResult] + ARGS_TYPE = Union[Dict, Callable[[RESULTS_DICT], Dict], Awaitable[Dict]] + + +class InvalidStepConfiguration(Exception): + pass + + +def _get_dependency_graph(steps: STEP_TREE) -> Dict[str, List[str]]: + """ + Get the dependency graph of a step tree. + """ + dependency_graph: Dict[str, List[str]] = {} + for step in steps: + if isinstance(step, StepToRun): + dependency_graph[step.id] = step.depends_on + elif isinstance(step, list): + nested_dependency_graph = _get_dependency_graph(list(step)) + dependency_graph = {**dependency_graph, **nested_dependency_graph} + else: + raise Exception(f"Unexpected step type: {type(step)}") + + return dependency_graph + + +def _get_transitive_dependencies_for_step_id( + dependency_graph: Dict[str, List[str]], step_id: str, visited: Optional[Set[str]] = None +) -> List[str]: + """Get the transitive dependencies for a step id. + + Args: + dependency_graph (Dict[str, str]): The dependency graph to use. + step_id (str): The step id to get the transitive dependencies for. + visited (Optional[Set[str]], optional): The set of visited step ids. Defaults to None. + + Returns: + List[str]: List of transitive dependencies as step ids. + """ + if visited is None: + visited = set() + + if step_id not in visited: + visited.add(step_id) + + dependencies: List[str] = dependency_graph.get(step_id, []) + for dependency in dependencies: + dependencies.extend(_get_transitive_dependencies_for_step_id(dependency_graph, dependency, visited)) + + return dependencies + else: + return [] + + +@dataclass +class RunStepOptions: + """Options for the run_step function.""" + + fail_fast: bool = True + skip_steps: List[str] = field(default_factory=list) + keep_steps: List[str] = field(default_factory=list) + log_step_tree: bool = True + concurrency: int = 10 + step_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.skip_steps and self.keep_steps: + raise ValueError("Cannot use both skip_steps and keep_steps at the same time") + + def get_step_ids_to_skip(self, runnables: STEP_TREE) -> List[str]: + if self.skip_steps: + return self.skip_steps + if self.keep_steps: + step_ids_to_keep = set(self.keep_steps) + dependency_graph = _get_dependency_graph(runnables) + all_step_ids = set(dependency_graph.keys()) + for step_id in self.keep_steps: + step_ids_to_keep.update(_get_transitive_dependencies_for_step_id(dependency_graph, step_id)) + return list(all_step_ids - step_ids_to_keep) + return [] + + +@dataclass(frozen=True) +class StepToRun: + """ + A class to wrap a Step with its id and args. + + Used to coordinate the execution of multiple steps inside a pipeline. + """ + + id: CONNECTOR_TEST_STEP_ID + step: Step + args: ARGS_TYPE = field(default_factory=dict) + depends_on: List[str] = field(default_factory=list) + + +STEP_TREE = List[StepToRun | List[StepToRun]] + + +async def evaluate_run_args(args: ARGS_TYPE, results: RESULTS_DICT) -> Dict: + """ + Evaluate the args of a StepToRun using the results of previous steps. + """ + if inspect.iscoroutinefunction(args): + return await args(results) + elif callable(args): + return args(results) + elif isinstance(args, dict): + return args + + raise TypeError(f"Unexpected args type: {type(args)}") + + +def _skip_remaining_steps(remaining_steps: STEP_TREE) -> RESULTS_DICT: + """ + Skip all remaining steps. + """ + skipped_results: Dict[str, StepResult] = {} + for runnable_step in remaining_steps: + if isinstance(runnable_step, StepToRun): + skipped_results[runnable_step.id] = runnable_step.step.skip() + elif isinstance(runnable_step, list): + nested_skipped_results = _skip_remaining_steps(list(runnable_step)) + skipped_results = {**skipped_results, **nested_skipped_results} + else: + raise Exception(f"Unexpected step type: {type(runnable_step)}") + + return skipped_results + + +def _step_dependencies_succeeded(step_to_eval: StepToRun, results: RESULTS_DICT) -> bool: + """ + Check if all dependencies of a step have succeeded. + """ + main_logger.info(f"Checking if dependencies {step_to_eval.depends_on} have succeeded") + + # Check if all depends_on keys are in the results dict + # If not, that means a step has not been run yet + # Implying that the order of the steps are not correct + for step_id in step_to_eval.depends_on: + if step_id not in results: + raise InvalidStepConfiguration( + f"Step {step_to_eval.id} depends on {step_id} which has not been run yet. This implies that the order of the steps is not correct. Please check that the steps are in the correct order." + ) + + return all(results[step_id] and results[step_id].status is StepStatus.SUCCESS for step_id in step_to_eval.depends_on) + + +def _filter_skipped_steps(steps_to_evaluate: STEP_TREE, skip_steps: List[str], results: RESULTS_DICT) -> Tuple[STEP_TREE, RESULTS_DICT]: + """ + Filter out steps that should be skipped. + + Either because they are in the skip list or because one of their dependencies failed. + """ + steps_to_run: STEP_TREE = [] + for step_to_eval in steps_to_evaluate: + + # ignore nested steps + if isinstance(step_to_eval, list): + steps_to_run.append(step_to_eval) + continue + + # skip step if its id is in the skip list + if step_to_eval.id in skip_steps: + main_logger.info(f"Skipping step {step_to_eval.id}") + results[step_to_eval.id] = step_to_eval.step.skip("Skipped by user") + + # skip step if a dependency failed + elif not _step_dependencies_succeeded(step_to_eval, results): + main_logger.info( + f"Skipping step {step_to_eval.id} because one of the dependencies have not been met: {step_to_eval.depends_on}" + ) + results[step_to_eval.id] = step_to_eval.step.skip("Skipped because a dependency was not met") + + else: + steps_to_run.append(step_to_eval) + + return steps_to_run, results + + +def _get_next_step_group(steps: STEP_TREE) -> Tuple[STEP_TREE, STEP_TREE]: + """ + Get the next group of steps to run concurrently. + """ + if not steps: + return [], [] + + if isinstance(steps[0], list): + return list(steps[0]), list(steps[1:]) + else: + # Termination case: if the next step is not a list that means we have reached the max depth + return steps, [] + + +def _log_step_tree(step_tree: STEP_TREE, options: RunStepOptions, depth: int = 0) -> None: + """ + Log the step tree to the console. + + e.g. + Step tree + - step1 + - step2 + - step3 + - step4 (skip) + - step5 + - step6 + """ + indent = " " + for steps in step_tree: + if isinstance(steps, list): + _log_step_tree(list(steps), options, depth + 1) + else: + if steps.id in options.skip_steps: + main_logger.info(f"{indent * depth}- {steps.id} (skip)") + else: + main_logger.info(f"{indent * depth}- {steps.id}") + + +async def run_steps( + runnables: STEP_TREE, + results: RESULTS_DICT = {}, + options: RunStepOptions = RunStepOptions(), +) -> RESULTS_DICT: + """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist. + + Examples + -------- + >>> from pipelines.models.steps import Step, StepResult, StepStatus + >>> class TestStep(Step): + ... async def _run(self) -> StepResult: + ... return StepResult(step=self, status=StepStatus.SUCCESS) + >>> steps = [ + ... StepToRun(id="step1", step=TestStep()), + ... [ + ... StepToRun(id="step2", step=TestStep()), + ... StepToRun(id="step3", step=TestStep()), + ... ], + ... StepToRun(id="step4", step=TestStep()), + ... ] + >>> results = await run_steps(steps) + >>> results["step1"].status + + >>> results["step2"].status + + >>> results["step3"].status + + >>> results["step4"].status + + + + Args: + runnables (List[StepToRun]): List of steps to run. + results (RESULTS_DICT, optional): Dictionary of step results, used for recursion. + + Returns: + RESULTS_DICT: Dictionary of step results. + """ + # If there are no steps to run, return the results + if not runnables: + return results + + step_ids_to_skip = options.get_step_ids_to_skip(runnables) + # Log the step tree + if options.log_step_tree: + main_logger.info(f"STEP TREE: {runnables}") + _log_step_tree(runnables, options) + options.log_step_tree = False + + # If any of the previous steps failed, skip the remaining steps + if options.fail_fast and any(result.status is StepStatus.FAILURE for result in results.values()): + skipped_results = _skip_remaining_steps(runnables) + return {**results, **skipped_results} + + # Pop the next step to run + steps_to_evaluate, remaining_steps = _get_next_step_group(runnables) + + # Remove any skipped steps + steps_to_run, results = _filter_skipped_steps(steps_to_evaluate, step_ids_to_skip, results) + + # Run all steps in list concurrently + semaphore = anyio.Semaphore(options.concurrency) + async with semaphore: + async with asyncer.create_task_group() as task_group: + tasks = [] + for step_to_run in steps_to_run: + # if the step to run is a list, run it in parallel + if isinstance(step_to_run, list): + tasks.append(task_group.soonify(run_steps)(list(step_to_run), results, options)) + else: + step_args = await evaluate_run_args(step_to_run.args, results) + step_to_run.step.extra_params = options.step_params.get(step_to_run.id, {}) + main_logger.info(f"QUEUING STEP {step_to_run.id}") + tasks.append(task_group.soonify(step_to_run.step.run)(**step_args)) + + # Apply new results + new_results: Dict[str, Any] = {} + for i, task in enumerate(tasks): + step_to_run = steps_to_run[i] + if isinstance(step_to_run, list): + new_results = {**new_results, **task.value} + else: + new_results[step_to_run.id] = task.value + + return await run_steps( + runnables=remaining_steps, + results={**results, **new_results}, + options=options, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py index 17e4c016d263..71fbce43b44b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py @@ -4,10 +4,10 @@ import json from pathlib import Path -from typing import Optional, Tuple +from typing import Tuple -from google.cloud import storage -from google.oauth2 import service_account +from google.cloud import storage # type: ignore +from google.oauth2 import service_account # type: ignore from pipelines import main_logger from pipelines.consts import GCS_PUBLIC_DOMAIN @@ -36,7 +36,7 @@ def upload_to_gcs(file_path: Path, bucket_name: str, object_name: str, credentia return gcs_uri, public_url -def sanitize_gcs_credentials(raw_value: Optional[str]) -> Optional[str]: +def sanitize_gcs_credentials(raw_value: str) -> str: """Try to parse the raw string input that should contain a json object with the GCS credentials. It will raise an exception if the parsing fails and help us to fail fast on invalid credentials input. @@ -46,6 +46,4 @@ def sanitize_gcs_credentials(raw_value: Optional[str]) -> Optional[str]: Returns: str: The raw value string if it was successfully parsed. """ - if raw_value is None: - return None return json.dumps(json.loads(raw_value)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py index 0ed1daf81ac5..682b77cd4503 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -6,7 +6,8 @@ from typing import Set import git -from dagger import Connection +from dagger import Connection, SessionError +from pipelines.consts import CIContext from pipelines.dagger.containers.git import checked_out_git_container from pipelines.helpers.utils import DAGGER_CONFIG, DIFF_FILTER @@ -20,14 +21,20 @@ def get_current_git_branch() -> str: # noqa D103 async def get_modified_files_in_branch_remote( - current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master" + current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master", retries: int = 3 ) -> Set[str]: """Use git diff to spot the modified files on the remote branch.""" - async with Connection(DAGGER_CONFIG) as dagger_client: - container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision, diffed_branch) - modified_files = await container.with_exec( - ["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_branch}"] - ).stdout() + try: + async with Connection(DAGGER_CONFIG) as dagger_client: + container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision, diffed_branch) + modified_files = await container.with_exec( + ["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_branch}"] + ).stdout() + except SessionError: + if retries > 0: + return await get_modified_files_in_branch_remote(current_git_branch, current_git_revision, diffed_branch, retries - 1) + else: + raise return set(modified_files.split("\n")) @@ -53,10 +60,16 @@ async def get_modified_files_in_branch( return await get_modified_files_in_branch_remote(current_git_branch, current_git_revision, diffed_branch) -async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str) -> Set[str]: - async with Connection(DAGGER_CONFIG) as dagger_client: - container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision) - modified_files = await container.with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"]).stdout() +async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str, retries: int = 3) -> Set[str]: + try: + async with Connection(DAGGER_CONFIG) as dagger_client: + container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision) + modified_files = await container.with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"]).stdout() + except SessionError: + if retries > 0: + return await get_modified_files_in_commit_remote(current_git_branch, current_git_revision, retries - 1) + else: + raise return set(modified_files.split("\n")) @@ -82,4 +95,19 @@ def get_git_repo() -> git.Repo: @functools.cache def get_git_repo_path() -> str: """Retrieve the git repo path.""" - return get_git_repo().working_tree_dir + return str(get_git_repo().working_tree_dir) + + +async def get_modified_files(git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext) -> Set[str]: + """Get the list of modified files in the current git branch. + If the current branch is master, it will return the list of modified files in the head commit. + The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge. + Pipelines like "publish on merge" are triggered on each new commit on master. + + If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff. + If the current branch is not master, it will return the list of modified files in the current branch. + This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event. + """ + if ci_context is CIContext.MASTER or (ci_context is CIContext.MANUAL and git_branch == "master"): + return await get_modified_files_in_commit(git_branch, git_revision, is_local) + return await get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index 3bde8c61f9d2..867c0fa896b7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -9,7 +9,7 @@ import os from typing import TYPE_CHECKING, Optional -from connector_ops.utils import console +from connector_ops.utils import console # type: ignore from pipelines import main_logger from pipelines.consts import CIContext @@ -33,8 +33,15 @@ def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> Non def update_commit_status_check( - sha: str, state: str, target_url: str, description: str, context: str, is_optional=False, should_send=True, logger: Logger = None -): + sha: str, + state: str, + target_url: str, + description: str, + context: str, + is_optional: bool = False, + should_send: bool = True, + logger: Optional[Logger] = None, +) -> None: """Call the GitHub API to create commit status check. Args: @@ -77,7 +84,7 @@ def update_commit_status_check( safe_log(logger, f"Created {state} status for commit {sha} on Github in {context} context with desc: {description}.") -def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest: +def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest.PullRequest: """Get a pull request object from its number. Args: @@ -91,7 +98,7 @@ def get_pull_request(pull_request_number: int, github_access_token: str) -> Pull return airbyte_repo.get_pull(pull_request_number) -def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Logger = None): +def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Optional[Logger] = None) -> None: update_commit_status_check( click_context["git_revision"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py new file mode 100644 index 000000000000..5076e6b401ce --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Optional + +import requests # type: ignore + + +def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool: + """ + Check if a package with a specific version is published on a python registry. + """ + if not package_name or not version: + return False + + url = f"{registry_url}/{package_name}/{version}/json" + + try: + response = requests.get(url) + return response.status_code == 200 + except requests.exceptions.ConnectionError: + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py index da36bb015ebd..fb1a44138a60 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py @@ -1,24 +1,33 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import importlib.metadata import os +from typing import TYPE_CHECKING import sentry_sdk -from connector_ops.utils import Connector +from connector_ops.utils import Connector # type: ignore +if TYPE_CHECKING: + from typing import Any, Callable, Dict, Optional -def initialize(): + from asyncclick import Command, Context + from pipelines.models.steps import Step + + +def initialize() -> None: if "SENTRY_DSN" in os.environ: sentry_sdk.init( dsn=os.environ.get("SENTRY_DSN"), + environment=os.environ.get("SENTRY_ENVIRONMENT") or "production", before_send=before_send, release=f"pipelines@{importlib.metadata.version('pipelines')}", ) -def before_send(event, hint): +def before_send(event: Dict[str, Any], hint: Dict[str, Any]) -> Optional[Dict[str, Any]]: # Ignore logged errors that do not contain an exception if "log_record" in hint and "exc_info" not in hint: return None @@ -26,8 +35,8 @@ def before_send(event, hint): return event -def with_step_context(func): - def wrapper(self, *args, **kwargs): +def with_step_context(func: Callable) -> Callable: + def wrapper(self: Step, *args: Any, **kwargs: Any) -> Step: with sentry_sdk.configure_scope() as scope: step_name = self.__class__.__name__ scope.set_tag("pipeline_step", step_name) @@ -61,8 +70,8 @@ def wrapper(self, *args, **kwargs): return wrapper -def with_command_context(func): - def wrapper(self, ctx, *args, **kwargs): +def with_command_context(func: Callable) -> Callable: + def wrapper(self: Command, ctx: Context, *args: Any, **kwargs: Any) -> Command: with sentry_sdk.configure_scope() as scope: scope.set_tag("pipeline_command", self.name) scope.set_context( diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index ecc4e23e0996..619be4278b57 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -8,7 +8,7 @@ from pipelines import main_logger -def send_message_to_webhook(message: str, channel: str, webhook: str) -> dict: +def send_message_to_webhook(message: str, channel: str, webhook: str) -> requests.Response: payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message} response = requests.post(webhook, data={"payload": json.dumps(payload)}) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py deleted file mode 100644 index 95ad77c0b996..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""The actions package is made to declare reusable pipeline components.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, List, Tuple, Union - -import asyncer -from pipelines.models.steps import Step, StepStatus - -if TYPE_CHECKING: - from pipelines.models.steps import StepResult - - -async def run_steps( - steps_and_run_args: List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]], results: List[StepResult] = [] -) -> List[StepResult]: - """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist. - - Args: - steps_and_run_args (List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]]): List of steps to run, if steps are wrapped in a sublist they will be executed in parallel. run function arguments can be passed as a tuple along the Step instance. - results (List[StepResult], optional): List of step results, used for recursion. - - Returns: - List[StepResult]: List of step results. - """ - # If there are no steps to run, return the results - if not steps_and_run_args: - return results - - # If any of the previous steps failed, skip the remaining steps - if any(result.status is StepStatus.FAILURE for result in results): - skipped_results = [] - for step_and_run_args in steps_and_run_args: - if isinstance(step_and_run_args, Tuple): - skipped_results.append(step_and_run_args[0].skip()) - else: - skipped_results.append(step_and_run_args.skip()) - return results + skipped_results - - # Pop the next step to run - steps_to_run, remaining_steps = steps_and_run_args[0], steps_and_run_args[1:] - - # wrap the step in a list if it is not already (allows for parallel steps) - if not isinstance(steps_to_run, list): - steps_to_run = [steps_to_run] - - async with asyncer.create_task_group() as task_group: - tasks = [] - for step in steps_to_run: - if isinstance(step, Step): - tasks.append(task_group.soonify(step.run)()) - elif isinstance(step, Tuple) and isinstance(step[0], Step) and isinstance(step[1], Tuple): - step, run_args = step - tasks.append(task_group.soonify(step.run)(*run_args)) - - new_results = [task.value for task in tasks] - - return await run_steps(remaining_steps, results + new_results) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 83ea8b42a847..cd8d31dd4d40 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -11,17 +11,20 @@ import re import sys import unicodedata +import xml.sax.saxutils from io import TextIOWrapper from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set, Tuple +from typing import TYPE_CHECKING import anyio +import asyncclick as click import asyncer -import click -from dagger import Client, Config, Container, ExecError, File, ImageLayerCompression, Platform, QueryError, Secret +from dagger import Client, Config, Container, Directory, ExecError, File, ImageLayerCompression, Platform, Secret from more_itertools import chunked if TYPE_CHECKING: + from typing import Any, Callable, Generator, List, Optional, Set, Tuple + from pipelines.airbyte_ci.connectors.context import ConnectorContext DAGGER_CONFIG = Config(log_output=sys.stderr) @@ -52,7 +55,7 @@ async def check_path_in_workdir(container: Container, path: str) -> bool: return False -def secret_host_variable(client: Client, name: str, default: str = ""): +def secret_host_variable(client: Client, name: str, default: str = "") -> Callable[[Container], Container]: """Add a host environment variable as a secret in a container. Example: @@ -68,7 +71,7 @@ def secret_host_variable(client: Client, name: str, default: str = ""): Callable[[Container], Container]: A function that can be used in a `Container.with_()` method. """ - def _secret_host_variable(container: Container): + def _secret_host_variable(container: Container) -> Container: return container.with_secret_variable(name, get_secret_host_variable(client, name, default)) return _secret_host_variable @@ -99,17 +102,14 @@ async def get_file_contents(container: Container, path: str) -> Optional[str]: Returns: Optional[str]: The file content if the file exists in the container, None otherwise. """ - try: - return await container.file(path).contents() - except QueryError as e: - if "no such file or directory" not in str(e): - # this error could come from a network issue - raise - return None + dir_name, file_name = os.path.split(path) + if file_name not in set(await container.directory(dir_name).entries()): + return None + return await container.file(path).contents() @contextlib.contextmanager -def catch_exec_error_group(): +def catch_exec_error_group() -> Generator: try: yield except anyio.ExceptionGroup as eg: @@ -197,7 +197,7 @@ def get_current_epoch_time() -> int: # noqa D103 return round(datetime.datetime.utcnow().timestamp()) -def slugify(value: Any, allow_unicode: bool = False): +def slugify(value: object, allow_unicode: bool = False) -> str: """ Taken from https://github.com/django/django/blob/master/django/utils/text.py. @@ -257,7 +257,7 @@ def create_and_open_file(file_path: Path) -> TextIOWrapper: return file_path.open("w") -async def execute_concurrently(steps: List[Callable], concurrency=5): +async def execute_concurrently(steps: List[Callable], concurrency: int = 5) -> List[Any]: tasks = [] # Asyncer does not have builtin semaphore, so control concurrency via chunks of steps # Anyio has semaphores but does not have the soonify method which allow access to results via the value task attribute. @@ -322,8 +322,55 @@ def transform_strs_to_paths(str_paths: Set[str]) -> List[Path]: return sorted([Path(str_path) for str_path in str_paths]) -def fail_if_missing_docker_hub_creds(ctx: click.Context): +def fail_if_missing_docker_hub_creds(ctx: click.Context) -> None: if ctx.obj["docker_hub_username"] is None or ctx.obj["docker_hub_password"] is None: raise click.UsageError( "You need to be logged to DockerHub registry to run this command. Please set DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables." ) + + +def java_log_scrub_pattern(secrets_to_mask: List[str]) -> str: + """Transforms a list of secrets into a LOG_SCRUB_PATTERN env var value for our log4j test configuration.""" + # Build a regex pattern that matches any of the secrets to mask. + regex_pattern = "|".join(map(re.escape, secrets_to_mask)) + # Now, make this string safe to consume by the log4j configuration. + # Its parser is XML-based so the pattern needs to be escaped again, and carefully. + return xml.sax.saxutils.escape( + regex_pattern, + # Annoyingly, the log4j properties file parser is quite brittle when it comes to + # handling log message patterns. In our case the env var is injected like this: + # + # ${env:LOG_SCRUB_PATTERN:-defaultvalue} + # + # We must avoid confusing the parser with curly braces or colons otherwise the + # printed log messages will just consist of `%replace`. + { + "\t": " ", + "'": "'", + '"': """, + "{": "{", + "}": "}", + ":": ":", + }, + ) + + +def dagger_directory_as_zip_file(dagger_client: Client, directory: Directory, directory_name: str) -> File: + """Compress a directory and return a File object representing the zip file. + + Args: + dagger_client (Client): The dagger client. + directory (Path): The directory to compress. + directory_name (str): The name of the directory. + + Returns: + File: The File object representing the zip file. + """ + return ( + dagger_client.container() + .from_("alpine:3.19.1") + .with_exec(sh_dash_c(["apk update", "apk add zip"])) + .with_mounted_directory(f"/{directory_name}", directory) + .with_exec(["zip", "-r", "/zipped.zip", f"/{directory_name}"]) + .file("/zipped.zip") + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py new file mode 100644 index 000000000000..f1deafd445e7 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import dagger +from pipelines.consts import GCS_PUBLIC_DOMAIN +from pipelines.dagger.actions import remote_storage + + +@dataclass(kw_only=True) +class Artifact: + """A dataclass to represent an artifact produced by a pipeline execution.""" + + name: str + content_type: str + content: dagger.File + to_upload: bool = True + local_path: Optional[Path] = None + gcs_url: Optional[str] = None + + async def save_to_local_path(self, path: Path) -> Path: + exported = await self.content.export(str(path)) + if exported: + self.local_path = path + return path + else: + raise Exception(f"Failed to save artifact {self.name} to local path {path}") + + async def upload_to_gcs(self, dagger_client: dagger.Client, bucket: str, key: str, gcs_credentials: dagger.Secret) -> str: + gcs_cp_flags = [f'--content-disposition=filename="{self.name}"'] + if self.content_type is not None: + gcs_cp_flags = gcs_cp_flags + [f"--content-type={self.content_type}"] + + report_upload_exit_code, _, _ = await remote_storage.upload_to_gcs( + dagger_client=dagger_client, + file_to_upload=self.content, + key=key, + bucket=bucket, + gcs_credentials=gcs_credentials, + flags=gcs_cp_flags, + ) + if report_upload_exit_code != 0: + raise Exception(f"Failed to upload artifact {self.name} to GCS. Exit code: {report_upload_exit_code}.") + self.gcs_url = f"{GCS_PUBLIC_DOMAIN}/{bucket}/{key}" + return f"{GCS_PUBLIC_DOMAIN}/{bucket}/{key}" diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py new file mode 100644 index 000000000000..8da589d2fffa --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from dataclasses import dataclass +from importlib import metadata + + +@dataclass +class CIRequirements: + """ + A dataclass to store the CI requirements. + It used to make airbyte-ci client define the CI runners it will run on. + """ + + dagger_version = metadata.version("dagger-io") + + @property + def dagger_engine_image(self) -> str: + return f"registry.dagger.io/engine:v{self.dagger_version}" + + def to_json(self) -> str: + return json.dumps( + { + "dagger_version": self.dagger_version, + "dagger_engine_image": self.dagger_engine_image, + } + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index d983343d3c71..e3182bbd0377 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -2,13 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import io import sys -from typing import Any, Callable, Optional, TextIO +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, Optional, TextIO, Tuple import anyio import dagger from asyncclick import Context, get_current_context -from dagger.api.gen import Client, Container +from pipelines import main_logger from pipelines.cli.click_decorators import LazyPassDecorator from pydantic import BaseModel, Field, PrivateAttr @@ -23,13 +26,13 @@ class ClickPipelineContext(BaseModel, Singleton): Dagger client, which is used to create containers for running pipelines. """ - dockerd_service: Optional[Container] = Field(default=None) - _dagger_client: Optional[Client] = PrivateAttr(default=None) + dockerd_service: Optional[dagger.Container] = Field(default=None) + _dagger_client: Optional[dagger.Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) - _og_click_context: Callable[[], Context] = PrivateAttr(default=None) + _og_click_context: Context = PrivateAttr(default=None) @property - def params(self): + def params(self) -> Dict[str, Any]: """ Returns a combination of the click context object and the click context params. @@ -54,7 +57,7 @@ def params(self): class Config: arbitrary_types_allowed = True - def __init__(self, **data: dict[str, Any]): + def __init__(self, **data: dict[str, Any]) -> None: """ Initialize the ClickPipelineContext instance. @@ -76,15 +79,15 @@ def __init__(self, **data: dict[str, Any]): _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) - async def get_dagger_client(self, pipeline_name: Optional[str] = None, log_output: Optional[TextIO] = sys.stdout) -> Client: + async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> dagger.Client: """ Get (or initialize) the Dagger Client instance. """ if not self._dagger_client: async with self._dagger_client_lock: if not self._dagger_client: - connection = dagger.Connection(dagger.Config(log_output=log_output)) + connection = dagger.Connection(dagger.Config(log_output=self.get_log_output())) """ Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. @@ -97,6 +100,23 @@ async def get_dagger_client(self, pipeline_name: Optional[str] = None, log_outpu assert self._dagger_client, "Error initializing Dagger client" return self._dagger_client.pipeline(pipeline_name) if pipeline_name else self._dagger_client + def get_log_output(self) -> TextIO: + # This `show_dagger_logs` flag is likely going to be removed in the future. + # See https://github.com/airbytehq/airbyte/issues/33487 + if self.params.get("show_dagger_logs", False): + return sys.stdout + else: + log_output, self._click_context().obj["dagger_logs_path"] = self._create_dagger_client_log_file() + return log_output + + def _create_dagger_client_log_file(self) -> Tuple[TextIO, Path]: + """ + Create the dagger client log file. + """ + dagger_logs_file_descriptor, dagger_logs_temp_file_path = tempfile.mkstemp(dir="/tmp", prefix="dagger_client_", suffix=".log") + main_logger.info(f"Dagger client logs stored in {dagger_logs_temp_file_path}") + return io.TextIOWrapper(io.FileIO(dagger_logs_file_descriptor, "w+")), Path(dagger_logs_temp_file_path) + # Create @pass_pipeline_context decorator for use in click commands pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py index 29fefe8d3679..b99fc5e6b992 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -4,44 +4,62 @@ """Module declaring context related classes.""" +from __future__ import annotations + import logging import os from datetime import datetime +from functools import lru_cache from glob import glob from types import TracebackType -from typing import List, Optional +from typing import TYPE_CHECKING from asyncer import asyncify -from dagger import Client, Directory, File, Secret +from dagger import Client, Directory, File, GitRepository, Secret, Service from github import PullRequest +from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import CIContext, ContextState +from pipelines.helpers.execution.run_steps import RunStepOptions from pipelines.helpers.gcs import sanitize_gcs_credentials from pipelines.helpers.github import update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook -from pipelines.helpers.utils import AIRBYTE_REPO_URL +from pipelines.helpers.utils import AIRBYTE_REPO_URL, java_log_scrub_pattern from pipelines.models.reports import Report +if TYPE_CHECKING: + from typing import List, Optional + class PipelineContext: """The pipeline context is used to store configuration for a specific pipeline run.""" + _dagger_client: Optional[Client] + _report: Optional[Report | ConnectorReport] + dockerd_service: Optional[Service] + started_at: Optional[datetime] + stopped_at: Optional[datetime] + + secrets_to_mask: List[str] + PRODUCTION = bool(os.environ.get("PRODUCTION", False)) # Set this to True to enable production mode (e.g. to send PR comments) - DEFAULT_EXCLUDED_FILES = ( - [".git", "airbyte-ci/connectors/pipelines/*"] - + glob("**/build", recursive=True) - + glob("**/.venv", recursive=True) - + glob("**/secrets", recursive=True) - + glob("**/__pycache__", recursive=True) - + glob("**/*.egg-info", recursive=True) - + glob("**/.vscode", recursive=True) - + glob("**/.pytest_cache", recursive=True) - + glob("**/.eggs", recursive=True) - + glob("**/.mypy_cache", recursive=True) - + glob("**/.DS_Store", recursive=True) - + glob("**/airbyte_ci_logs", recursive=True) - + glob("**/.gradle", recursive=True) - ) + @lru_cache + def get_default_excluded_files(self) -> list[str]: + return ( + [".git", "airbyte-ci/connectors/pipelines/*"] + + glob("**/build", recursive=True) + + glob("**/.venv", recursive=True) + + glob("**/secrets", recursive=True) + + glob("**/__pycache__", recursive=True) + + glob("**/*.egg-info", recursive=True) + + glob("**/.vscode", recursive=True) + + glob("**/.pytest_cache", recursive=True) + + glob("**/.eggs", recursive=True) + + glob("**/.mypy_cache", recursive=True) + + glob("**/.DS_Store", recursive=True) + + glob("**/airbyte_ci_logs", recursive=True) + + glob("**/.gradle", recursive=True) + ) def __init__( self, @@ -49,6 +67,7 @@ def __init__( is_local: bool, git_branch: str, git_revision: str, + report_output_prefix: str, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, @@ -56,13 +75,14 @@ def __init__( is_ci_optional: bool = False, slack_webhook: Optional[str] = None, reporting_slack_channel: Optional[str] = None, - pull_request: PullRequest = None, + pull_request: Optional[PullRequest.PullRequest] = None, ci_report_bucket: Optional[str] = None, ci_gcs_credentials: Optional[str] = None, ci_git_user: Optional[str] = None, ci_github_access_token: Optional[str] = None, + run_step_options: RunStepOptions = RunStepOptions(), enable_report_auto_open: bool = True, - ): + ) -> None: """Initialize a pipeline context. Args: @@ -70,6 +90,7 @@ def __init__( is_local (bool): Whether the context is for a local run or a CI run. git_branch (str): The current git branch name. git_revision (str): The current git revision, commit hash. + report_output_prefix (str): The prefix to use for the report output. gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. @@ -83,6 +104,7 @@ def __init__( self.is_local = is_local self.git_branch = git_branch self.git_revision = git_revision + self.report_output_prefix = report_output_prefix self.gha_workflow_run_url = gha_workflow_run_url self.dagger_logs_url = dagger_logs_url self.pipeline_start_timestamp = pipeline_start_timestamp @@ -94,7 +116,7 @@ def __init__( self.reporting_slack_channel = reporting_slack_channel self.pull_request = pull_request self.logger = logging.getLogger(self.pipeline_name) - self.dagger_client = None + self._dagger_client = None self._report = None self.dockerd_service = None self.ci_gcs_credentials = sanitize_gcs_credentials(ci_gcs_credentials) if ci_gcs_credentials else None @@ -104,49 +126,63 @@ def __init__( self.started_at = None self.stopped_at = None self.secrets_to_mask = [] + self.run_step_options = run_step_options self.enable_report_auto_open = enable_report_auto_open update_commit_status_check(**self.github_commit_status) @property - def dagger_client(self) -> Client: # noqa D102 + def dagger_client(self) -> Client: + assert self._dagger_client is not None, "The dagger client was not set on this PipelineContext" return self._dagger_client @dagger_client.setter - def dagger_client(self, dagger_client: Client): # noqa D102 + def dagger_client(self, dagger_client: Client) -> None: self._dagger_client = dagger_client @property - def is_ci(self): # noqa D102 + def is_ci(self) -> bool: return self.is_local is False @property - def is_pr(self): # noqa D102 + def is_pr(self) -> bool: return self.ci_context == CIContext.PULL_REQUEST @property - def repo(self): # noqa D102 + def repo(self) -> GitRepository: return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True) @property - def report(self) -> Report: # noqa D102 + def report(self) -> Report | ConnectorReport | None: return self._report @report.setter - def report(self, report: Report): # noqa D102 + def report(self, report: Report | ConnectorReport) -> None: self._report = report @property def ci_gcs_credentials_secret(self) -> Secret: + assert self.ci_gcs_credentials is not None, "The ci_gcs_credentials was not set on this PipelineContext." return self.dagger_client.set_secret("ci_gcs_credentials", self.ci_gcs_credentials) @property def ci_github_access_token_secret(self) -> Secret: + assert self.ci_github_access_token is not None, "The ci_github_access_token was not set on this PipelineContext." return self.dagger_client.set_secret("ci_github_access_token", self.ci_github_access_token) + @property + def java_log_scrub_pattern_secret(self) -> Optional[Secret]: + if not self.secrets_to_mask: + return None + return self.dagger_client.set_secret("log_scrub_pattern", java_log_scrub_pattern(self.secrets_to_mask)) + @property def github_commit_status(self) -> dict: """Build a dictionary used as kwargs to the update_commit_status_check function.""" - target_url = self.report.html_report_url if self.report else self.gha_workflow_run_url + target_url: Optional[str] = self.gha_workflow_run_url + + if self.state not in [ContextState.RUNNING, ContextState.INITIALIZED] and isinstance(self.report, ConnectorReport): + target_url = self.report.html_report_url + return { "sha": self.git_revision, "state": self.state.value["github_state"], @@ -167,7 +203,7 @@ def has_dagger_cloud_token(self) -> bool: return "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN" in os.environ @property - def dagger_cloud_url(self) -> str: + def dagger_cloud_url(self) -> Optional[str]: """Gets the link to the Dagger Cloud runs page for the current commit.""" if self.is_local or not self.has_dagger_cloud_token: return None @@ -201,10 +237,11 @@ def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, i Returns: Directory: The selected repo directory. """ + if exclude is None: - exclude = self.DEFAULT_EXCLUDED_FILES + exclude = self.get_default_excluded_files() else: - exclude += self.DEFAULT_EXCLUDED_FILES + exclude += self.get_default_excluded_files() exclude = list(set(exclude)) exclude.sort() # sort to make sure the order is always the same to not burst the cache. Casting exclude to set can change the order if subdir != ".": @@ -215,7 +252,7 @@ def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, i def create_slack_message(self) -> str: raise NotImplementedError() - async def __aenter__(self): + async def __aenter__(self) -> PipelineContext: """Perform setup operation for the PipelineContext. Updates the current commit status on Github. @@ -232,7 +269,10 @@ async def __aenter__(self): self.logger.info("Caching the latest CDK version...") await asyncify(update_commit_status_check)(**self.github_commit_status) if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)( + self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook # type: ignore + ) return self @staticmethod @@ -274,19 +314,23 @@ async def __aexit__( Returns: bool: Whether the teardown operation ran successfully. """ - self.state = self.determine_final_state(self.report, exception_value) - self.stopped_at = datetime.utcnow() - if exception_value: self.logger.error("An error was handled by the Pipeline", exc_info=True) + if self.report is None: self.logger.error("No test report was provided. This is probably due to an upstream error") self.report = Report(self, steps_results=[]) + self.state = self.determine_final_state(self.report, exception_value) + self.stopped_at = datetime.utcnow() + self.report.print() await asyncify(update_commit_status_check)(**self.github_commit_status) if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)( + self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook # type: ignore + ) # supress the exception if it was handled return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py new file mode 100644 index 000000000000..2b406e0d7887 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional, Type + +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL + + +@dataclass +class PythonPackageMetadata: + name: Optional[str] + version: Optional[str] + + +class PythonRegistryPublishContext(PipelineContext): + def __init__( + self, + python_registry_token: str, + registry_check_url: str, + package_path: str, + report_output_prefix: str, + is_local: bool, + git_branch: str, + git_revision: str, + ci_report_bucket: Optional[str] = None, + registry: str = DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + package_name: Optional[str] = None, + version: Optional[str] = None, + ) -> None: + self.python_registry_token = python_registry_token + self.registry = registry + self.registry_check_url = registry_check_url + self.package_path = package_path + self.package_metadata = PythonPackageMetadata(package_name, version) + + pipeline_name = f"Publish PyPI {package_path}" + + super().__init__( + pipeline_name=pipeline_name, + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ci_gcs_credentials=ci_gcs_credentials, + ) + + @classmethod + async def from_publish_connector_context( + cls: Type["PythonRegistryPublishContext"], connector_context: PublishConnectorContext + ) -> Optional["PythonRegistryPublishContext"]: + """ + Create a PythonRegistryPublishContext from a ConnectorContext. + + The metadata of the connector is read from the current workdir to capture changes that are not yet published. + If pypi is not enabled, this will return None. + """ + + current_metadata = connector_context.connector.metadata + connector_context.logger.info(f"Current metadata: {str(current_metadata)}") + if ( + "remoteRegistries" not in current_metadata + or "pypi" not in current_metadata["remoteRegistries"] + or not current_metadata["remoteRegistries"]["pypi"]["enabled"] + ): + return None + + version = current_metadata["dockerImageTag"] + if connector_context.pre_release: + # use current date as pre-release version + # we can't use the git revision because not all python registries allow local version identifiers. Public version identifiers must conform to PEP 440 and only allow digits. + release_candidate_tag = datetime.now().strftime("%Y%m%d%H%M") + version = f"{version}.dev{release_candidate_tag}" + + pypi_context = cls( + python_registry_token=str(connector_context.python_registry_token), + registry=str(connector_context.python_registry_url), + registry_check_url=str(connector_context.python_registry_check_url), + package_path=str(connector_context.connector.code_directory), + package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], + version=version, + ci_report_bucket=connector_context.ci_report_bucket, + report_output_prefix=connector_context.report_output_prefix, + is_local=connector_context.is_local, + git_branch=connector_context.git_branch, + git_revision=connector_context.git_revision, + gha_workflow_run_url=connector_context.gha_workflow_run_url, + dagger_logs_url=connector_context.dagger_logs_url, + pipeline_start_timestamp=connector_context.pipeline_start_timestamp, + ci_context=connector_context.ci_context, + ci_gcs_credentials=connector_context.ci_gcs_credentials, + ) + pypi_context.dagger_client = connector_context.dagger_client + return pypi_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py index 8e1ecfec6c34..4cf5c33f8055 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py @@ -7,17 +7,17 @@ from __future__ import annotations import json +import time import typing from dataclasses import dataclass, field from datetime import datetime, timedelta +from pathlib import Path from typing import List -import anyio -from anyio import Path -from connector_ops.utils import console -from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT -from pipelines.dagger.actions import remote_storage -from pipelines.helpers.utils import format_duration +from connector_ops.utils import console # type: ignore +from pipelines.consts import LOCAL_REPORTS_PATH_ROOT +from pipelines.helpers.utils import format_duration, slugify +from pipelines.models.artifacts import Artifact from pipelines.models.steps import StepResult, StepStatus from rich.console import Group from rich.panel import Panel @@ -26,92 +26,114 @@ from rich.text import Text if typing.TYPE_CHECKING: - from pipelines.models.steps import PipelineContext + from pipelines.models.contexts.pipeline_context import PipelineContext + from rich.tree import RenderableType @dataclass(frozen=True) class Report: """A dataclass to build reports to share pipelines executions results with the user.""" - pipeline_context: "PipelineContext" + pipeline_context: PipelineContext steps_results: List[StepResult] created_at: datetime = field(default_factory=datetime.utcnow) name: str = "REPORT" filename: str = "output" @property - def report_output_prefix(self) -> str: # noqa D102 + def report_output_prefix(self) -> str: return self.pipeline_context.report_output_prefix @property - def json_report_file_name(self) -> str: # noqa D102 + def report_dir_path(self) -> Path: + return Path(f"{LOCAL_REPORTS_PATH_ROOT}/{self.report_output_prefix}") + + @property + def json_report_file_name(self) -> str: return self.filename + ".json" @property - def json_report_remote_storage_key(self) -> str: # noqa D102 + def json_report_remote_storage_key(self) -> str: return f"{self.report_output_prefix}/{self.json_report_file_name}" @property - def failed_steps(self) -> List[StepResult]: # noqa D102 + def failed_steps(self) -> List[StepResult]: return [step_result for step_result in self.steps_results if step_result.status is StepStatus.FAILURE] @property - def successful_steps(self) -> List[StepResult]: # noqa D102 + def successful_steps(self) -> List[StepResult]: return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SUCCESS] @property - def skipped_steps(self) -> List[StepResult]: # noqa D102 + def skipped_steps(self) -> List[StepResult]: return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SKIPPED] @property - def success(self) -> bool: # noqa D102 + def success(self) -> bool: return len(self.failed_steps) == 0 and (len(self.skipped_steps) > 0 or len(self.successful_steps) > 0) @property - def run_duration(self) -> timedelta: # noqa D102 + def run_duration(self) -> timedelta: + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." return self.pipeline_context.stopped_at - self.pipeline_context.started_at @property - def lead_duration(self) -> timedelta: # noqa D102 + def lead_duration(self) -> timedelta: + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." return self.pipeline_context.stopped_at - self.pipeline_context.created_at @property - def remote_storage_enabled(self) -> bool: # noqa D102 + def remote_storage_enabled(self) -> bool: return self.pipeline_context.is_ci - async def save_local(self, filename: str, content: str) -> Path: - """Save the report files locally.""" - local_path = anyio.Path(f"{LOCAL_REPORTS_PATH_ROOT}/{self.report_output_prefix}/{filename}") - await local_path.parents[0].mkdir(parents=True, exist_ok=True) - await local_path.write_text(content) - return local_path - - async def save_remote(self, local_path: Path, remote_key: str, content_type: str = None) -> int: - gcs_cp_flags = None if content_type is None else [f"--content-type={content_type}"] - local_file = self.pipeline_context.dagger_client.host().directory(".", include=[str(local_path)]).file(str(local_path)) - report_upload_exit_code, _, _ = await remote_storage.upload_to_gcs( - dagger_client=self.pipeline_context.dagger_client, - file_to_upload=local_file, - key=remote_key, - bucket=self.pipeline_context.ci_report_bucket, - gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, - flags=gcs_cp_flags, - ) - gcs_uri = "gs://" + self.pipeline_context.ci_report_bucket + "/" + remote_key - public_url = f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{remote_key}" - if report_upload_exit_code != 0: - self.pipeline_context.logger.error(f"Uploading {local_path} to {gcs_uri} failed.") - else: - self.pipeline_context.logger.info(f"Uploading {local_path} to {gcs_uri} succeeded. Public URL: {public_url}") - return report_upload_exit_code - async def save(self) -> None: - """Save the report files.""" - local_json_path = await self.save_local(self.json_report_file_name, self.to_json()) - absolute_path = await local_json_path.absolute() + self.report_dir_path.mkdir(parents=True, exist_ok=True) + await self.save_json_report() + await self.save_step_result_artifacts() + + async def save_json_report(self) -> None: + """Save the report as JSON, upload it to GCS if the pipeline is running in CI""" + + json_report_path = self.report_dir_path / self.json_report_file_name + report_dir = self.pipeline_context.dagger_client.host().directory(str(self.report_dir_path)) + local_json_report_file = report_dir.with_new_file(self.json_report_file_name, self.to_json()).file(self.json_report_file_name) + json_report_artifact = Artifact(name="JSON Report", content_type="application/json", content=local_json_report_file) + await json_report_artifact.save_to_local_path(json_report_path) + absolute_path = json_report_path.absolute() self.pipeline_context.logger.info(f"Report saved locally at {absolute_path}") - if self.remote_storage_enabled: - await self.save_remote(local_json_path, self.json_report_remote_storage_key, "application/json") + if self.remote_storage_enabled and self.pipeline_context.ci_report_bucket and self.pipeline_context.ci_gcs_credentials_secret: + gcs_url = await json_report_artifact.upload_to_gcs( + dagger_client=self.pipeline_context.dagger_client, + bucket=self.pipeline_context.ci_report_bucket, + key=self.json_report_remote_storage_key, + gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, + ) + self.pipeline_context.logger.info(f"JSON Report uploaded to {gcs_url}") + + async def save_step_result_artifacts(self) -> None: + local_artifacts_dir = self.report_dir_path / "artifacts" + local_artifacts_dir.mkdir(parents=True, exist_ok=True) + # TODO: concurrent save and upload + for step_result in self.steps_results: + for artifact in step_result.artifacts: + step_artifacts_dir = local_artifacts_dir / slugify(step_result.step.title) + step_artifacts_dir.mkdir(parents=True, exist_ok=True) + await artifact.save_to_local_path(step_artifacts_dir / artifact.name) + if ( + self.remote_storage_enabled + and self.pipeline_context.ci_report_bucket + and self.pipeline_context.ci_gcs_credentials_secret + ): + upload_time = int(time.time()) + gcs_url = await artifact.upload_to_gcs( + dagger_client=self.pipeline_context.dagger_client, + bucket=self.pipeline_context.ci_report_bucket, + key=f"{self.report_output_prefix}/artifacts/{slugify(step_result.step.title)}/{upload_time}_{artifact.name}", + gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, + ) + self.pipeline_context.logger.info(f"Artifact {artifact.name} for {step_result.step.title} uploaded to {gcs_url}") def to_json(self) -> str: """Create a JSON representation of the report. @@ -119,15 +141,18 @@ def to_json(self) -> str: Returns: str: The JSON representation of the report. """ + assert self.pipeline_context.pipeline_start_timestamp is not None, "The pipeline start timestamp must be set to save reports." + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." return json.dumps( { "pipeline_name": self.pipeline_context.pipeline_name, "run_timestamp": self.pipeline_context.started_at.isoformat(), "run_duration": self.run_duration.total_seconds(), "success": self.success, - "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], - "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], - "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], # type: ignore + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], # type: ignore + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], # type: ignore "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, "pipeline_end_timestamp": round(self.pipeline_context.stopped_at.timestamp()), @@ -140,7 +165,7 @@ def to_json(self) -> str: } ) - def print(self): + def print(self) -> None: """Print the test report to the console in a nice way.""" pipeline_name = self.pipeline_context.pipeline_name main_panel_title = Text(f"{pipeline_name.upper()} - {self.name}") @@ -160,14 +185,15 @@ def print(self): if step_result.status is StepStatus.SKIPPED: step_results_table.add_row(step, result, "N/A") else: + assert step_result.step.started_at is not None, "The step started_at timestamp must be set to print reports." run_time = format_duration((step_result.created_at - step_result.step.started_at)) step_results_table.add_row(step, result, run_time) - to_render = [step_results_table] + to_render: List[RenderableType] = [step_results_table] if self.failed_steps: sub_panels = [] for failed_step in self.failed_steps: - errors = Text(failed_step.stderr) + errors = Text(failed_step.stderr) if failed_step.stderr else Text("") panel_title = Text(f"{pipeline_name} {failed_step.step.title.lower()} failures") panel_title.stylize(Style(color="red", bold=True)) sub_panel = Panel(errors, title=panel_title) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 349d9a46f995..7fdb4b6f5dda 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -19,7 +19,7 @@ class Singleton: _instances: dict[Type["Singleton"], Any] = {} _initialized: dict[Type["Singleton"], bool] = {} - def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: + def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: # noqa: ANN401 if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) cls._initialized[cls] = False diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index 5657ca240517..86be5550f713 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -10,64 +10,83 @@ from datetime import datetime, timedelta from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union +from typing import TYPE_CHECKING, Dict, List import anyio import asyncer import click -from dagger import Container, DaggerError +from dagger import Client, Container, DaggerError from pipelines import main_logger from pipelines.helpers import sentry_utils from pipelines.helpers.utils import format_duration, get_exec_result +from pipelines.models.artifacts import Artifact if TYPE_CHECKING: + from typing import Any, ClassVar, Optional, Union + from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.models.contexts.pipeline_context import PipelineContext + from abc import ABC from rich.style import Style +STEP_PARAMS = Dict[str, List[str]] + @dataclass class MountPath: - path: Path + path: Union[Path, str] optional: bool = False - def _cast_fields(self): + def _cast_fields(self) -> None: self.path = Path(self.path) self.optional = bool(self.optional) - def _check_exists(self): - if not self.path.exists(): + def _check_exists(self) -> None: + if not self.get_path().exists(): message = f"{self.path} does not exist." if self.optional: main_logger.warning(message) else: raise FileNotFoundError(message) - def __post_init__(self): + def get_path(self) -> Path: + return Path(self.path) + + def __post_init__(self) -> None: self._cast_fields() self._check_exists() - def __str__(self): + def __str__(self) -> str: return str(self.path) @property def is_file(self) -> bool: - return self.path.is_file() + return self.get_path().is_file() -@dataclass(frozen=True) -class StepResult: - """A dataclass to capture the result of a step.""" - - step: Union[Step, click.command] +@dataclass(kw_only=True, frozen=True) +class Result: status: StepStatus created_at: datetime = field(default_factory=datetime.utcnow) stderr: Optional[str] = None stdout: Optional[str] = None - output_artifact: Any = None + report: Optional[str] = None exc_info: Optional[Exception] = None + output: Any = None + artifacts: List[Artifact] = field(default_factory=list) + + @property + def success(self) -> bool: + return self.status is StepStatus.SUCCESS + + +@dataclass(kw_only=True, frozen=True) +class StepResult(Result): + """A dataclass to capture the result of a step.""" + + step: Step def __repr__(self) -> str: # noqa D105 return f"{self.step.title}: {self.status.value}" @@ -75,11 +94,11 @@ def __repr__(self) -> str: # noqa D105 def __str__(self) -> str: # noqa D105 return f"{self.step.title}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" - def __post_init__(self): + def __post_init__(self) -> None: if self.stderr: - super().__setattr__("stderr", self.redact_secrets_from_string(self.stderr)) + object.__setattr__(self, "stderr", self.redact_secrets_from_string(self.stderr)) if self.stdout: - super().__setattr__("stdout", self.redact_secrets_from_string(self.stdout)) + object.__setattr__(self, "stdout", self.redact_secrets_from_string(self.stdout)) def redact_secrets_from_string(self, value: str) -> str: for secret in self.step.context.secrets_to_mask: @@ -87,17 +106,11 @@ def redact_secrets_from_string(self, value: str) -> str: return value -@dataclass(frozen=True) -class CommandResult: +@dataclass(kw_only=True, frozen=True) +class CommandResult(Result): """A dataclass to capture the result of a command.""" - command: click.command - status: StepStatus - created_at: datetime = field(default_factory=datetime.utcnow) - stderr: Optional[str] = None - stdout: Optional[str] = None - exc_info: Optional[Exception] = None - output_artifact: Any = None + command: click.Command | FormatCommand def __repr__(self) -> str: # noqa D105 return f"{self.command.name}: {self.status.value}" @@ -105,9 +118,35 @@ def __repr__(self) -> str: # noqa D105 def __str__(self) -> str: # noqa D105 return f"{self.command.name}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" - @property - def success(self) -> bool: - return self.status is StepStatus.SUCCESS + +@dataclass(kw_only=True, frozen=True) +class PoeTaskResult(Result): + + task_name: str + + def __repr__(self) -> str: # noqa D105 + return f"{self.task_name}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.task_name}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + def log(self, logger: logging.Logger, verbose: bool = False) -> None: + """Log the step result. + + Args: + logger (logging.Logger): The logger to use. + """ + if self.status is StepStatus.FAILURE: + logger.exception(self.exc_info) + else: + logger.info(f"{self.status.get_emoji()} - Poe {self.task_name} - {self.status.value}") + if verbose: + if self.stdout: + for line in self.stdout.splitlines(): + logger.info(line) + if self.stderr: + for line in self.stderr.splitlines(): + logger.error(line) class StepStatus(Enum): @@ -135,6 +174,14 @@ def get_emoji(self) -> str: if self is StepStatus.SKIPPED: return "🟡" + def get_github_state(self) -> str: + """Match state used in the GitHub commit checks to the step status.""" + if self in [StepStatus.SUCCESS, StepStatus.SKIPPED]: + return "success" + if self is StepStatus.FAILURE: + return "failure" + raise NotImplementedError(f"Unknown state for {self}") + def __str__(self) -> str: # noqa D105 return self.value @@ -142,23 +189,63 @@ def __str__(self) -> str: # noqa D105 class Step(ABC): """An abstract class to declare and run pipeline step.""" - title: ClassVar[str] max_retries: ClassVar[int] = 0 max_dagger_error_retries: ClassVar[int] = 3 should_log: ClassVar[bool] = True success_exit_code: ClassVar[int] = 0 - skipped_exit_code: ClassVar[int] = None + skipped_exit_code: ClassVar[Optional[int]] = None # The max duration of a step run. If the step run for more than this duration it will be considered as timed out. # The default of 5 hours is arbitrary and can be changed if needed. max_duration: ClassVar[timedelta] = timedelta(hours=5) - retry_delay = timedelta(seconds=10) + accept_extra_params: bool = False def __init__(self, context: PipelineContext) -> None: # noqa D107 self.context = context self.retry_count = 0 - self.started_at = None - self.stopped_at = None + self.started_at: Optional[datetime] = None + self.stopped_at: Optional[datetime] = None + self._extra_params: STEP_PARAMS = {} + + @property + def extra_params(self) -> STEP_PARAMS: + return self._extra_params + + @extra_params.setter + def extra_params(self, value: STEP_PARAMS) -> None: + if value and not self.accept_extra_params: + raise ValueError(f"{self.__class__.__name__} does not accept extra params.") + self._extra_params = value + self.logger.info(f"Will run with the following parameters: {self.params}") + + @property + def default_params(self) -> STEP_PARAMS: + return {} + + @property + def params(self) -> STEP_PARAMS: + return self.default_params | self.extra_params + + @property + def params_as_cli_options(self) -> List[str]: + """Return the step params as a list of CLI options. + + Returns: + List[str]: The step params as a list of CLI options. + """ + cli_options: List[str] = [] + for name, values in self.params.items(): + if not values: + # If no values are available, we assume it is a flag + cli_options.append(name) + else: + cli_options.extend(f"{name}={value}" for value in values) + return cli_options + + @property + def title(self) -> str: + """The title of the step.""" + raise NotImplementedError("Steps must define a 'title' attribute.") @property def run_duration(self) -> timedelta: @@ -177,19 +264,20 @@ def logger(self) -> logging.Logger: return disabled_logger @property - def dagger_client(self) -> Container: + def dagger_client(self) -> Client: return self.context.dagger_client.pipeline(self.title) async def log_progress(self, completion_event: anyio.Event) -> None: """Log the step progress every 30 seconds until the step is done.""" while not completion_event.is_set(): + assert self.started_at is not None, "The step must be started before logging its progress." duration = datetime.utcnow() - self.started_at elapsed_seconds = duration.total_seconds() if elapsed_seconds > 30 and round(elapsed_seconds) % 30 == 0: self.logger.info(f"⏳ Still running... (duration: {format_duration(duration)})") await anyio.sleep(1) - async def run_with_completion(self, completion_event: anyio.Event, *args, **kwargs) -> StepResult: + async def run_with_completion(self, completion_event: anyio.Event, *args: Any, **kwargs: Any) -> StepResult: """Run the step with a timeout and set the completion event when the step is done.""" try: with anyio.fail_after(self.max_duration.total_seconds()): @@ -203,7 +291,7 @@ async def run_with_completion(self, completion_event: anyio.Event, *args, **kwar return self._get_timed_out_step_result() @sentry_utils.with_step_context - async def run(self, *args, **kwargs) -> StepResult: + async def run(self, *args: Any, **kwargs: Any) -> StepResult: """Public method to run the step. It output a step result. If an unexpected dagger error happens it outputs a failed step result with the exception payload. @@ -221,7 +309,7 @@ async def run(self, *args, **kwargs) -> StepResult: step_result = soon_result.value except DaggerError as e: self.logger.error("Step failed with an unexpected dagger error", exc_info=e) - step_result = StepResult(self, StepStatus.FAILURE, stderr=str(e), exc_info=e) + step_result = StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e), exc_info=e) self.stopped_at = datetime.utcnow() self.log_step_result(step_result) @@ -237,7 +325,7 @@ def should_retry(self, step_result: StepResult) -> bool: max_retries = self.max_dagger_error_retries if step_result.exc_info else self.max_retries return self.retry_count < max_retries and max_retries > 0 - async def retry(self, step_result, *args, **kwargs) -> StepResult: + async def retry(self, step_result: StepResult, *args: Any, **kwargs: Any) -> StepResult: self.retry_count += 1 self.logger.warn( f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." @@ -260,7 +348,7 @@ def log_step_result(self, result: StepResult) -> None: self.logger.info(f"{result.status.get_emoji()} was successful (duration: {duration})") @abstractmethod - async def _run(self, *args, **kwargs) -> StepResult: + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: """Implement the execution of the step and return a step result. Returns: @@ -268,7 +356,7 @@ async def _run(self, *args, **kwargs) -> StepResult: """ raise NotImplementedError("Steps must define a '_run' attribute.") - def skip(self, reason: str = None) -> StepResult: + def skip(self, reason: Optional[str] = None) -> StepResult: """Declare a step as skipped. Args: @@ -277,7 +365,7 @@ def skip(self, reason: str = None) -> StepResult: Returns: StepResult: A skipped step result. """ - return StepResult(self, StepStatus.SKIPPED, stdout=reason) + return StepResult(step=self, status=StepStatus.SKIPPED, stdout=reason) def get_step_status_from_exit_code( self, @@ -301,7 +389,7 @@ def get_step_status_from_exit_code( else: return StepStatus.FAILURE - async def get_step_result(self, container: Container) -> StepResult: + async def get_step_result(self, container: Container, *args: Any, **kwargs: Any) -> StepResult: """Concurrent retrieval of exit code, stdout and stdout of a container. Create a StepResult object from these objects. @@ -314,16 +402,16 @@ async def get_step_result(self, container: Container) -> StepResult: """ exit_code, stdout, stderr = await get_exec_result(container) return StepResult( - self, - self.get_step_status_from_exit_code(exit_code), + step=self, + status=self.get_step_status_from_exit_code(exit_code), stderr=stderr, stdout=stdout, - output_artifact=container, + output=container, ) def _get_timed_out_step_result(self) -> StepResult: return StepResult( - self, - StepStatus.FAILURE, + step=self, + status=StepStatus.FAILURE, stdout=f"Timed out after the max duration of {format_duration(self.max_duration)}. Please checkout the Dagger logs to see what happened.", ) diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index a0501242b292..04bcf0652f85 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,8 +1,8 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-connectors-base-images" -version = "0.1.2" +version = "1.0.1" description = "This package is used to generate and publish the base images for Airbyte Connectors." optional = false python-versions = "^3.10" @@ -11,7 +11,7 @@ develop = true [package.dependencies] connector-ops = {path = "../connector_ops", develop = true} -dagger-io = "0.6.4" +dagger-io = "==0.9.6" gitpython = "^3.1.35" inquirer = "^3.1.3" jinja2 = "^3.1.2" @@ -24,17 +24,17 @@ url = "../base_images" [[package]] name = "airbyte-protocol-models" -version = "1.0.1" +version = "0.6.0" description = "Declares the Airbyte Protocol." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-1.0.1-py3-none-any.whl", hash = "sha256:2c214fb8cb42b74aa6408beeea2cd52f094bc8a3ba0e78af20bb358e5404f4a8"}, - {file = "airbyte_protocol_models-1.0.1.tar.gz", hash = "sha256:caa860d15c9c9073df4b221f58280b9855d36de07519e010d1e610546458d0a7"}, + {file = "airbyte_protocol_models-0.6.0-py3-none-any.whl", hash = "sha256:dda91403c9731ecbadffaf05dbe8d24f0d318a189d26fcb727627291837a085c"}, + {file = "airbyte_protocol_models-0.6.0.tar.gz", hash = "sha256:84a0bb0fbedc777f8066295960461ab4a8ab6af63985c21c39bb589569786bc2"}, ] [package.dependencies] -pydantic = ">=1.9.2,<1.10.0" +pydantic = ">=1.9.2,<2.0.0" [[package]] name = "altgraph" @@ -81,16 +81,17 @@ trio = ["trio (<0.22)"] [[package]] name = "asyncclick" -version = "8.1.3.4" +version = "8.1.7.1" description = "Composable command line interface toolkit, async version" optional = false python-versions = ">=3.7" files = [ - {file = "asyncclick-8.1.3.4-py3-none-any.whl", hash = "sha256:f8db604e37dabd43922d58f857817b1dfd8f88695b75c4cc1afe7ff1cc238a7b"}, - {file = "asyncclick-8.1.3.4.tar.gz", hash = "sha256:81d98cbf6c8813f9cd5599f586d56cfc532e9e6441391974d10827abb90fe833"}, + {file = "asyncclick-8.1.7.1-py3-none-any.whl", hash = "sha256:e0fea5f0223ac45cfc26153cc80a58cc65fc077ac8de79be49248c918e8c3422"}, + {file = "asyncclick-8.1.7.1.tar.gz", hash = "sha256:a47b61258a689212cf9463fbf3b4cc52d05bfd03185f6ead2315fc03fd17ef75"}, ] [package.dependencies] +anyio = "*" colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] @@ -119,21 +120,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -148,20 +150,20 @@ files = [ [[package]] name = "beartype" -version = "0.16.4" +version = "0.17.0" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, - {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, + {file = "beartype-0.17.0-py3-none-any.whl", hash = "sha256:fa84b77a8d037f2a39c4aa2f3dc71854afc7d79312e55a66b338da68fdd48c60"}, + {file = "beartype-0.17.0.tar.gz", hash = "sha256:3226fbba8c53b4e698acdb47dcaf3c0640151c4d405618c281e6631f4112947d"}, ] [package.extras] all = ["typing-extensions (>=3.10.0.0)"] -dev = ["autoapi (>=0.9.0)", "coverage (>=5.5)", "mypy (>=0.800)", "numpy", "pandera", "pydata-sphinx-theme (<=0.7.2)", "pytest (>=4.0.0)", "sphinx", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)", "tox (>=3.20.1)", "typing-extensions (>=3.10.0.0)"] +dev = ["autoapi (>=0.9.0)", "coverage (>=5.5)", "equinox", "mypy (>=0.800)", "numpy", "pandera", "pydata-sphinx-theme (<=0.7.2)", "pytest (>=4.0.0)", "sphinx", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)", "torch", "tox (>=3.20.1)", "typing-extensions (>=3.10.0.0)"] doc-rtd = ["autoapi (>=0.9.0)", "pydata-sphinx-theme (<=0.7.2)", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)"] -test-tox = ["mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "typing-extensions (>=3.10.0.0)"] +test-tox = ["equinox", "mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "torch", "typing-extensions (>=3.10.0.0)"] test-tox-coverage = ["coverage (>=5.5)"] [[package]] @@ -193,28 +195,28 @@ files = [ [[package]] name = "cattrs" -version = "23.1.2" +version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, - {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, ] [package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] -bson = ["pymongo (>=4.2.0,<5.0.0)"] -cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] -msgpack = ["msgpack (>=1.0.2,<2.0.0)"] -orjson = ["orjson (>=3.5.2,<4.0.0)"] -pyyaml = ["PyYAML (>=6.0,<7.0)"] -tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] -ujson = ["ujson (>=5.4.0,<6.0.0)"] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" @@ -454,7 +456,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.3.2" +version = "0.3.3" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" @@ -481,63 +483,63 @@ url = "../connector_ops" [[package]] name = "coverage" -version = "7.3.2" +version = "7.4.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf"}, - {file = "coverage-7.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9"}, - {file = "coverage-7.3.2-cp310-cp310-win32.whl", hash = "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f"}, - {file = "coverage-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312"}, - {file = "coverage-7.3.2-cp311-cp311-win32.whl", hash = "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640"}, - {file = "coverage-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a"}, - {file = "coverage-7.3.2-cp312-cp312-win32.whl", hash = "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb"}, - {file = "coverage-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76"}, - {file = "coverage-7.3.2-cp38-cp38-win32.whl", hash = "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92"}, - {file = "coverage-7.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083"}, - {file = "coverage-7.3.2-cp39-cp39-win32.whl", hash = "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce"}, - {file = "coverage-7.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f"}, - {file = "coverage-7.3.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637"}, - {file = "coverage-7.3.2.tar.gz", hash = "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, + {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, + {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, + {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, + {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, + {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, + {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, + {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, + {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, + {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, + {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, + {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, + {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, ] [package.dependencies] @@ -587,13 +589,13 @@ test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)" [[package]] name = "dagger-io" -version = "0.6.4" +version = "0.9.6" description = "A client package for running Dagger pipelines in Python." optional = false python-versions = ">=3.10" files = [ - {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, - {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, ] [package.dependencies] @@ -604,11 +606,8 @@ gql = ">=3.4.0" graphql-core = ">=3.2.3" httpx = ">=0.23.1" platformdirs = ">=2.6.2" -typing-extensions = ">=4.4.0" - -[package.extras] -cli = ["typer[all] (>=0.6.1)"] -server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" [[package]] name = "deprecated" @@ -648,15 +647,41 @@ websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.3)"] +[[package]] +name = "dpath" +version = "2.1.6" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.1.6-py3-none-any.whl", hash = "sha256:31407395b177ab63ef72e2f6ae268c15e938f2990a8ecf6510f5686c02b6db73"}, + {file = "dpath-2.1.6.tar.gz", hash = "sha256:f1e07c72e8605c6a9e80b64bc8f42714de08a789c7de417e49c3f87a19692e47"}, +] + +[[package]] +name = "editor" +version = "1.6.6" +description = "🖋 Open the default text editor 🖋" +optional = false +python-versions = ">=3.8" +files = [ + {file = "editor-1.6.6-py3-none-any.whl", hash = "sha256:e818e6913f26c2a81eadef503a2741d7cca7f235d20e217274a009ecd5a74abf"}, + {file = "editor-1.6.6.tar.gz", hash = "sha256:bb6989e872638cd119db9a4fce284cd8e13c553886a1c044c6b8d8a160c871f8"}, +] + +[package.dependencies] +runs = "*" +xmod = "*" + [[package]] name = "exceptiongroup" -version = "1.1.3" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -664,13 +689,13 @@ test = ["pytest (>=6)"] [[package]] name = "freezegun" -version = "1.2.2" +version = "1.4.0" description = "Let your Python tests travel through time" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, - {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, ] [package.dependencies] @@ -692,30 +717,30 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.40" +version = "3.1.41" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.40-py3-none-any.whl", hash = "sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a"}, - {file = "GitPython-3.1.40.tar.gz", hash = "sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4"}, + {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, + {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-instafail", "pytest-subtests", "pytest-sugar"] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] [[package]] name = "google-api-core" -version = "2.14.0" +version = "2.16.2" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.14.0.tar.gz", hash = "sha256:5368a4502b793d9bbf812a5912e13e4e69f9bd87f6efb508460c43f5bbd1ce41"}, - {file = "google_api_core-2.14.0-py3-none-any.whl", hash = "sha256:de2fb50ed34d47ddbb2bd2dcf680ee8fead46279f4ed6b16de362aca23a18952"}, + {file = "google-api-core-2.16.2.tar.gz", hash = "sha256:032d37b45d1d6bdaf68fb11ff621e2593263a239fa9246e2e94325f9c47876d2"}, + {file = "google_api_core-2.16.2-py3-none-any.whl", hash = "sha256:449ca0e3f14c179b4165b664256066c7861610f70b6ffe54bb01a04e9b466929"}, ] [package.dependencies] @@ -731,13 +756,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.23.4" +version = "2.27.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.23.4.tar.gz", hash = "sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3"}, - {file = "google_auth-2.23.4-py2.py3-none-any.whl", hash = "sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2"}, + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, ] [package.dependencies] @@ -754,13 +779,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-core" -version = "2.3.3" +version = "2.4.1" description = "Google Cloud API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, - {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, ] [package.dependencies] @@ -768,17 +793,17 @@ google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)"] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.13.0" +version = "2.14.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.13.0.tar.gz", hash = "sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7"}, - {file = "google_cloud_storage-2.13.0-py2.py3-none-any.whl", hash = "sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d"}, + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, ] [package.dependencies] @@ -874,13 +899,13 @@ testing = ["pytest"] [[package]] name = "google-resumable-media" -version = "2.6.0" +version = "2.7.0" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">= 3.7" files = [ - {file = "google-resumable-media-2.6.0.tar.gz", hash = "sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7"}, - {file = "google_resumable_media-2.6.0-py2.py3-none-any.whl", hash = "sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b"}, + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, ] [package.dependencies] @@ -892,13 +917,13 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.61.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.61.0.tar.gz", hash = "sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b"}, - {file = "googleapis_common_protos-1.61.0-py2.py3-none-any.whl", hash = "sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -909,29 +934,31 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" @@ -978,19 +1005,19 @@ trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.25.1" +version = "0.26.0" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.25.1-py3-none-any.whl", hash = "sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a"}, - {file = "httpx-0.25.1.tar.gz", hash = "sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [package.dependencies] anyio = "*" certifi = "*" -httpcore = "*" +httpcore = "==1.*" idna = "*" sniffio = "*" @@ -1002,13 +1029,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -1024,29 +1051,29 @@ files = [ [[package]] name = "inquirer" -version = "3.1.3" +version = "3.2.3" description = "Collection of common interactive command line user interfaces, based on Inquirer.js" optional = false -python-versions = ">=3.8" +python-versions = ">=3.8.1" files = [ - {file = "inquirer-3.1.3-py3-none-any.whl", hash = "sha256:a7441fd74d06fcac4385218a1f5e8703f7a113f7944e01af47b8c58e84f95ce5"}, - {file = "inquirer-3.1.3.tar.gz", hash = "sha256:aac309406f5b49d4b8ab7c6872117f43bf082a552dc256aa16bc95e16bb58bec"}, + {file = "inquirer-3.2.3-py3-none-any.whl", hash = "sha256:68fa2cfaa652212f035f73794aa1db2e6c0a9c8cef81ab6825b45120fa8ea345"}, + {file = "inquirer-3.2.3.tar.gz", hash = "sha256:0cba57d901b206dd597d8809b58c378c47fbc804a1fc9b33e2780ca2f9b43ac7"}, ] [package.dependencies] blessed = ">=1.19.0" -python-editor = ">=1.0.4" +editor = ">=1.6.0" readchar = ">=3.0.6" [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -1057,13 +1084,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jinxed" -version = "1.2.0" +version = "1.2.1" description = "Jinxed Terminal Library" optional = false python-versions = "*" files = [ - {file = "jinxed-1.2.0-py2.py3-none-any.whl", hash = "sha256:cfc2b2e4e3b4326954d546ba6d6b9a7a796ddcb0aef8d03161d005177eb0d48b"}, - {file = "jinxed-1.2.0.tar.gz", hash = "sha256:032acda92d5c57cd216033cbbd53de731e6ed50deb63eb4781336ca55f72cda5"}, + {file = "jinxed-1.2.1-py2.py3-none-any.whl", hash = "sha256:37422659c4925969c66148c5e64979f553386a4226b9484d910d3094ced37d30"}, + {file = "jinxed-1.2.1.tar.gz", hash = "sha256:30c3f861b73279fea1ed928cfd4dfb1f273e16cd62c8a32acfac362da0f78f3f"}, ] [package.dependencies] @@ -1109,61 +1136,71 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "markupsafe" -version = "2.1.3" +version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] [[package]] @@ -1177,17 +1214,6 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] -[[package]] -name = "monotonic" -version = "1.6" -description = "An implementation of time.monotonic() for Python 2 & < 3.3" -optional = false -python-versions = "*" -files = [ - {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, - {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, -] - [[package]] name = "more-itertools" version = "8.14.0" @@ -1201,130 +1227,204 @@ files = [ [[package]] name = "multidict" -version = "6.0.4" +version = "6.0.5" description = "multidict implementation" optional = false python-versions = ">=3.7" files = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "mypy" +version = "1.8.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] name = "numpy" -version = "1.26.2" +version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.26.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f"}, - {file = "numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523"}, - {file = "numpy-1.26.2-cp310-cp310-win32.whl", hash = "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9"}, - {file = "numpy-1.26.2-cp310-cp310-win_amd64.whl", hash = "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8"}, - {file = "numpy-1.26.2-cp311-cp311-win32.whl", hash = "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186"}, - {file = "numpy-1.26.2-cp311-cp311-win_amd64.whl", hash = "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec"}, - {file = "numpy-1.26.2-cp312-cp312-win32.whl", hash = "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167"}, - {file = "numpy-1.26.2-cp312-cp312-win_amd64.whl", hash = "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36"}, - {file = "numpy-1.26.2-cp39-cp39-win32.whl", hash = "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80"}, - {file = "numpy-1.26.2-cp39-cp39-win_amd64.whl", hash = "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841"}, - {file = "numpy-1.26.2.tar.gz", hash = "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1340,67 +1440,71 @@ files = [ [[package]] name = "pandas" -version = "2.1.3" +version = "2.2.0" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acf08a73b5022b479c1be155d4988b72f3020f308f7a87c527702c5f8966d34f"}, - {file = "pandas-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cc4469ff0cf9aa3a005870cb49ab8969942b7156e0a46cc3f5abd6b11051dfb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35172bff95f598cc5866c047f43c7f4df2c893acd8e10e6653a4b792ed7f19bb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dfe0e65a2f3988e940224e2a70932edc964df79f3356e5f2997c7d63e758b4"}, - {file = "pandas-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0296a66200dee556850d99b24c54c7dfa53a3264b1ca6f440e42bad424caea03"}, - {file = "pandas-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:465571472267a2d6e00657900afadbe6097c8e1dc43746917db4dfc862e8863e"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04d4c58e1f112a74689da707be31cf689db086949c71828ef5da86727cfe3f82"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fa2ad4ff196768ae63a33f8062e6838efed3a319cf938fdf8b95e956c813042"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4441ac94a2a2613e3982e502ccec3bdedefe871e8cea54b8775992485c5660ef"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ded6ff28abbf0ea7689f251754d3789e1edb0c4d0d91028f0b980598418a58"}, - {file = "pandas-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca5680368a5139d4920ae3dc993eb5106d49f814ff24018b64d8850a52c6ed2"}, - {file = "pandas-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:de21e12bf1511190fc1e9ebc067f14ca09fccfb189a813b38d63211d54832f5f"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a5d53c725832e5f1645e7674989f4c106e4b7249c1d57549023ed5462d73b140"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cf4cf26042476e39394f1f86868d25b265ff787c9b2f0d367280f11afbdee6d"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72c84ec1b1d8e5efcbff5312abe92bfb9d5b558f11e0cf077f5496c4f4a3c99e"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f539e113739a3e0cc15176bf1231a553db0239bfa47a2c870283fd93ba4f683"}, - {file = "pandas-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc77309da3b55732059e484a1efc0897f6149183c522390772d3561f9bf96c00"}, - {file = "pandas-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:08637041279b8981a062899da0ef47828df52a1838204d2b3761fbd3e9fcb549"}, - {file = "pandas-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b99c4e51ef2ed98f69099c72c75ec904dd610eb41a32847c4fcbc1a975f2d2b8"}, - {file = "pandas-2.1.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7ea8ae8004de0381a2376662c0505bb0a4f679f4c61fbfd122aa3d1b0e5f09d"}, - {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd76d67ca2d48f56e2db45833cf9d58f548f97f61eecd3fdc74268417632b8a"}, - {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1329dbe93a880a3d7893149979caa82d6ba64a25e471682637f846d9dbc10dd2"}, - {file = "pandas-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:321ecdb117bf0f16c339cc6d5c9a06063854f12d4d9bc422a84bb2ed3207380a"}, - {file = "pandas-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:11a771450f36cebf2a4c9dbd3a19dfa8c46c4b905a3ea09dc8e556626060fe71"}, - {file = "pandas-2.1.3.tar.gz", hash = "sha256:22929f84bca106921917eb73c1521317ddd0a4c71b395bcf767a106e3494209f"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, ] [package.dependencies] numpy = {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""} python-dateutil = ">=2.8.2" pytz = ">=2020.1" -tzdata = ">=2022.1" +tzdata = ">=2022.7" [package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] -aws = ["s3fs (>=2022.05.0)"] -clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] -compression = ["zstandard (>=0.17.0)"] -computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2022.05.0)"] -gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] -hdf5 = ["tables (>=3.7.0)"] -html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] -mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] -spss = ["pyreadstat (>=1.1.5)"] -sql-other = ["SQLAlchemy (>=1.4.36)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.8.0)"] +xml = ["lxml (>=4.9.2)"] [[package]] name = "pastel" @@ -1426,28 +1530,28 @@ files = [ [[package]] name = "platformdirs" -version = "4.0.0" +version = "4.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, - {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] [[package]] name = "pluggy" -version = "1.3.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] @@ -1456,13 +1560,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "poethepoet" -version = "0.24.3" +version = "0.24.4" description = "A task runner that works well with poetry." optional = false python-versions = ">=3.8" files = [ - {file = "poethepoet-0.24.3-py3-none-any.whl", hash = "sha256:8817c6d3d8492776bbb17eb29b7b815b2905aefaa0ad887137e69e53349e2235"}, - {file = "poethepoet-0.24.3.tar.gz", hash = "sha256:73f1060200d1c8f21e303d06a879c5c0e17b96ab16740da70aee2dcc3e4350e4"}, + {file = "poethepoet-0.24.4-py3-none-any.whl", hash = "sha256:fb4ea35d7f40fe2081ea917d2e4102e2310fda2cde78974050ca83896e229075"}, + {file = "poethepoet-0.24.4.tar.gz", hash = "sha256:ff4220843a87c888cbcb5312c8905214701d0af60ac7271795baa8369b428fef"}, ] [package.dependencies] @@ -1474,22 +1578,22 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] [[package]] name = "protobuf" -version = "4.25.0" +version = "4.25.2" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.0-cp310-abi3-win32.whl", hash = "sha256:5c1203ac9f50e4853b0a0bfffd32c67118ef552a33942982eeab543f5c634395"}, - {file = "protobuf-4.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:c40ff8f00aa737938c5378d461637d15c442a12275a81019cc2fef06d81c9419"}, - {file = "protobuf-4.25.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:cf21faba64cd2c9a3ed92b7a67f226296b10159dbb8fbc5e854fc90657d908e4"}, - {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:32ac2100b0e23412413d948c03060184d34a7c50b3e5d7524ee96ac2b10acf51"}, - {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:683dc44c61f2620b32ce4927de2108f3ebe8ccf2fd716e1e684e5a50da154054"}, - {file = "protobuf-4.25.0-cp38-cp38-win32.whl", hash = "sha256:1a3ba712877e6d37013cdc3476040ea1e313a6c2e1580836a94f76b3c176d575"}, - {file = "protobuf-4.25.0-cp38-cp38-win_amd64.whl", hash = "sha256:b2cf8b5d381f9378afe84618288b239e75665fe58d0f3fd5db400959274296e9"}, - {file = "protobuf-4.25.0-cp39-cp39-win32.whl", hash = "sha256:63714e79b761a37048c9701a37438aa29945cd2417a97076048232c1df07b701"}, - {file = "protobuf-4.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:d94a33db8b7ddbd0af7c467475fb9fde0c705fb315a8433c0e2020942b863a1f"}, - {file = "protobuf-4.25.0-py3-none-any.whl", hash = "sha256:1a53d6f64b00eecf53b65ff4a8c23dc95df1fa1e97bb06b8122e5a64f49fc90a"}, - {file = "protobuf-4.25.0.tar.gz", hash = "sha256:68f7caf0d4f012fd194a301420cf6aa258366144d814f358c5b32558228afa7c"}, + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, ] [[package]] @@ -1505,13 +1609,13 @@ files = [ [[package]] name = "pyasn1" -version = "0.5.0" +version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, - {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, ] [[package]] @@ -1541,50 +1645,51 @@ files = [ [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.14" description = "Data validation and settings management using python type hints" optional = false -python-versions = ">=3.6.1" -files = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, ] [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -1592,54 +1697,53 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pydash" -version = "7.0.6" +version = "7.0.7" description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"}, - {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"}, + {file = "pydash-7.0.7-py3-none-any.whl", hash = "sha256:c3c5b54eec0a562e0080d6f82a14ad4d5090229847b7e554235b5c1558c745e1"}, + {file = "pydash-7.0.7.tar.gz", hash = "sha256:cc935d5ac72dd41fb4515bdf982e7c864c8b5eeea16caffbab1936b849aaa49a"}, ] [package.dependencies] typing-extensions = ">=3.10,<4.6.0 || >4.6.0" [package.extras] -dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] +dev = ["black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] [[package]] name = "pygit2" -version = "1.13.2" +version = "1.14.0" description = "Python bindings for libgit2." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pygit2-1.13.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:781aefab7efc464852e688965bf3b4acc7af951cebea174d69f86b213aa5d5fb"}, - {file = "pygit2-1.13.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3038b5ecef43e2c853e7cf405676241e0395bb37b37ae477ef3b73a91f12378"}, - {file = "pygit2-1.13.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c00927a2626325b64ebc9f860f024a3ae0b4c036663f6ada8d5de0e2393560ca"}, - {file = "pygit2-1.13.2-cp310-cp310-win32.whl", hash = "sha256:6988fc6cf99a3dbc03bd64060888c3b194ee27c810cb61624519ee3813f2da3d"}, - {file = "pygit2-1.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:aec3df351b722ec7cdf7a7e642e421e3a15f3f2e3a51e57380d62d4992acf36d"}, - {file = "pygit2-1.13.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0eb53cc5212fad90e36693c0cd2ffd0d470efaea2506ce1c0d04f8d7fcf6767c"}, - {file = "pygit2-1.13.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32803ec881cd8f7dba91e03927e1fb13857e795bbe85cd3ec156b4798b933294"}, - {file = "pygit2-1.13.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba7297204e72c5cfdcd7a0c0d318af9d654a1d79b1cfe6cc8330570c749bec1f"}, - {file = "pygit2-1.13.2-cp311-cp311-win32.whl", hash = "sha256:2291707e648f5bba5b5c5e7ed652bc4563bd520718eb31e19525ccaceba5503c"}, - {file = "pygit2-1.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:96e534e92e485c4c1d4c3e151ce960655fed38ab9a1d65e2b16650cf24b3e088"}, - {file = "pygit2-1.13.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75f3b6d754d91dbe47b27b53d5a4440d861906b2f476284e6fb7c46cafe244d7"}, - {file = "pygit2-1.13.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30e145730dc65a9b902a889efdca0126d6b274c0b14427ebb085e090b50f6470"}, - {file = "pygit2-1.13.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2311ca16e1d0b77bc3694407c367391c7f2f78e2f725dc858721a5d4e3635fdd"}, - {file = "pygit2-1.13.2-cp312-cp312-win32.whl", hash = "sha256:a027e06c44f987a217c6197970bb29de9fbc78524c81b1f37888711978a64ce2"}, - {file = "pygit2-1.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:9844fb5a38119a34b31012dddc9b439f81bb0411cbf4a4f8e92a044f6f3e7462"}, - {file = "pygit2-1.13.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2f3a5362c702a42e28c3bc84ff324b57676c8bfdbfab445c96f5e776873630a6"}, - {file = "pygit2-1.13.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7d5d1c3508b66e5e13883ff472b616d2d60feb7a4afea52d3b501e9f5ee5d08"}, - {file = "pygit2-1.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2507d99584c7e3976342566adf6bc48aca825c170b86f999fe7bd32f8aa1858e"}, - {file = "pygit2-1.13.2-cp38-cp38-win32.whl", hash = "sha256:acda61b726c33ada3639cac5ddc5898678f7bb7b8415e84e3ff07a2af94b1ac3"}, - {file = "pygit2-1.13.2-cp38-cp38-win_amd64.whl", hash = "sha256:4a86c4cae2e717acdd9d7ff00d196395fafe1abfc5efab5ada63650b49d5d47f"}, - {file = "pygit2-1.13.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ae9a77be5c5df5f4c9e586fbd53f1095bced2bba86ec669ead92c4c1e02f8373"}, - {file = "pygit2-1.13.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25ef1dcfb59e73f6a59491343393b6e843739cbc92e8088a551c73cd367a54d0"}, - {file = "pygit2-1.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b88d21ed961fe422275c9c20d2613e6ecff2fa8127ac7620a29aba1d001fc41"}, - {file = "pygit2-1.13.2-cp39-cp39-win32.whl", hash = "sha256:14b458af1e8c6b634d55110edeab055e3bd9075543792cb75d2fdb8b434c202a"}, - {file = "pygit2-1.13.2-cp39-cp39-win_amd64.whl", hash = "sha256:565b311c783a07768b91382620ad2b474fe40778411cb18c576f667be43d1299"}, - {file = "pygit2-1.13.2.tar.gz", hash = "sha256:75c7eb86b47c70f6f1434bcf3b5eb41f4e8006a15cee6bef606651b97d23788c"}, + {file = "pygit2-1.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ab5a983cb116d617c136cdc23832e16aed17f5fdd3b7bb46d85c0aabde0162ee"}, + {file = "pygit2-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e352b77c2e6f8a1900b406bc10a9471718782775a6029d847c71e5363c3166f9"}, + {file = "pygit2-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12a5f456ab9ac2e7718c95c8ac2bfa1fd23908545deb7cb7693e035c2d0f037a"}, + {file = "pygit2-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bb10402c983d8513c3bceb6a3f6f52ec19c69b0244801cebe95aab6dbf19f679"}, + {file = "pygit2-1.14.0-cp310-cp310-win32.whl", hash = "sha256:0d7526a7ad2bb91b36ba43c87452182052f58cb068311cf8173ed5391ca7788e"}, + {file = "pygit2-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:80d0baca5ab9a06ca6a709716737ed6993e10349db7a98f1f3966278d39098fd"}, + {file = "pygit2-1.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86f5295e7996927238dfebdb3c8d81dae83332bc8ced61971806a606261d60ff"}, + {file = "pygit2-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84dd4b36e38c9736736ba57e7257b6efe604932232c98503a64c94283dada7de"}, + {file = "pygit2-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adf7fd8af9bc3b6e11e4920abb0121cdad6f8299ed1d7643e756ab49dbb4e34"}, + {file = "pygit2-1.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a98c3db4f06bae8266263bdc7b7447801debc30b6223f0826e07709abe9c0929"}, + {file = "pygit2-1.14.0-cp311-cp311-win32.whl", hash = "sha256:4c74aba5b40d6dac2f04bf4f3ca529304bdbf77888de0e87c706d243c9fa0693"}, + {file = "pygit2-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:613bc82b0a17ccd5334b8f5d3b963698b45e228910bcea27fa52f84c60f50b1a"}, + {file = "pygit2-1.14.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0384fb21af58149d59dc37f73f9daea7e6cfec2de7d067be40cc08049b4a62b"}, + {file = "pygit2-1.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb53c367f66cdd8d41552ed2a01a15a0499d8373dcca37360f3abfb7bf947f71"}, + {file = "pygit2-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:807cf57e02947ad448ae91226d193ebe0999540a56f5a95183a502e28c50b7ff"}, + {file = "pygit2-1.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a83fe40e2cdac3abf926b633e07be434ddae353085720c1a6e3afb2a4b72f9c1"}, + {file = "pygit2-1.14.0-cp312-cp312-win32.whl", hash = "sha256:ffe8b5b7fb482c3f8625384eb60e83390e1c2c1b74e66aff2f812e74c9754c5d"}, + {file = "pygit2-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:47d8223440096e59bd6367c341692cd0191e98601665dd4986ba2e00bc5ef769"}, + {file = "pygit2-1.14.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ed9e67e58f11f285e2fa2077c6f45852763826f8b8a2a777937f1fd2313eed5d"}, + {file = "pygit2-1.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec66cb115afd5552d50ba96a29e60da4556cd060396a1b38e97aefc047bd124"}, + {file = "pygit2-1.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ea6fd663ebe59e6e872a25a0f1af2d83c7d75147461a352a22bca4df70c8d0"}, + {file = "pygit2-1.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:65cc2e696f5d6add54d34dbf7336a420f7b1df31c525e3ed5c8a123f4f1d67de"}, + {file = "pygit2-1.14.0-cp39-cp39-win32.whl", hash = "sha256:34a05d47b05e1fe2cc44164d778035253868b179819b300a4d1c6cb75ff48847"}, + {file = "pygit2-1.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:0f101c08fe2f81cc05a44f5c95ea5396310df3240e24d9f5dc2cf1871a794fcb"}, + {file = "pygit2-1.14.0.tar.gz", hash = "sha256:f529ed9660edbf9b625ccae7e51098ef73662e61496609009772d4627a826aa8"}, ] [package.dependencies] @@ -1664,37 +1768,38 @@ requests = ">=2.14.0" [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyinstaller" -version = "6.2.0" +version = "6.3.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false python-versions = "<3.13,>=3.8" files = [ - {file = "pyinstaller-6.2.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:a1adbd3cf25dc90926d783eae0f444d65cdfecc7bcdf6da522c3ae3ff47b4c25"}, - {file = "pyinstaller-6.2.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:29d164394f1e949072f78a64c1e040f1c47b7f4aff08514c7666a031c8b44996"}, - {file = "pyinstaller-6.2.0-py3-none-manylinux2014_i686.whl", hash = "sha256:ba602a38d7403de89c38b8956b221ce6de0280730d269bab522492fcad82ee33"}, - {file = "pyinstaller-6.2.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:ebac06d99b80d2035594c3cc2fb5f2612d86289edd0510dbcbeb20a873f51d5a"}, - {file = "pyinstaller-6.2.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:fcfabc0ff1d38a4262c051dea3fdc1f7f106405c1f1b491b4c79cd28df19cab6"}, - {file = "pyinstaller-6.2.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:104430686149b2f1c135b2c17aa2967c85d54ef77dc92feb4e179ec846c0c467"}, - {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:e87fd60292b53bb9965cb5a84122875469a2bd475fd0d0db0052a3f1be351f75"}, - {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:8ec9d6c98972bb922cedb16a6638257aa66e5deadd79e2953f3464696237c413"}, - {file = "pyinstaller-6.2.0-py3-none-win32.whl", hash = "sha256:e5561e9a9b946d835c8dbc11ae4c16cc21e62bc77d10cc043406dc2992dfb4c6"}, - {file = "pyinstaller-6.2.0-py3-none-win_amd64.whl", hash = "sha256:3b586196277c4c54b69880650984c39c28bb6258c2b4b64200032e6ac69d53a0"}, - {file = "pyinstaller-6.2.0-py3-none-win_arm64.whl", hash = "sha256:d0c87b605bf13c3a04dfaa1d2fa7cd36765b8137000eeadccba865e1d6a19bf0"}, - {file = "pyinstaller-6.2.0.tar.gz", hash = "sha256:1ce77043929bf525be38289d78feecde0fcf15506215eda6500176a8715c5047"}, + {file = "pyinstaller-6.3.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:75a6f2a6f835a2e6e0899d10e60c10caf5defd25aced38b1dd48fbbabc89de07"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:de25beb176f73a944758553caacec46cc665bf3910ad8a174706d79cf6e95340"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_i686.whl", hash = "sha256:e436fcc0ea87c3f132baac916d508c24c84a8f6d8a06c3154fbc753f169b76c7"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:b721d793a33b6d9946c7dd95d3ea7589c0424b51cf1b9fe580f03c544f1336b2"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:96c37a1ee5b2fd5bb25c098ef510661d6d17b6515d0b86d8fc93727dd2475ba3"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:abe91106a3bbccc3f3a27af4325676ecdb6f46cb842ac663625002a870fc503b"}, + {file = "pyinstaller-6.3.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:41c937fe8f07ae02009b3b5a96ac3eb0800a4f8a97af142d4100060fe2135bb9"}, + {file = "pyinstaller-6.3.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:886b3b995b674905a20ad5b720b47cc395897d7b391117831027a4c8c5d67a58"}, + {file = "pyinstaller-6.3.0-py3-none-win32.whl", hash = "sha256:0597fb04337695e5cc5250253e0655530bf14f264b7a5b7d219cc65f6889c4bd"}, + {file = "pyinstaller-6.3.0-py3-none-win_amd64.whl", hash = "sha256:156b32ba943e0090bcc68e40ae1cb68fd92b7f1ab6fe0bdf8faf3d3cfc4e12dd"}, + {file = "pyinstaller-6.3.0-py3-none-win_arm64.whl", hash = "sha256:1eadbd1fae84e2e6c678d8b4ed6a232ec5c8fe3a839aea5a3071c4c0282f98cc"}, + {file = "pyinstaller-6.3.0.tar.gz", hash = "sha256:914d4c96cc99472e37ac552fdd82fbbe09e67bb592d0717fcffaa99ea74273df"}, ] [package.dependencies] @@ -1712,15 +1817,19 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.10" +version = "2024.0" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2023.10.tar.gz", hash = "sha256:4b4a998036abb713774cb26534ca06b7e6e09e4c628196017a10deb11a48747f"}, - {file = "pyinstaller_hooks_contrib-2023.10-py2.py3-none-any.whl", hash = "sha256:6dc1786a8f452941245d5bb85893e2a33632ebdcbc4c23eea41f2ee08281b0c0"}, + {file = "pyinstaller-hooks-contrib-2024.0.tar.gz", hash = "sha256:a7118c1a5c9788595e5c43ad058a7a5b7b6d59e1eceb42362f6ec1f0b61986b0"}, + {file = "pyinstaller_hooks_contrib-2024.0-py2.py3-none-any.whl", hash = "sha256:469b5690df53223e2e8abffb2e44d6ee596e7d79d4b1eed9465123b67439875a"}, ] +[package.dependencies] +packaging = ">=22.0" +setuptools = ">=42.0.0" + [[package]] name = "pyjwt" version = "2.8.0" @@ -1840,27 +1949,15 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "python-editor" -version = "1.0.4" -description = "Programmatically open an editor, capture the result." -optional = false -python-versions = "*" -files = [ - {file = "python-editor-1.0.4.tar.gz", hash = "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b"}, - {file = "python_editor-1.0.4-py2-none-any.whl", hash = "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8"}, - {file = "python_editor-1.0.4-py3-none-any.whl", hash = "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d"}, -] - [[package]] name = "pytz" -version = "2023.3.post1" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -1909,6 +2006,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1916,8 +2014,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1934,6 +2039,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1941,6 +2047,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1983,13 +2090,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "13.6.0" +version = "13.7.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245"}, - {file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef"}, + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, ] [package.dependencies] @@ -2013,20 +2120,59 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "ruff" +version = "0.1.15" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5fe8d54df166ecc24106db7dd6a68d44852d14eb0729ea4672bb4d96c320b7df"}, + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f0bfbb53c4b4de117ac4d6ddfd33aa5fc31beeaa21d23c45c6dd249faf9126f"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d432aec35bfc0d800d4f70eba26e23a352386be3a6cf157083d18f6f5881c8"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9405fa9ac0e97f35aaddf185a1be194a589424b8713e3b97b762336ec79ff807"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c66ec24fe36841636e814b8f90f572a8c0cb0e54d8b5c2d0e300d28a0d7bffec"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6f8ad828f01e8dd32cc58bc28375150171d198491fc901f6f98d2a39ba8e3ff5"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86811954eec63e9ea162af0ffa9f8d09088bab51b7438e8b6488b9401863c25e"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd4025ac5e87d9b80e1f300207eb2fd099ff8200fa2320d7dc066a3f4622dc6b"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b17b93c02cdb6aeb696effecea1095ac93f3884a49a554a9afa76bb125c114c1"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ddb87643be40f034e97e97f5bc2ef7ce39de20e34608f3f829db727a93fb82c5"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:abf4822129ed3a5ce54383d5f0e964e7fef74a41e48eb1dfad404151efc130a2"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6c629cf64bacfd136c07c78ac10a54578ec9d1bd2a9d395efbee0935868bf852"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1bab866aafb53da39c2cadfb8e1c4550ac5340bb40300083eb8967ba25481447"}, + {file = "ruff-0.1.15-py3-none-win32.whl", hash = "sha256:2417e1cb6e2068389b07e6fa74c306b2810fe3ee3476d5b8a96616633f40d14f"}, + {file = "ruff-0.1.15-py3-none-win_amd64.whl", hash = "sha256:3837ac73d869efc4182d9036b1405ef4c73d9b1f88da2413875e34e0d6919587"}, + {file = "ruff-0.1.15-py3-none-win_arm64.whl", hash = "sha256:9a933dfb1c14ec7a33cceb1e49ec4a16b51ce3c20fd42663198746efc0427360"}, + {file = "ruff-0.1.15.tar.gz", hash = "sha256:f6dfa8c1b21c913c326919056c390966648b680966febcb796cc9d1aaab8564e"}, +] + +[[package]] +name = "runs" +version = "1.2.2" +description = "🏃 Run a block of text as a subprocess 🏃" +optional = false +python-versions = ">=3.8" +files = [ + {file = "runs-1.2.2-py3-none-any.whl", hash = "sha256:0980dcbc25aba1505f307ac4f0e9e92cbd0be2a15a1e983ee86c24c87b839dfd"}, + {file = "runs-1.2.2.tar.gz", hash = "sha256:9dc1815e2895cfb3a48317b173b9f1eac9ba5549b36a847b5cc60c3bf82ecef1"}, +] + +[package.dependencies] +xmod = "*" + [[package]] name = "segment-analytics-python" -version = "2.2.3" +version = "2.3.1" description = "The hassle-free way to integrate analytics into any python application." optional = false python-versions = ">=3.6.0" files = [ - {file = "segment-analytics-python-2.2.3.tar.gz", hash = "sha256:0df5908e3df74b4482f33392fdd450df4c8351bf54974376fbe6bf33b0700865"}, - {file = "segment_analytics_python-2.2.3-py2.py3-none-any.whl", hash = "sha256:06cc3d8e79103f02c3878ec66cb66152415473d0d2a142b98a0ee18da972e109"}, + {file = "segment_analytics_python-2.3.1-py2.py3-none-any.whl", hash = "sha256:b5d415247f983e8698de7e094f141cf48f9098b49cc95e108c5bf1e08127d636"}, ] [package.dependencies] backoff = ">=2.1,<3.0" -monotonic = ">=1.5,<2.0" +PyJWT = ">=2.8.0,<2.9.0" python-dateutil = ">=2.2,<3.0" requests = ">=2.7,<3.0" @@ -2046,13 +2192,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.35.0" +version = "1.40.2" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.35.0.tar.gz", hash = "sha256:04e392db9a0d59bd49a51b9e3a92410ac5867556820465057c2ef89a38e953e9"}, - {file = "sentry_sdk-1.35.0-py2.py3-none-any.whl", hash = "sha256:a7865952701e46d38b41315c16c075367675c48d049b90a4cc2e41991ebc7efa"}, + {file = "sentry-sdk-1.40.2.tar.gz", hash = "sha256:c98c8e9bb4dc8ff1e67473caf6467acfccf915dadcc26d0efb0d6791a8652610"}, + {file = "sentry_sdk-1.40.2-py2.py3-none-any.whl", hash = "sha256:696ef61a323a207e6a20b018ddc6591adb81c671434c88d1a4f2e95ffa75556c"}, ] [package.dependencies] @@ -2078,7 +2224,7 @@ huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -2091,17 +2237,17 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.2.2" +version = "69.0.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -2171,26 +2317,84 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomli-w" +version = "1.0.0" +description = "A lil' TOML writer" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli_w-1.0.0-py3-none-any.whl", hash = "sha256:9f2a07e8be30a0729e533ec968016807069991ae2fd921a78d42f429ae5f4463"}, + {file = "tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9"}, +] + +[[package]] +name = "types-requests" +version = "2.28.2" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.28.2.tar.gz", hash = "sha256:398f88cd9302c796cb63d1021af2a1fb7ae507741a3d508edf8e0746d8c16a04"}, + {file = "types_requests-2.28.2-py3-none-any.whl", hash = "sha256:c164696bfdce0123901165c5f097a6cc4f6326268c65815d4b6a57eacfec5e81"}, +] + +[package.dependencies] +types-urllib3 = "<1.27" + +[[package]] +name = "types-toml" +version = "0.10.8.7" +description = "Typing stubs for toml" +optional = false +python-versions = "*" +files = [ + {file = "types-toml-0.10.8.7.tar.gz", hash = "sha256:58b0781c681e671ff0b5c0319309910689f4ab40e8a2431e205d70c94bb6efb1"}, + {file = "types_toml-0.10.8.7-py3-none-any.whl", hash = "sha256:61951da6ad410794c97bec035d59376ce1cbf4453dc9b6f90477e81e4442d631"}, +] + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + +[[package]] +name = "types-xmltodict" +version = "0.13.0.3" +description = "Typing stubs for xmltodict" +optional = false +python-versions = "*" +files = [ + {file = "types-xmltodict-0.13.0.3.tar.gz", hash = "sha256:8884534bab0364c4b22d5973f3c8153ff40d413a801d9e70eb893e676909f1fc"}, + {file = "types_xmltodict-0.13.0.3-py3-none-any.whl", hash = "sha256:cb251c59e838986d8402b10d804225ade9fd6c9f66d01dc45cd6cfdf43640128"}, +] + [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] name = "tzdata" -version = "2023.3" +version = "2023.4" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] @@ -2211,24 +2415,24 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" -version = "0.2.10" +version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" files = [ - {file = "wcwidth-0.2.10-py2.py3-none-any.whl", hash = "sha256:aec5179002dd0f0d40c456026e74a729661c9d468e1ed64405e3a6c2176ca36f"}, - {file = "wcwidth-0.2.10.tar.gz", hash = "sha256:390c7454101092a6a5e43baad8f83de615463af459201709556b6e4b1c861f97"}, + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] [[package]] name = "websocket-client" -version = "1.6.4" +version = "1.7.0" description = "WebSocket client for Python with low level API options" optional = false python-versions = ">=3.8" files = [ - {file = "websocket-client-1.6.4.tar.gz", hash = "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df"}, - {file = "websocket_client-1.6.4-py3-none-any.whl", hash = "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24"}, + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, ] [package.extras] @@ -2315,87 +2519,125 @@ files = [ {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + +[[package]] +name = "xmod" +version = "1.8.1" +description = "🌱 Turn any object into a module 🌱" +optional = false +python-versions = ">=3.8" +files = [ + {file = "xmod-1.8.1-py3-none-any.whl", hash = "sha256:a24e9458a4853489042522bdca9e50ee2eac5ab75c809a91150a8a7f40670d48"}, + {file = "xmod-1.8.1.tar.gz", hash = "sha256:38c76486b9d672c546d57d8035df0beb7f4a9b088bc3fb2de5431ae821444377"}, +] + [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -2405,4 +2647,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "77faac7dd922e0ae74553daf492ab7d25b6ef3fe72a99aa32eb99730e5da2118" +content-hash = "f1c22f429416235274b390c130d1c02b046ee94bcbeb206b2322009cf240fdcf" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 76a70b7719f2..cc7abba8285d 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,13 +4,13 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.10.7" +version = "4.4.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] [tool.poetry.dependencies] python = "~3.10" -dagger-io = "^0.6.4" +dagger-io = "==0.9.6" asyncer = "^0.0.2" anyio = "^3.4.1" more-itertools = "^8.11.0" @@ -27,11 +27,11 @@ segment-analytics-python = "^2.2.3" pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" - -[tool.poetry.group.test.dependencies] -pytest = "^6.2.5" -pytest-mock = "^3.10.0" - +tomli = "^2.0.1" +tomli-w = "^1.0.0" +types-requests = "2.28.2" +dpath = "^2.1.6" +xmltodict = "^0.13.0" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" @@ -40,11 +40,28 @@ pyinstaller = "^6.1.0" poethepoet = "^0.24.2" pytest = "^6.2.5" pytest-mock = "^3.10.0" +mypy = "^1.7.1" +ruff = "^0.1.9" +types-toml = "^0.10.8" +types-requests = "2.28.2" +types-xmltodict = "^0.13.0" [tool.poetry.scripts] airbyte-ci = "pipelines.cli.airbyte_ci:airbyte_ci" airbyte-ci-dev = "pipelines.cli.airbyte_ci:airbyte_ci" + + [tool.poe.tasks.build-release-binary] shell = "pyinstaller --additional-hooks-dir=pyinstaller_hooks --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name $ARTIFACT_NAME --onefile pipelines/cli/airbyte_ci.py" args = [{name = "ARTIFACT_NAME", default="airbyte-ci", positional = true}] + +[tool.poe.tasks] +test = "pytest tests -m 'not flaky'" +type_check = "mypy pipelines --disallow-untyped-defs" +lint = "ruff check pipelines" + +[tool.airbyte_ci] +extra_poetry_groups = ["dev"] +poe_tasks = ["type_check", "lint", "test"] +mount_docker_socket = true diff --git a/airbyte-ci/connectors/pipelines/pytest.ini b/airbyte-ci/connectors/pipelines/pytest.ini index b228671b5fa2..3c9d16b1fe5d 100644 --- a/airbyte-ci/connectors/pipelines/pytest.ini +++ b/airbyte-ci/connectors/pipelines/pytest.ini @@ -2,3 +2,4 @@ addopts = --cov=pipelines markers = slow: marks tests as slow (deselect with '-m "not slow"') + flaky: marks tests as flaky (deselect with '-m "not flaky"') diff --git a/airbyte-ci/connectors/pipelines/ruff.toml b/airbyte-ci/connectors/pipelines/ruff.toml new file mode 100644 index 000000000000..7530d724d090 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/ruff.toml @@ -0,0 +1,22 @@ +target-version = "py310" + +ignore = ["ANN101", "ANN002", "ANN003"] + + + +[lint] + +extend-select = [ + +"ANN", # flake8-annotations + +] + + + +[lint.pydocstyle] + +convention = "google" + +[lint.flake8-annotations] +allow-star-arg-any = true diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py index 5b4547df1e45..15808f2c88de 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_bases.py +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -22,7 +22,7 @@ class DummyStep(steps.Step): async def _run(self, run_duration: timedelta) -> steps.StepResult: await anyio.sleep(run_duration.total_seconds()) - return steps.StepResult(self, steps.StepStatus.SUCCESS) + return steps.StepResult(step=self, status=steps.StepStatus.SUCCESS) @pytest.fixture def test_context(self, mocker): @@ -39,7 +39,7 @@ async def test_run_with_timeout(self, test_context): assert step_result.status == timed_out_step_result.status assert step_result.stdout == timed_out_step_result.stdout assert step_result.stderr == timed_out_step_result.stderr - assert step_result.output_artifact == timed_out_step_result.output_artifact + assert step_result.output == timed_out_step_result.output assert step.retry_count == step.max_retries + 1 @pytest.mark.parametrize( @@ -67,7 +67,8 @@ async def test_run_with_retries(self, mocker, test_context, step_status, exc_inf step.max_duration = timedelta(seconds=60) step.retry_delay = timedelta(seconds=0) step._run = mocker.AsyncMock( - side_effect=[steps.StepResult(step, step_status, exc_info=exc_info)] * (max(max_dagger_error_retries, max_retries) + 1) + side_effect=[steps.StepResult(step=step, status=step_status, exc_info=exc_info)] + * (max(max_dagger_error_retries, max_retries) + 1) ) step_result = await step.run() @@ -87,21 +88,23 @@ def test_context(self, mocker): def test_report_failed_if_it_has_no_step_result(self, test_context): report = reports.Report(test_context, []) assert not report.success - report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.FAILURE)]) + report = reports.Report(test_context, [steps.StepResult(step=None, status=steps.StepStatus.FAILURE)]) assert not report.success report = reports.Report( - test_context, [steps.StepResult(None, steps.StepStatus.FAILURE), steps.StepResult(None, steps.StepStatus.SUCCESS)] + test_context, + [steps.StepResult(step=None, status=steps.StepStatus.FAILURE), steps.StepResult(step=None, status=steps.StepStatus.SUCCESS)], ) assert not report.success - report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.SUCCESS)]) + report = reports.Report(test_context, [steps.StepResult(step=None, status=steps.StepStatus.SUCCESS)]) assert report.success report = reports.Report( - test_context, [steps.StepResult(None, steps.StepStatus.SUCCESS), steps.StepResult(None, steps.StepStatus.SKIPPED)] + test_context, + [steps.StepResult(step=None, status=steps.StepStatus.SUCCESS), steps.StepResult(step=None, status=steps.StepStatus.SKIPPED)], ) assert report.success - report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.SKIPPED)]) + report = reports.Report(test_context, [steps.StepResult(step=None, status=steps.StepStatus.SKIPPED)]) assert report.success diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py index bb8ac23a10ea..084911d82cf1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py @@ -117,14 +117,14 @@ async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_ container_built_from_base.with_exec.assert_called_with(["spec"]) assert step_result.status is StepStatus.SUCCESS for platform in all_platforms: - assert step_result.output_artifact[platform] == container_built_from_base + assert step_result.output[platform] == container_built_from_base @pytest.mark.slow async def test_building_from_base_image_for_real(self, test_context_with_real_connector_using_base_image, current_platform): step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image) step_result = await step._run() step_result.status is StepStatus.SUCCESS - built_container = step_result.output_artifact[current_platform] + built_container = step_result.output[current_platform] assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join( build_customization.get_entrypoint(step.context.connector) ) @@ -146,7 +146,7 @@ async def test_building_from_base_image_with_customization_for_real( step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image_with_build_customization) step_result = await step._run() step_result.status is StepStatus.SUCCESS - built_container = step_result.output_artifact[current_platform] + built_container = step_result.output[current_platform] assert await built_container.env_variable("MY_PRE_BUILD_ENV_VAR") == "my_pre_build_env_var_value" assert await built_container.env_variable("MY_POST_BUILD_ENV_VAR") == "my_post_build_env_var_value" @@ -161,7 +161,7 @@ async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_ container_built_from_dockerfile.with_exec.assert_called_with(["spec"]) assert step_result.status is StepStatus.SUCCESS for platform in all_platforms: - assert step_result.output_artifact[platform] == container_built_from_dockerfile + assert step_result.output[platform] == container_built_from_dockerfile async def test_building_from_dockerfile_for_real(self, test_context_with_real_connector_without_base_image): step = python_connectors.BuildConnectorImages(test_context_with_real_connector_without_base_image) diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py index 8b2fbbcf1e3d..b2289abb503d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py @@ -80,7 +80,7 @@ async def test_run(self, dagger_client, test_context, platforms): docker_client.images.get(full_image_name) # CI can't run docker arm64 containers - if platform is LOCAL_BUILD_PLATFORM or (os.getenv("CI") != "True"): + if platform is LOCAL_BUILD_PLATFORM or (os.environ.get("CI", "false").lower() != "true"): docker_client.containers.run(full_image_name, "spec") docker_client.images.remove(full_image_name, force=True) diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index 2bb2c3f7fa9e..5f3c1a72e43d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -1,6 +1,8 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import datetime + import pytest import requests from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -42,6 +44,7 @@ def context_with_setup(dagger_client, python_connector_with_setup_not_latest_cdk report_output_prefix="test", is_local=True, use_remote_secrets=False, + pipeline_start_timestamp=datetime.datetime.now().isoformat(), ) context.dagger_client = dagger_client return context @@ -61,7 +64,9 @@ async def test_with_python_connector_installed_from_setup(context_with_setup, py ) # Uninstall and reinstall the latest cdk version cdk_install_latest_output = ( - await container.with_exec(["pip", "uninstall", "-y", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) + await container.with_env_variable("CACHEBUSTER", datetime.datetime.now().isoformat()) + # .with_exec(["pip", "install", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) + .with_exec(["pip", "uninstall", "-y", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) .with_exec(["pip", "install", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) .stdout() ) diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index 82f47f803117..96a397e72855 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations from pathlib import Path @@ -17,9 +18,10 @@ class TestGradleTask: class DummyStep(gradle.GradleTask): gradle_task_name = "dummyTask" + title = "Dummy Step" async def _run(self) -> steps.StepResult: - return steps.StepResult(self, steps.StepStatus.SUCCESS) + return steps.StepResult(step=self, status=steps.StepStatus.SUCCESS) @pytest.fixture def test_context(self, mocker, dagger_client): @@ -34,3 +36,11 @@ def test_context(self, mocker, dagger_client): async def test_build_include(self, test_context): step = self.DummyStep(test_context) assert step.build_include + + def test_params(self, test_context): + step = self.DummyStep(test_context) + step.extra_params = {"-x": ["dummyTask", "dummyTask2"]} + assert set(step.params_as_cli_options) == { + "-x=dummyTask", + "-x=dummyTask2", + } diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py new file mode 100644 index 000000000000..7201a2b83059 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import enum +import time + +import anyio +import pytest +from pipelines.helpers.execution import argument_parsing + + +class SupportedStepIds(enum.Enum): + STEP1 = "step1" + STEP2 = "step2" + STEP3 = "step3" + + +def test_build_extra_params_mapping(mocker): + ctx = mocker.Mock() + argument = mocker.Mock() + + raw_extra_params = ( + "--step1.param1=value1", + "--step2.param2=value2", + "--step3.param3=value3", + "--step1.param4", + ) + + result = argument_parsing.build_extra_params_mapping(SupportedStepIds)(ctx, argument, raw_extra_params) + + expected_result = { + SupportedStepIds.STEP1.value: {"param1": ["value1"], "param4": []}, + SupportedStepIds.STEP2.value: {"param2": ["value2"]}, + SupportedStepIds.STEP3.value: {"param3": ["value3"]}, + } + + assert result == expected_result diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py new file mode 100644 index 000000000000..2d0193676b1b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py @@ -0,0 +1,424 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import time + +import anyio +import pytest +from pipelines.helpers.execution.run_steps import InvalidStepConfiguration, RunStepOptions, StepToRun, run_steps +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import Step, StepResult, StepStatus + +test_context = PipelineContext(pipeline_name="test", is_local=True, git_branch="test", git_revision="test", report_output_prefix="test") + + +class TestStep(Step): + title = "Test Step" + + async def _run(self, result_status=StepStatus.SUCCESS) -> StepResult: + return StepResult(step=self, status=result_status) + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "desc, steps, expected_results, options", + [ + ( + "All consecutive steps succeed", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps all succeed with parallel steps", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + ], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps after a failed step are skipped, when fail_fast is True", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SKIPPED, "step4": StepStatus.SKIPPED}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps after a failed step are not skipped, when fail_fast is False", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "fail fast has no effect on parallel steps", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + ], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "Nested parallel steps execute properly", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context)), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.SUCCESS, + "step6": StepStatus.SUCCESS, + }, + RunStepOptions(fail_fast=True), + ), + ( + "When fail_fast is True, nested parallel steps skip at the first failure", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE}), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.FAILURE, + "step6": StepStatus.SKIPPED, + }, + RunStepOptions(fail_fast=True), + ), + ( + "When fail_fast is False, nested parallel steps do not skip at the first failure", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE}), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.FAILURE, + "step6": StepStatus.SUCCESS, + }, + RunStepOptions(fail_fast=False), + ), + ( + "When fail_fast is False, consecutive steps still operate as expected", + [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context)), + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "skip_steps skips the specified steps", + [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context)), + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SKIPPED, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False, skip_steps=["step2"]), + ), + ( + "step is skipped if the dependency fails", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context), depends_on=["step2"])], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SKIPPED}, + RunStepOptions(fail_fast=False), + ), + ], +) +async def test_run_steps_output(desc, steps, expected_results, options): + results = await run_steps(steps, options=options) + + for step_id, expected_status in expected_results.items(): + assert results[step_id].status == expected_status, desc + + +@pytest.mark.anyio +async def test_run_steps_throws_on_invalid_order(): + concurrent_steps = [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context), depends_on=["step1"]), + ] + + with pytest.raises(InvalidStepConfiguration): + await run_steps(concurrent_steps) + + +@pytest.mark.anyio +async def test_run_steps_concurrent(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + await anyio.sleep(sleep) + ran_at[name] = time.time() + return StepResult(step=self, status=StepStatus.SUCCESS) + + steps = [ + StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 2}), + StepToRun(id="step2", step=SleepStep(test_context), args={"name": "step2", "sleep": 2}), + StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 2}), + StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 0}), + ] + + await run_steps(steps) + + # assert that step4 is the first step to finish + assert ran_at["step4"] < ran_at["step1"] + assert ran_at["step4"] < ran_at["step2"] + assert ran_at["step4"] < ran_at["step3"] + + +@pytest.mark.anyio +async def test_run_steps_concurrency_of_1(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + ran_at[name] = time.time() + await anyio.sleep(sleep) + return StepResult(step=self, status=StepStatus.SUCCESS) + + steps = [ + StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 1}), + StepToRun(id="step2", step=SleepStep(test_context), args={"name": "step2", "sleep": 1}), + StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 1}), + StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 1}), + ] + + await run_steps(steps, options=RunStepOptions(concurrency=1)) + + # Assert that they run sequentially + assert ran_at["step1"] < ran_at["step2"] + assert ran_at["step2"] < ran_at["step3"] + assert ran_at["step3"] < ran_at["step4"] + + +@pytest.mark.anyio +async def test_run_steps_sequential(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + await anyio.sleep(sleep) + ran_at[name] = time.time() + return StepResult(step=self, status=StepStatus.SUCCESS) + + steps = [ + [StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 1})], + [StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step2", "sleep": 1})], + [StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 1})], + [StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 0})], + ] + + await run_steps(steps) + + # assert that steps are run in order + assert ran_at["step1"] < ran_at["step2"] + assert ran_at["step2"] < ran_at["step3"] + assert ran_at["step3"] < ran_at["step4"] + + +@pytest.mark.anyio +async def test_run_steps_passes_results(): + """ + Example pattern + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=_create_integration_step_args_factory(context), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, True), + args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + + """ + + class Simple(Step): + title = "Test Step" + + async def _run(self, arg1, arg2) -> StepResult: + output = f"{arg1}:{arg2}" + return StepResult(step=self, status=StepStatus.SUCCESS, output=output) + + async def async_args(results): + return {"arg1": results["step2"].output, "arg2": "4"} + + steps = [ + [StepToRun(id="step1", step=Simple(test_context), args={"arg1": "1", "arg2": "2"})], + [StepToRun(id="step2", step=Simple(test_context), args=lambda results: {"arg1": results["step1"].output, "arg2": "3"})], + [StepToRun(id="step3", step=Simple(test_context), args=async_args)], + ] + + results = await run_steps(steps) + + assert results["step1"].output == "1:2" + assert results["step2"].output == "1:2:3" + assert results["step3"].output == "1:2:3:4" + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "invalid_args", + [ + 1, + True, + "string", + [1, 2], + None, + ], +) +async def test_run_steps_throws_on_invalid_args(invalid_args): + steps = [ + [StepToRun(id="step1", step=TestStep(test_context), args=invalid_args)], + ] + + with pytest.raises(TypeError): + await run_steps(steps) + + +@pytest.mark.anyio +async def test_run_steps_with_params(): + steps = [StepToRun(id="step1", step=TestStep(test_context))] + options = RunStepOptions(fail_fast=True, step_params={"step1": {"--param1": ["value1"]}}) + TestStep.accept_extra_params = False + with pytest.raises(ValueError): + await run_steps(steps, options=options) + assert steps[0].step.params_as_cli_options == [] + TestStep.accept_extra_params = True + await run_steps(steps, options=options) + assert steps[0].step.params_as_cli_options == ["--param1=value1"] + + +class TestRunStepOptions: + def test_init(self): + options = RunStepOptions() + assert options.fail_fast is True + assert options.concurrency == 10 + assert options.skip_steps == [] + assert options.step_params == {} + + options = RunStepOptions(fail_fast=False, concurrency=1, skip_steps=["step1"], step_params={"step1": {"--param1": ["value1"]}}) + assert options.fail_fast is False + assert options.concurrency == 1 + assert options.skip_steps == ["step1"] + assert options.step_params == {"step1": {"--param1": ["value1"]}} + + with pytest.raises(ValueError): + RunStepOptions(skip_steps=["step1"], keep_steps=["step2"]) + + @pytest.mark.parametrize( + "step_tree, options, expected_skipped_ids", + [ + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step3", "step1"]), + StepToRun(id="step5", step=TestStep(test_context)), + ], + RunStepOptions(keep_steps=["step4"]), + {"step2", "step5"}, + ), + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + [ + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step1"]), + StepToRun(id="step6", step=TestStep(test_context), depends_on=["step4", "step5"]), + ], + StepToRun(id="step5", step=TestStep(test_context), depends_on=["step3"]), + ], + RunStepOptions(keep_steps=["step6"]), + {"step2"}, + ), + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + [ + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step1"]), + StepToRun(id="step6", step=TestStep(test_context), depends_on=["step4", "step5"]), + ], + StepToRun(id="step5", step=TestStep(test_context), depends_on=["step3"]), + ], + RunStepOptions(skip_steps=["step1"]), + {"step1"}, + ), + ], + ) + def test_get_step_ids_to_skip(self, step_tree, options, expected_skipped_ids): + skipped_ids = options.get_step_ids_to_skip(step_tree) + assert set(skipped_ids) == expected_skipped_ids diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py new file mode 100644 index 000000000000..26605c675849 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from pipelines.helpers.pip import is_package_published + + +@pytest.mark.parametrize( + "package_name, version, registry_url, expected", + [ + pytest.param(None, None, "https://pypi.org/pypi", False, id="package_name and version are None"), + pytest.param(None, "0.2.0", "https://pypi.org/pypi", False, id="package_name is None"), + pytest.param("airbyte-source-pokeapi", None, "https://pypi.org/pypi", False, id="version is None"), + pytest.param("airbyte-source-pokeapi", "0.2.0", "https://pypi.org/pypi", True, id="published on pypi"), + pytest.param("airbyte-source-pokeapi", "0.1.0", "https://pypi.org/pypi", False, id="version not published on pypi"), + pytest.param("airbyte-source-nonexisting", "0.1.0", "https://pypi.org/pypi", False, id="package not published on pypi"), + pytest.param("airbyte-source-pokeapi", "0.2.1", "https://test.pypi.org/pypi", True, id="published on test.pypi"), + pytest.param("airbyte-source-pokeapi", "0.1.0", "https://test.pypi.org/pypi", False, id="version not published on test.pypi"), + pytest.param("airbyte-source-nonexisting", "0.1.0", "https://test.pypi.org/pypi", False, id="package not published on test.pypi"), + pytest.param("airbyte-source-pokeapi", "0.2.0", "https://some-non-existing-host.com", False, id="host does not exist"), + ], +) +def test_is_package_published(package_name, version, registry_url, expected): + assert is_package_published(package_name, version, registry_url) == expected diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py index f63bd000bbc7..aa7e305d809a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py @@ -12,6 +12,7 @@ from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers import utils from pipelines.helpers.connectors.modifed import get_connector_modified_files, get_modified_connectors +from pipelines.models.contexts.pipeline_context import PipelineContext from tests.utils import pick_a_random_connector @@ -193,6 +194,12 @@ def test_sh_dash_c(): assert utils.sh_dash_c([]) == ["sh", "-c", "set -o xtrace"] +def test_java_log_scrub_pattern(): + assert utils.java_log_scrub_pattern([]) == "" + assert utils.java_log_scrub_pattern(["foo", "bar"]) == "foo|bar" + assert utils.java_log_scrub_pattern(["|", "'\"{}\t[]<>&"]) == "\\||'"\\{\\}\\ \\[\\]<>\\&" + + @pytest.mark.anyio @pytest.mark.parametrize("tar_file_name", [None, "custom_tar_name.tar"]) async def test_export_container_to_tarball(mocker, dagger_client, tmp_path, tar_file_name): @@ -235,3 +242,19 @@ async def test_export_container_to_tarball_failure(mocker, tmp_path): str(tmp_path / f"my_connector_my_git_revision_{platform.replace('/', '_')}.tar"), forced_compression=dagger.ImageLayerCompression.Gzip, ) + + +# @pytest.mark.anyio +async def test_get_repo_dir(dagger_client): + test_context = PipelineContext(pipeline_name="test", is_local=True, git_branch="test", git_revision="test", report_output_prefix="test") + test_context.dagger_client = dagger_client + # we know airbyte-ci/connectors/pipelines/ is excluded + filtered_entries = await test_context.get_repo_dir("airbyte-ci/connectors/pipelines/").entries() + assert not filtered_entries + unfiltered_entries = await dagger_client.host().directory("airbyte-ci/connectors/pipelines/").entries() + assert unfiltered_entries + # we also know that **/secrets is excluded and that source-mysql contains a secrets file + filtered_entries = await test_context.get_repo_dir("airbyte-integrations/connectors/source-mysql/").entries() + assert "secrets" not in filtered_entries + unfiltered_entries = await dagger_client.host().directory("airbyte-integrations/connectors/source-mysql/").entries() + assert "secrets" in unfiltered_entries diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index 9a685372c6cd..4efb8b9e7b0a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -15,6 +15,8 @@ def cli(): pass ctx = click.Context(cli) + ctx.obj = {"foo": "bar"} + ctx.params = {"baz": "qux"} async with ctx.scope(): click_pipeline_context = ClickPipelineContext() diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py new file mode 100644 index 000000000000..69fb4699c989 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +import requests +from dagger import Client, Platform +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def context(dagger_client: Client): + context = PythonRegistryPublishContext( + package_path="test", + version="0.2.0", + python_registry_token="test", + package_name="test", + registry_check_url="http://local_registry:8080/", + registry="http://local_registry:8080/", + is_local=True, + git_branch="test", + git_revision="test", + report_output_prefix="test", + ci_report_bucket="test", + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "package_path, package_name, expected_asset", + [ + pytest.param( + "airbyte-integrations/connectors/source-apify-dataset", + "airbyte-source-apify-dataset", + "airbyte_source_apify_dataset-0.2.0-py3-none-any.whl", + id="setup.py project", + ), + pytest.param( + "airbyte-integrations/connectors/destination-duckdb", + "destination-duckdb", + "destination_duckdb-0.2.0-py3-none-any.whl", + id="poetry project", + ), + ], +) +async def test_run_poetry_publish(context: PythonRegistryPublishContext, package_path: str, package_name: str, expected_asset: str): + context.package_metadata = PythonPackageMetadata(package_name, "0.2.0") + context.package_path = package_path + pypi_registry = ( + # need to use linux/amd64 because the pypiserver image is only available for that platform + context.dagger_client.container(platform=Platform("linux/amd64")) + .from_("pypiserver/pypiserver:v2.0.1") + .with_exec(["run", "-P", ".", "-a", "."]) + .with_exposed_port(8080) + .as_service() + ) + + base_container = with_poetry(context).with_service_binding("local_registry", pypi_registry) + step = publish_pipeline.PublishToPythonRegistry(context) + step._get_base_container = MagicMock(return_value=base_container) + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Query the registry to check that the package was published + tunnel = await context.dagger_client.host().tunnel(pypi_registry).start() + endpoint = await tunnel.endpoint(scheme="http") + list_url = f"{endpoint}/simple/" + list_response = requests.get(list_url) + assert list_response.status_code == 200 + assert package_name in list_response.text + url = f"{endpoint}/simple/{package_name}" + response = requests.get(url) + assert response.status_code == 200 + assert expected_asset in response.text diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index e2b7bac964ab..f5cb73d0ca9e 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -1,8 +1,8 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import json +import os import random from typing import List @@ -154,6 +154,7 @@ def test_parse_spec_output_no_spec(self, publish_context): (publish_pipeline, "PushConnectorImageToRegistry"), (publish_pipeline, "PullConnectorImageFromRegistry"), (publish_pipeline.steps, "run_connector_build"), + (publish_pipeline, "CheckPythonRegistryPackageDoesNotExist"), ] @@ -277,12 +278,12 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( name="check_connector_image_does_not_exist_result", status=StepStatus.SUCCESS ) - # have output_artifact.values return [] + # have output.values return [] built_connector_platform = mocker.Mock() built_connector_platform.values.return_value = ["linux/amd64"] publish_pipeline.steps.run_connector_build.return_value = mocker.Mock( - name="build_connector_for_publish_result", status=build_step_status, output_artifact=built_connector_platform + name="build_connector_for_publish_result", status=build_step_status, output=built_connector_platform ) publish_pipeline.PushConnectorImageToRegistry.return_value.run.return_value = mocker.Mock( @@ -334,3 +335,76 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( publish_pipeline.PullConnectorImageFromRegistry.return_value.run.assert_not_called() publish_pipeline.UploadSpecToCache.return_value.run.assert_not_called() publish_pipeline.MetadataUpload.return_value.run.assert_not_called() + + +@pytest.mark.parametrize( + "pypi_enabled, pypi_package_does_not_exist_status, publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called,api_token", + [ + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, True, True, "test", id="happy_path"), + pytest.param(False, StepStatus.SUCCESS, StepStatus.SUCCESS, False, True, "test", id="pypi_disabled, skip all pypi steps"), + pytest.param(True, StepStatus.SKIPPED, StepStatus.SUCCESS, False, True, "test", id="pypi_package_exists, skip publish_to_pypi"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.FAILURE, True, False, "test", id="publish_step_fails, abort"), + pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, "test", id="pypi_package_does_not_exist_fails, abort"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, False, False, None, id="no_api_token, abort"), + ], +) +async def test_run_connector_python_registry_publish_pipeline( + mocker, + pypi_enabled, + pypi_package_does_not_exist_status, + publish_step_status, + expect_publish_to_pypi_called, + expect_build_connector_called, + api_token, +): + + for module, to_mock in STEPS_TO_PATCH: + mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) + + mocked_publish_to_python_registry = mocker.patch( + "pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPythonRegistry", return_value=mocker.AsyncMock() + ) + + for step in [ + publish_pipeline.MetadataValidation, + publish_pipeline.CheckConnectorImageDoesNotExist, + publish_pipeline.UploadSpecToCache, + publish_pipeline.MetadataUpload, + publish_pipeline.PushConnectorImageToRegistry, + publish_pipeline.PullConnectorImageFromRegistry, + ]: + step.return_value.run.return_value = mocker.Mock(name=f"{step.title}_result", status=StepStatus.SUCCESS) + + mocked_publish_to_python_registry.return_value.run.return_value = mocker.Mock( + name="publish_to_python_registry_result", status=publish_step_status + ) + + publish_pipeline.CheckPythonRegistryPackageDoesNotExist.return_value.run.return_value = mocker.Mock( + name="python_registry_package_does_not_exist_result", status=pypi_package_does_not_exist_status + ) + + context = mocker.MagicMock( + ci_gcs_credentials="", + pre_release=False, + connector=mocker.MagicMock( + code_directory="path/to/connector", + metadata={"dockerImageTag": "1.2.3", "remoteRegistries": {"pypi": {"enabled": pypi_enabled, "packageName": "test"}}}, + ), + python_registry_token=api_token, + python_registry_url="https://test.pypi.org/legacy/", + ) + semaphore = anyio.Semaphore(1) + await publish_pipeline.run_connector_publish_pipeline(context, semaphore) + if expect_publish_to_pypi_called: + mocked_publish_to_python_registry.return_value.run.assert_called_once() + # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context + assert mocked_publish_to_python_registry.call_args.args[0].python_registry_token == api_token + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.name == "test" + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.version == "1.2.3" + assert mocked_publish_to_python_registry.call_args.args[0].registry == "https://test.pypi.org/legacy/" + assert mocked_publish_to_python_registry.call_args.args[0].package_path == "path/to/connector" + else: + mocked_publish_to_python_registry.return_value.run.assert_not_called() + + if expect_build_connector_called: + publish_pipeline.steps.run_connector_build.assert_called_once() diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py index 27d5765037fe..01b83f561e1f 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -22,6 +22,7 @@ def context(dagger_client): is_local=True, git_branch="test", git_revision="test", + report_output_prefix="test", ) context.dagger_client = dagger_client return context diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 7f79cdea8680..047a130e097a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -11,6 +11,7 @@ import pytest import yaml from freezegun import freeze_time +from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps import common from pipelines.dagger.actions.system import docker from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles @@ -39,8 +40,19 @@ def get_dummy_cat_container(dagger_client: dagger.Client, exit_code: int, secret return container.with_new_file("/stupid_bash_script.sh", contents=f"echo {stdout}; echo {stderr} >&2; exit {exit_code}") @pytest.fixture - def test_context(self, mocker, dagger_client): - return mocker.MagicMock(connector=ConnectorWithModifiedFiles("source-faker", frozenset()), dagger_client=dagger_client) + def test_context_ci(self, current_platform, dagger_client): + context = ConnectorContext( + pipeline_name="test", + connector=ConnectorWithModifiedFiles("source-faker", frozenset()), + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=False, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context @pytest.fixture def dummy_connector_under_test_container(self, dagger_client) -> dagger.Container: @@ -50,9 +62,9 @@ def dummy_connector_under_test_container(self, dagger_client) -> dagger.Containe def another_dummy_connector_under_test_container(self, dagger_client) -> dagger.File: return dagger_client.container().from_("airbyte/source-pokeapi:latest") - async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context): - test_context.connector = mocker.MagicMock(acceptance_test_config=None) - acceptance_test_step = common.AcceptanceTests(test_context) + async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context_ci): + test_context_ci.connector = mocker.MagicMock(acceptance_test_config=None) + acceptance_test_step = common.AcceptanceTests(test_context_ci) step_result = await acceptance_test_step._run(None) assert step_result.status == StepStatus.SKIPPED @@ -112,7 +124,7 @@ async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context ) async def test__run( self, - test_context, + test_context_ci, mocker, exit_code: int, expected_status: StepStatus, @@ -122,23 +134,23 @@ async def test__run( ): """Test the behavior of the run function using a dummy container.""" cat_container = self.get_dummy_cat_container( - test_context.dagger_client, exit_code, secrets_file_names, stdout="hello", stderr="world" + test_context_ci.dagger_client, exit_code, secrets_file_names, stdout="hello", stderr="world" ) async_mock = mocker.AsyncMock(return_value=cat_container) mocker.patch.object(common.AcceptanceTests, "_build_connector_acceptance_test", side_effect=async_mock) mocker.patch.object(common.AcceptanceTests, "get_cat_command", return_value=["bash", "/stupid_bash_script.sh"]) - test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) - acceptance_test_step = common.AcceptanceTests(test_context) + test_context_ci.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + acceptance_test_step = common.AcceptanceTests(test_context_ci) step_result = await acceptance_test_step._run(None) assert step_result.status == expected_status assert step_result.stdout.strip() == "hello" assert step_result.stderr.strip() == "world" if expect_updated_secrets: assert ( - await test_context.updated_secrets_dir.entries() + await test_context_ci.updated_secrets_dir.entries() == await cat_container.directory(f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}").entries() ) - assert any("updated_configurations" in str(file_name) for file_name in await test_context.updated_secrets_dir.entries()) + assert any("updated_configurations" in str(file_name) for file_name in await test_context_ci.updated_secrets_dir.entries()) @pytest.fixture def test_input_dir(self, dagger_client, tmpdir): @@ -146,17 +158,18 @@ def test_input_dir(self, dagger_client, tmpdir): yaml.safe_dump({"connector_image": "airbyte/connector_under_test_image:dev"}, f) return dagger_client.host().directory(str(tmpdir)) - def get_patched_acceptance_test_step(self, dagger_client, mocker, test_context, test_input_dir): - test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) - test_context.connector_acceptance_test_image = "bash:latest" - test_context.connector_secrets = {"config.json": dagger_client.set_secret("config.json", "connector_secret")} + def get_patched_acceptance_test_step(self, dagger_client, mocker, test_context_ci, test_input_dir): + test_secrets = {"config.json": dagger_client.set_secret("config.json", "connector_secret")} + test_context_ci.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + test_context_ci.connector_acceptance_test_image = "bash:latest" + test_context_ci.get_connector_secrets = mocker.AsyncMock(return_value=test_secrets) mocker.patch.object(docker, "load_image_to_docker_host", return_value="image_sha") mocker.patch.object(docker, "with_bound_docker_host", lambda _, cat_container: cat_container) - return common.AcceptanceTests(test_context) + return common.AcceptanceTests(test_context_ci) async def test_cat_container_provisioning( - self, dagger_client, mocker, test_context, test_input_dir, dummy_connector_under_test_container + self, dagger_client, mocker, test_context_ci, test_input_dir, dummy_connector_under_test_container ): """Check that the acceptance test container is correctly provisioned. We check that: @@ -168,9 +181,8 @@ async def test_cat_container_provisioning( # The mounted_connector_secrets behaves differently when the test is run locally or in CI. # It is not masking the secrets when run locally. # We want to confirm that the secrets are correctly masked when run in CI. - test_context.is_local = False - test_context.is_ci = True - acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) + + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) cat_container = await acceptance_test_step._build_connector_acceptance_test(dummy_connector_under_test_container, test_input_dir) assert (await cat_container.with_exec(["pwd"]).stdout()).strip() == acceptance_test_step.CONTAINER_TEST_INPUT_DIRECTORY test_input_ls_result = await cat_container.with_exec(["ls"]).stdout() @@ -181,11 +193,15 @@ async def test_cat_container_provisioning( env_vars = {await env_var.name(): await env_var.value() for env_var in await cat_container.env_variables()} assert "CACHEBUSTER" in env_vars + @pytest.mark.flaky + # This test has shown some flakiness in CI + # This should be investigated and fixed + # https://github.com/airbytehq/airbyte-internal-issues/issues/6304 async def test_cat_container_caching( self, dagger_client, mocker, - test_context, + test_context_ci, test_input_dir, dummy_connector_under_test_container, another_dummy_connector_under_test_container, @@ -195,89 +211,39 @@ async def test_cat_container_caching( initial_datetime = datetime.datetime(year=1992, month=6, day=19, hour=13, minute=1, second=0) with freeze_time(initial_datetime) as frozen_datetime: - acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) - cat_container = await acceptance_test_step._build_connector_acceptance_test( + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) + first_cat_container = await acceptance_test_step._build_connector_acceptance_test( dummy_connector_under_test_container, test_input_dir ) - cat_container = cat_container.with_exec(["date"]) - fist_date_result = await cat_container.stdout() + fist_date_result = await first_cat_container.with_exec(["date"]).stdout() frozen_datetime.tick(delta=datetime.timedelta(hours=5)) # Check that cache is used in the same day - cat_container = await acceptance_test_step._build_connector_acceptance_test( + second_cat_container = await acceptance_test_step._build_connector_acceptance_test( dummy_connector_under_test_container, test_input_dir ) - cat_container = cat_container.with_exec(["date"]) - second_date_result = await cat_container.stdout() + + second_date_result = await second_cat_container.with_exec(["date"]).stdout() assert fist_date_result == second_date_result # Check that cache bursted after a day - frozen_datetime.tick(delta=datetime.timedelta(days=1, seconds=1)) - cat_container = await acceptance_test_step._build_connector_acceptance_test( + frozen_datetime.tick(delta=datetime.timedelta(days=1, minutes=10)) + third_cat_container = await acceptance_test_step._build_connector_acceptance_test( dummy_connector_under_test_container, test_input_dir ) - cat_container = cat_container.with_exec(["date"]) - third_date_result = await cat_container.stdout() + third_date_result = await third_cat_container.with_exec(["date"]).stdout() assert third_date_result != second_date_result time.sleep(1) # Check that changing the container invalidates the cache - cat_container = await acceptance_test_step._build_connector_acceptance_test( + fourth_cat_container = await acceptance_test_step._build_connector_acceptance_test( another_dummy_connector_under_test_container, test_input_dir ) - cat_container = cat_container.with_exec(["date"]) - fourth_date_result = await cat_container.stdout() + fourth_date_result = await fourth_cat_container.with_exec(["date"]).stdout() assert fourth_date_result != third_date_result - -class TestCheckBaseImageIsUsed: - @pytest.fixture - def certified_connector_no_base_image(self, all_connectors): - for connector in all_connectors: - if connector.metadata.get("supportLevel") == "certified": - if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is None: - return connector - pytest.skip("No certified connector without base image found") - - @pytest.fixture - def certified_connector_with_base_image(self, all_connectors): - for connector in all_connectors: - if connector.metadata.get("supportLevel") == "certified": - if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None: - return connector - pytest.skip("No certified connector with base image found") - - @pytest.fixture - def community_connector_no_base_image(self, all_connectors): - for connector in all_connectors: - if connector.metadata.get("supportLevel") == "community": - if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is None: - return connector - pytest.skip("No certified connector without base image found") - - @pytest.fixture - def test_context(self, mocker, dagger_client): - return mocker.MagicMock(dagger_client=dagger_client) - - async def test_pass_on_community_connector_no_base_image(self, mocker, dagger_client, community_connector_no_base_image): - test_context = mocker.MagicMock(dagger_client=dagger_client, connector=community_connector_no_base_image) - check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) - step_result = await check_base_image_is_used_step.run() - assert step_result.status == StepStatus.SKIPPED - - async def test_pass_on_certified_connector_with_base_image(self, mocker, dagger_client, certified_connector_with_base_image): - dagger_connector_dir = dagger_client.host().directory(str(certified_connector_with_base_image.code_directory)) - test_context = mocker.MagicMock( - dagger_client=dagger_client, - connector=certified_connector_with_base_image, - get_connector_dir=mocker.AsyncMock(return_value=dagger_connector_dir), - ) - check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) - step_result = await check_base_image_is_used_step.run() - assert step_result.status == StepStatus.SUCCESS - - async def test_fail_on_certified_connector_no_base_image(self, mocker, dagger_client, certified_connector_no_base_image): - test_context = mocker.MagicMock(dagger_client=dagger_client, connector=certified_connector_no_base_image) - check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) - step_result = await check_base_image_is_used_step.run() - assert step_result.status == StepStatus.FAILURE + async def test_params(self, dagger_client, mocker, test_context_ci, test_input_dir): + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) + assert set(acceptance_test_step.params_as_cli_options) == {"-ra", "--disable-warnings", "--durations=3"} + acceptance_test_step.extra_params = {"--durations": ["5"], "--collect-only": []} + assert set(acceptance_test_step.params_as_cli_options) == {"-ra", "--disable-warnings", "--durations=5", "--collect-only"} diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index da63c33fb01c..f53b43ebe57a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -2,12 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from unittest.mock import patch + import pytest from connector_ops.utils import Connector, ConnectorLanguage from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.test.steps.python_connectors import UnitTests -from pipelines.models.steps import StepResult +from pipelines.airbyte_ci.connectors.test.steps.python_connectors import AirbyteLibValidation, UnitTests +from pipelines.models.steps import StepResult, StepStatus pytestmark = [ pytest.mark.anyio, @@ -28,7 +30,7 @@ def certified_connector_with_setup(self, all_connectors): pytest.skip("No certified connector with setup.py found.") @pytest.fixture - def context_for_certified_connector_with_setup(self, certified_connector_with_setup, dagger_client, current_platform): + def context_for_certified_connector_with_setup(self, mocker, certified_connector_with_setup, dagger_client, current_platform): context = ConnectorContext( pipeline_name="test unit tests", connector=certified_connector_with_setup, @@ -40,16 +42,16 @@ def context_for_certified_connector_with_setup(self, certified_connector_with_se targeted_platforms=[current_platform], ) context.dagger_client = dagger_client - context.connector_secrets = {} + context.get_connector_secrets = mocker.AsyncMock(return_value={}) return context @pytest.fixture async def certified_container_with_setup(self, context_for_certified_connector_with_setup, current_platform): result = await BuildConnectorImages(context_for_certified_connector_with_setup).run() - return result.output_artifact[current_platform] + return result.output[current_platform] @pytest.fixture - def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client, current_platform): + def context_for_connector_with_poetry(self, mocker, connector_with_poetry, dagger_client, current_platform): context = ConnectorContext( pipeline_name="test unit tests", connector=connector_with_poetry, @@ -61,13 +63,13 @@ def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client targeted_platforms=[current_platform], ) context.dagger_client = dagger_client - context.connector_secrets = {} + context.get_connector_secrets = mocker.AsyncMock(return_value={}) return context @pytest.fixture async def container_with_poetry(self, context_for_connector_with_poetry, current_platform): result = await BuildConnectorImages(context_for_connector_with_poetry).run() - return result.output_artifact[current_platform] + return result.output[current_platform] async def test__run_for_setup_py(self, context_for_certified_connector_with_setup, certified_container_with_setup): # Assume that the tests directory is available @@ -78,7 +80,7 @@ async def test__run_for_setup_py(self, context_for_certified_connector_with_setu "Total coverage:" in result.stdout ), "The pytest-cov package should be installed in the test environment and test coverage report should be displayed." assert "Required test coverage of" in result.stdout, "A test coverage threshold should be defined for certified connectors." - pip_freeze_output = await result.output_artifact.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout() + pip_freeze_output = await result.output.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout() assert ( context_for_certified_connector_with_setup.connector.technical_name in pip_freeze_output ), "The connector should be installed in the test environment." @@ -91,9 +93,86 @@ async def test__run_for_poetry(self, context_for_connector_with_poetry, containe assert isinstance(result, StepResult) # We only check for the presence of "test session starts" because we have no guarantee that the tests will pass assert "test session starts" in result.stdout or "test session starts" in result.stderr, "The pytest tests should have started." - pip_freeze_output = await result.output_artifact.with_exec(["poetry", "run", "pip", "freeze"], skip_entrypoint=True).stdout() + pip_freeze_output = await result.output.with_exec(["poetry", "run", "pip", "freeze"], skip_entrypoint=True).stdout() assert ( context_for_connector_with_poetry.connector.technical_name in pip_freeze_output ), "The connector should be installed in the test environment." assert "pytest" in pip_freeze_output, "The pytest package should be installed in the test environment." + + def test_params(self, context_for_certified_connector_with_setup): + step = UnitTests(context_for_certified_connector_with_setup) + assert step.params_as_cli_options == [ + "-s", + f"--cov={context_for_certified_connector_with_setup.connector.technical_name.replace('-', '_')}", + f"--cov-fail-under={step.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS}", + ] + + +class TestAirbyteLibValidationTests: + @pytest.fixture + def compatible_connector(self): + return Connector("source-faker") + + @pytest.fixture + def incompatible_connector(self): + return Connector("source-postgres") + + @pytest.fixture + def context_for_valid_connector(self, compatible_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test airbyte-lib validation", + connector=compatible_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + @pytest.fixture + def context_for_invalid_connector(self, incompatible_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test airbyte-lib validation", + connector=incompatible_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + async def test__run_validation_success(self, mocker, context_for_valid_connector: ConnectorContext): + result = await AirbyteLibValidation(context_for_valid_connector)._run(mocker.MagicMock()) + assert isinstance(result, StepResult) + assert result.status == StepStatus.SUCCESS + assert "Creating source and validating spec is returned successfully..." in result.stdout + + async def test__run_validation_skip_unpublished_connector( + self, + mocker, + context_for_invalid_connector: ConnectorContext, + ): + result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock()) + assert isinstance(result, StepResult) + assert result.status == StepStatus.SKIPPED + + async def test__run_validation_fail( + self, + mocker, + context_for_invalid_connector: ConnectorContext, + ): + metadata = context_for_invalid_connector.connector.metadata + metadata["remoteRegistries"] = {"pypi": {"enabled": True, "packageName": "airbyte-source-postgres"}} + metadata_mock = mocker.PropertyMock(return_value=metadata) + with patch.object(Connector, "metadata", metadata_mock): + result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock()) + assert isinstance(result, StepResult) + assert result.status == StepStatus.FAILURE + assert "is not installable" in result.stderr diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py new file mode 100644 index 000000000000..cbe91b4df3c1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import random +from pathlib import Path +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import anyio +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from dagger import Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def sample_connector(): + return Connector("source-postgres") + + +def get_sample_build_gradle(airbyte_cdk_version: str, useLocalCdk: str): + return f"""import org.jsonschema2pojo.SourceType + +plugins {{ + id 'application' + id 'airbyte-java-connector' + id "org.jsonschema2pojo" version "1.2.1" +}} + +java {{ + compileJava {{ + options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked" + }} +}} + +airbyteJavaConnector {{ + cdkVersionRequired = '{airbyte_cdk_version}' + features = ['db-sources'] + useLocalCdk = {useLocalCdk} +}} + + +application {{ + mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +}} +""" + + +@pytest.fixture +def connector_context(sample_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test", + connector=sample_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "build_gradle_content, expected_build_gradle_content", + [ + (get_sample_build_gradle("1.2.3", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("1.2.3", "true"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("6.6.6", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("6.6.6", "true"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("7.0.0", "false"), get_sample_build_gradle("6.6.6", "false")), + (get_sample_build_gradle("7.0.0", "true"), get_sample_build_gradle("6.6.6", "false")), + ], +) +async def test_run_connector_cdk_upgrade_pipeline( + connector_context: ConnectorContext, build_gradle_content: str, expected_build_gradle_content: str +): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", build_gradle_content) + + # For this test, replace the actual connector dir with an updated version that sets the build.gradle contents + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + # Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo + updated_connector_dir.diff = MagicMock(return_value=AsyncMock()) + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Check that the resulting directory that got passed to the mocked diff method looks as expected + resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0]) + files = await resulting_directory.entries() + # validate only build.gradle is changed + assert files == ["build.gradle"] + build_gradle = resulting_directory.file("build.gradle") + actual_build_gradle_content = await build_gradle.contents() + assert expected_build_gradle_content == actual_build_gradle_content + + # Assert that the diff was exported to the repo + assert updated_connector_dir.diff.return_value.export.call_count == 1 + + +async def test_skip_connector_cdk_upgrade_pipeline_on_missing_build_gradle(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.without_file("build.gradle") + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE + + +async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", get_sample_build_gradle("abc", "false")) + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py new file mode 100644 index 000000000000..67d855a1d91e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import random +from pathlib import Path +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import anyio +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from dagger import Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def sample_connector(): + return Connector("source-pokeapi") + + +def get_sample_setup_py(airbyte_cdk_dependency: str): + return f"""from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "{airbyte_cdk_dependency}", +] + +setup( + name="source_pokeapi", + description="Source implementation for Pokeapi.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, +) +""" + + +@pytest.fixture +def connector_context(sample_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test", + connector=sample_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "setup_py_content, expected_setup_py_content", + [ + (get_sample_setup_py("airbyte-cdk"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + (get_sample_setup_py("airbyte-cdk==1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk>=1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]>=1.2.3"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + (get_sample_setup_py("airbyte-cdk==1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk>=1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]>=1.2"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + ], +) +async def test_run_connector_cdk_upgrade_pipeline( + connector_context: ConnectorContext, setup_py_content: str, expected_setup_py_content: str +): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", setup_py_content) + + # For this test, replace the actual connector dir with an updated version that sets the setup.py contents + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + # Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo + updated_connector_dir.diff = MagicMock(return_value=AsyncMock()) + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Check that the resulting directory that got passed to the mocked diff method looks as expected + resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0]) + files = await resulting_directory.entries() + # validate only setup.py is changed + assert files == ["setup.py"] + setup_py = resulting_directory.file("setup.py") + actual_setup_py_content = await setup_py.contents() + assert expected_setup_py_content == actual_setup_py_content + + # Assert that the diff was exported to the repo + assert updated_connector_dir.diff.return_value.export.call_count == 1 + + +async def test_skip_connector_cdk_upgrade_pipeline_on_missing_setup_py(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.without_file("setup.py") + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SKIPPED + + +async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", get_sample_setup_py("another-lib==1.2.3")) + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE diff --git a/airbyte-ci/connectors/qa-engine/pyproject.toml b/airbyte-ci/connectors/qa-engine/pyproject.toml index 2ca0e4554103..05022c51d447 100644 --- a/airbyte-ci/connectors/qa-engine/pyproject.toml +++ b/airbyte-ci/connectors/qa-engine/pyproject.toml @@ -29,12 +29,10 @@ ruamel-yaml = "^0.17.30" connector-ops = {path = "../connector_ops"} [tool.poetry.group.dev.dependencies] -pyinstrument = "*" - -[tool.poetry.group.test.dependencies] pytest = "~6.2.5" pytest-mock = "~3.10.0" freezegun = "*" +pyinstrument = "*" [tool.poetry.scripts] run-qa-engine = "qa_engine.main:main" diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index a80a2274f156..0c2de175e2cc 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -1,27 +1,3 @@ plugins { - id 'java-library' id 'airbyte-docker-legacy' } - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - implementation 'commons-cli:commons-cli:1.4' - implementation 'net.i2p.crypto:eddsa:0.3.0' - implementation 'org.apache.sshd:sshd-mina:2.8.0' - // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java - // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation - implementation 'org.bouncycastle:bcprov-jdk15on:1.66' - implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' - implementation 'org.bouncycastle:bctls-jdk15on:1.66' - - implementation libs.jackson.annotations - implementation libs.testcontainers - implementation libs.testcontainers.jdbc - implementation libs.bundles.datadog - - testImplementation 'commons-lang:commons-lang:2.6' - implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' -} diff --git a/airbyte-integrations/bases/base-java/javabase.sh b/airbyte-integrations/bases/base-java/javabase.sh index d835d4e9e0eb..59ceb87713fa 100755 --- a/airbyte-integrations/bases/base-java/javabase.sh +++ b/airbyte-integrations/bases/base-java/javabase.sh @@ -16,7 +16,7 @@ if [[ $IS_CAPTURE_HEAP_DUMP_ON_ERROR = true ]]; then fi fi #30781 - Allocate 32KB for log4j appender buffer to ensure that each line is logged in a single println -JAVA_OPTS=$JAVA_OPTS" -Dlog4j.encoder.byteBufferSize=32768" +JAVA_OPTS=$JAVA_OPTS" -Dlog4j.encoder.byteBufferSize=32768 -Dlog4j2.configurationFile=log4j2.xml" export JAVA_OPTS # Wrap run script in a script so that we can lazy evaluate the value of APPLICATION. APPLICATION is diff --git a/airbyte-integrations/bases/base-java/run_with_normalization.sh b/airbyte-integrations/bases/base-java/run_with_normalization.sh index eb11d6443c42..669763021803 100755 --- a/airbyte-integrations/bases/base-java/run_with_normalization.sh +++ b/airbyte-integrations/bases/base-java/run_with_normalization.sh @@ -36,6 +36,10 @@ then elif test "$NORMALIZATION_TECHNIQUE" = 'LEGACY' && test "$USE_1S1T_FORMAT" != "true" then echo '{"type": "LOG","log":{"level":"INFO","message":"Starting in-connector normalization"}}' + # Normalization tries to create this file from the connector config and crashes if it already exists + # so just nuke it and let normalization recreate it. + # Use -f to avoid error if it doesn't exist, since it's only created for certain SSL modes. + rm -f ca.crt # the args in a write command are `write --catalog foo.json --config bar.json` # so if we remove the `write`, we can just pass the rest directly into normalization /airbyte/entrypoint.sh run ${@:2} --integration-type $AIRBYTE_NORMALIZATION_INTEGRATION | java -cp "/airbyte/lib/*" io.airbyte.cdk.integrations.destination.normalization.NormalizationLogParser diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 961ff66d0097..13f2dd53c9f9 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -1,29 +1,26 @@ -import org.apache.tools.ant.taskdefs.condition.Os - plugins { id 'airbyte-docker-legacy' id 'airbyte-python' } dependencies { - project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') + testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) } // we need to access the sshtunneling script from airbyte-workers for ssh support def copySshScript = tasks.register('copySshScript', Copy) { - from "${project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').buildDir}/resources/main" + from "${project(':airbyte-cdk:java:airbyte-cdk:dependencies').buildDir}/resources/testFixtures" into "${buildDir}" include "sshtunneling.sh" } copySshScript.configure { - dependsOn project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').tasks.named('processResources') + dependsOn project(':airbyte-cdk:java:airbyte-cdk:dependencies').tasks.named('processTestFixturesResources') } // make sure the copy task above worked (if it fails, it fails silently annoyingly) def checkSshScriptCopy = tasks.register('checkSshScriptCopy') { doFirst { - assert file("${buildDir}/sshtunneling.sh").exists() : - "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers." + assert file("${buildDir}/sshtunneling.sh").exists() : "Copy of sshtunneling.sh failed." } } checkSshScriptCopy.configure { @@ -39,6 +36,10 @@ tasks.named('check').configure { dependsOn generate } +tasks.named("jar").configure { + dependsOn copySshScript +} + [ 'bigquery', 'mysql', @@ -49,7 +50,6 @@ tasks.named('check').configure { 'mssql', 'clickhouse', 'tidb', - 'duckdb', ].each {destinationName -> tasks.matching { it.name == 'integrationTestPython' }.configureEach { dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index 814730706297..5166e0fc1dc0 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## 3.5.0 +Add `validate_stream_statuses` to TestBasicRead.test_read:: Validate all statuses for all streams in the catalogs were emitted in correct order. + +## 3.4.0 +Add TestConnectorDocumentation suite for validating connectors documentation structure and content. + +## 3.3.3 +Аix `NoAdditionalPropertiesValidator` if no type found in `items` + +## 3.3.2 +Fix TestBasicRead.test_read.validate_schema: set `additionalProperties` to False recursively for objects. + +## 3.3.1 +Fix TestSpec.test_oauth_is_default_method to skip connectors that doesn't have predicate_key object. + +## 3.3.0 +Add `test_certified_connector_has_allowed_hosts` and `test_certified_connector_has_suggested_streams` tests to the `connector_attribute` test suite + +## 3.2.0 +Add TestBasicRead.test_all_supported_file_types_present, which validates that all supported file types are present in the sandbox account for certified file-based connectors. + +## 3.1.0 +Add TestSpec.test_oauth_is_default_method test with OAuth is default option validation. + +## 3.0.1 +Upgrade to Dagger 0.9.6 + +## 3.0.0 +Upgrade to Dagger 0.9.5 + +## 2.2.0 +Add connector_attribute test suite and stream primary key validation ## 2.1.4 diff --git a/airbyte-integrations/bases/connector-acceptance-test/README.md b/airbyte-integrations/bases/connector-acceptance-test/README.md index 3b33b204b512..9ca4822380ed 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/README.md @@ -1,7 +1,7 @@ # Connector Acceptance Tests (CAT) This package gathers multiple test suites to assess the sanity of any Airbyte connector. It is shipped as a [pytest](https://docs.pytest.org/en/7.1.x/) plugin and relies on pytest to discover, configure and execute tests. -Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/)). +Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/). ## Configuration The acceptance tests are configured via the `acceptance-test-config.yml` YAML file, which is passed to the plugin via the `--acceptance-test-config` option. @@ -93,7 +93,7 @@ These iterations are more conveniently achieved by remaining in the current dire 8. Make sure you updated `docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md` according to your changes 9. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` 10. Open a PR on our GitHub repository -11. This [Github action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered an run the unit tests on your branch. +11. This [GitHub action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered and run the unit tests on your branch. 12. Publish the new acceptance test version if your PR is approved by running `/legacy-publish connector=bases/connector-acceptance-test run-tests=false` in a GitHub comment 13. Merge your PR diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index 0546a4a972bd..a0d62f646163 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -7,7 +7,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Generic, List, Mapping, Optional, Set, TypeVar +from typing import Any, Dict, Generic, List, Mapping, Optional, Set, TypeVar from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel @@ -42,6 +42,17 @@ class BackwardCompatibilityTestsConfig(BaseConfig): ) +class OAuthTestConfig(BaseConfig): + oauth = Field(True, description="Allow source to have another default method that OAuth.") + bypass_reason: Optional[str] = Field(description="Reason why OAuth is not default method.") + + @validator("oauth", always=True) + def validate_oauth(cls, oauth, values): + if oauth is False and not values.get("bypass_reason"): + raise ValueError("Please provide a bypass reason for Auth default method") + return oauth + + class SpecTestConfig(BaseConfig): spec_path: str = spec_path config_path: str = config_path @@ -50,6 +61,7 @@ class SpecTestConfig(BaseConfig): backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() ) + auth_default_method: Optional[OAuthTestConfig] = Field(description="Auth default method details.") class ConnectionTestConfig(BaseConfig): @@ -82,7 +94,8 @@ class Config: extra_fields: bool = Field(False, description="Allow records to have other fields") exact_order: bool = Field(False, description="Ensure that records produced in exact same order") extra_records: bool = Field( - True, description="Allow connector to produce extra records, but still enforce all records from the expected file to be produced" + True, + description="Allow connector to produce extra records, but still enforce all records from the expected file to be produced", ) @validator("exact_order", always=True) @@ -124,6 +137,48 @@ class IgnoredFieldsConfiguration(BaseConfig): ) +class NoPrimaryKeyConfiguration(BaseConfig): + name: str + bypass_reason: Optional[str] = Field(default=None, description="Reason why this stream does not support a primary key") + + +class AllowedHostsConfiguration(BaseConfig): + bypass_reason: Optional[str] = Field( + default=None, description="Reason why the Metadata `AllowedHosts` check should be skipped for this certified connector." + ) + + +class SuggestedStreamsConfiguration(BaseConfig): + bypass_reason: Optional[str] = Field( + default=None, description="Reason why the Metadata `SuggestedStreams` check should be skipped for this certified connector." + ) + + +class UnsupportedFileTypeConfig(BaseConfig): + extension: str + bypass_reason: Optional[str] = Field(description="Reason why this type is considered unsupported.") + + @validator("extension", always=True) + def extension_properly_formatted(cls, extension: str) -> str: + if not extension.startswith(".") or len(extension) < 2: + raise ValueError("Please provide a valid file extension (e.g. '.csv').") + return extension + + +class FileTypesConfig(BaseConfig): + bypass_reason: Optional[str] = Field(description="Reason why this test is bypassed.") + unsupported_types: Optional[List[UnsupportedFileTypeConfig]] = Field(description="A list of unsupported file types for the source.") + skip_test: Optional[bool] = Field(False, description="Skip file-based connector specific test.") + + @validator("skip_test", always=True) + def no_unsupported_types_when_skip_test(cls, skip_test: bool, values: Dict[str, Any]) -> bool: + if skip_test and values.get("unsupported_types"): + raise ValueError("You can't set 'unsupported_types' if the test is skipped.") + if not skip_test and values.get("bypass_reason") is not None: + raise ValueError("You can't set 'bypass_reason' if the test is not skipped.") + return skip_test + + class BasicReadTestConfig(BaseConfig): config_path: str = config_path deployment_mode: Optional[str] = deployment_mode @@ -133,6 +188,7 @@ class BasicReadTestConfig(BaseConfig): ) expect_records: Optional[ExpectedRecordsConfig] = Field(description="Expected records from the read") validate_schema: bool = Field(True, description="Ensure that records match the schema of the corresponding stream") + validate_stream_statuses: bool = Field(None, description="Ensure that all streams emit status messages") fail_on_extra_columns: bool = Field(True, description="Fail if extra top-level properties (i.e. columns) are detected in records.") # TODO: remove this field after https://github.com/airbytehq/airbyte/issues/8312 is done validate_data_points: bool = Field( @@ -141,6 +197,10 @@ class BasicReadTestConfig(BaseConfig): expect_trace_message_on_failure: bool = Field(True, description="Ensure that a trace message is emitted when the connector crashes") timeout_seconds: int = timeout_seconds ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]] = ignored_fields + file_types: Optional[FileTypesConfig] = Field( + default_factory=FileTypesConfig, + description="For file-based connectors, unsupported by source file types can be configured or a test can be skipped at all", + ) class FullRefreshConfig(BaseConfig): @@ -159,7 +219,7 @@ class FullRefreshConfig(BaseConfig): class FutureStateConfig(BaseConfig): future_state_path: Optional[str] = Field(description="Path to a state file with values in far future") - missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missings streams with valid bypass reasons.") + missing_streams: List[EmptyStreamConfiguration] = Field(default=[], description="List of missing streams with valid bypass reasons.") bypass_reason: Optional[str] @@ -177,6 +237,34 @@ class Config: smart_union = True +class ConnectorAttributesConfig(BaseConfig): + """ + Config that is used to verify that a connector and its streams uphold certain behavior and features that are + required to maintain enterprise-level standard of quality. + + Attributes: + streams_without_primary_key: A list of streams where a primary key is not available from the API or is not relevant to the record + """ + + timeout_seconds: int = timeout_seconds + config_path: str = config_path + + streams_without_primary_key: Optional[List[NoPrimaryKeyConfiguration]] = Field( + description="Streams that do not support a primary key such as reports streams" + ) + allowed_hosts: Optional[AllowedHostsConfiguration] = Field( + description="Used to bypass checking the `allowedHosts` field in a source's `metadata.yaml` when all external hosts should be reachable." + ) + suggested_streams: Optional[SuggestedStreamsConfiguration] = Field( + description="Used to bypass checking the `suggestedStreams` field in a source's `metadata.yaml` when certified source doesn't have any." + ) + + +class TestConnectorDocumentationConfig(BaseConfig): + timeout_seconds: int = timeout_seconds + config_path: str = config_path + + class GenericTestConfig(GenericModel, Generic[TestConfigT]): bypass_reason: Optional[str] tests: Optional[List[TestConfigT]] @@ -195,6 +283,8 @@ class AcceptanceTestConfigurations(BaseConfig): basic_read: Optional[GenericTestConfig[BasicReadTestConfig]] full_refresh: Optional[GenericTestConfig[FullRefreshConfig]] incremental: Optional[GenericTestConfig[IncrementalConfig]] + connector_attributes: Optional[GenericTestConfig[ConnectorAttributesConfig]] + connector_documentation: Optional[GenericTestConfig[TestConnectorDocumentationConfig]] class Config(BaseConfig): @@ -231,9 +321,9 @@ def migrate_legacy_to_current_config(legacy_config: dict) -> dict: """Convert configuration structure created prior to v0.2.12 into the current structure. e.g. This structure: - {"connector_image": "my-connector-image", "tests": {"spec": [{"spec_path": "my/spec/path.json"}]} + {"connector_image": "my-connector-image", "tests": {"spec": [{"spec_path": "my/spec/path.json"}]}} Gets converted to: - {"connector_image": "my-connector-image", "acceptance_tests": {"spec": {"tests": [{"spec_path": "my/spec/path.json"}]}} + {"connector_image": "my-connector-image", "acceptance_tests": {"spec": {"tests": [{"spec_path": "my/spec/path.json"}]}}} Args: legacy_config (dict): A legacy configuration @@ -278,7 +368,7 @@ def legacy_format_adapter(cls, values: dict) -> dict: dict: The migrated configuration if needed. """ if ALLOW_LEGACY_CONFIG and cls.is_legacy(values): - logging.warn("The acceptance-test-config.yml file is in a legacy format. Please migrate to the latest format.") + logging.warning("The acceptance-test-config.yml file is in a legacy format. Please migrate to the latest format.") return cls.migrate_legacy_to_current_config(values) else: return values diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index f0df880650fa..94e9f815f24a 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -395,3 +395,21 @@ def pytest_sessionfinish(session, exitstatus): @pytest.fixture(name="connector_metadata") def connector_metadata_fixture(base_path) -> dict: return load_yaml_or_json_path(base_path / "metadata.yaml") + + +@pytest.fixture(name="docs_path") +def docs_path_fixture(base_path, connector_metadata) -> Path: + path_to_docs = connector_metadata["data"]["documentationUrl"].replace("https://docs.airbyte.com", "docs") + ".md" + airbyte_path = Path(base_path).parents[6] + return airbyte_path / path_to_docs + + +@pytest.fixture(name="connector_documentation") +def connector_documentation_fixture(docs_path: str) -> str: + with open(docs_path, "r") as f: + return f.read().rstrip() + + +@pytest.fixture(name="is_connector_certified") +def connector_certification_status_fixture(connector_metadata: dict) -> bool: + return connector_metadata.get("data", {}).get("ab_internal", {}).get("ql", 0) >= 400 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py index cc93282d5660..5236bba39b4b 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py @@ -2,8 +2,8 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from .test_core import TestBasicRead, TestConnection, TestDiscovery, TestSpec +from .test_core import TestBasicRead, TestConnection, TestConnectorAttributes, TestDiscovery, TestSpec, TestConnectorDocumentation from .test_full_refresh import TestFullRefresh from .test_incremental import TestIncremental -__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestDiscovery", "TestFullRefresh", "TestIncremental"] +__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestConnectorAttributes", "TestDiscovery", "TestFullRefresh", "TestIncremental", "TestConnectorDocumentation"] diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_cloud.txt b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_cloud.txt new file mode 100644 index 000000000000..54946b70acdc --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_cloud.txt @@ -0,0 +1,5 @@ + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. Click Sources and then click + New source/destination. +3. On the Set up the source page, select {connector_name} from the Source type dropdown. +4. Enter a name for the {connector_name} connector. diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_open_source.txt b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_open_source.txt new file mode 100644 index 000000000000..c5249d21023a --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/for_airbyte_open_source.txt @@ -0,0 +1,2 @@ + +1. Navigate to the Airbyte Open Source dashboard. diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/source.txt b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/source.txt new file mode 100644 index 000000000000..d7b36f402209 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/source.txt @@ -0,0 +1,6 @@ + + + +This page contains the setup guide and reference information for the [{connector_name}]({docs_link}) source connector. + + diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/supported_sync_modes.txt b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/supported_sync_modes.txt new file mode 100644 index 000000000000..51e8f806ff65 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/supported_sync_modes.txt @@ -0,0 +1,2 @@ + +The {connector_name} source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/tutorials.txt b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/tutorials.txt new file mode 100644 index 000000000000..584c04f8daf5 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/doc_templates/tutorials.txt @@ -0,0 +1,2 @@ + +Now that you have set up the {connector_name} source connector, check out the following {connector_name} tutorials: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index 022ee511c56a..29e1fc4e39fd 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -9,15 +9,22 @@ from collections import Counter, defaultdict from functools import reduce from logging import Logger +from os.path import splitext +from pathlib import Path +from threading import Thread from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Set, Tuple from xmlrpc.client import Boolean +import connector_acceptance_test.utils.docs as docs_utils import dpath.util import jsonschema import pytest +import requests from airbyte_protocol.models import ( AirbyteRecordMessage, AirbyteStream, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, @@ -29,14 +36,18 @@ ) from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import ( + AllowedHostsConfiguration, BasicReadTestConfig, Config, ConnectionTestConfig, + ConnectorAttributesConfig, DiscoveryTestConfig, EmptyStreamConfiguration, ExpectedRecordsConfig, IgnoredFieldsConfiguration, + NoPrimaryKeyConfiguration, SpecTestConfig, + UnsupportedFileTypeConfig, ) from connector_acceptance_test.utils import ConnectorRunner, SecretDict, delete_fields, filter_output, make_hashable, verify_records_schema from connector_acceptance_test.utils.backward_compatibility import CatalogDiffChecker, SpecDiffChecker, validate_previous_configs @@ -118,6 +129,12 @@ async def skip_backward_compatibility_tests_fixture( pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") return False + @pytest.fixture(name="skip_oauth_default_method_test") + def skip_oauth_default_method_test_fixture(self, inputs: SpecTestConfig): + if inputs.auth_default_method and not inputs.auth_default_method.oauth: + pytest.skip(f"Skipping OAuth is default method test: {inputs.auth_default_method.bypass_reason}") + return False + def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict): """Check that config matches the actual schema from the spec call""" # Getting rid of technical variables that start with an underscore @@ -519,6 +536,31 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati diff = paths_to_validate - set(get_expected_schema_structure(spec_schema)) assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" + def test_oauth_is_default_method(self, skip_oauth_default_method_test: bool, actual_connector_spec: ConnectorSpecification): + """ + OAuth is default check. + If credentials do have oneOf: we check that the OAuth is listed at first. + If there is no oneOf and Oauth: OAuth is only option to authenticate the source and no check is needed. + """ + advanced_auth = actual_connector_spec.advanced_auth + if not advanced_auth: + pytest.skip("Source does not have OAuth method.") + if not advanced_auth.predicate_key: + pytest.skip("Advanced Auth object does not have predicate_key, only one option to authenticate.") + + spec_schema = actual_connector_spec.connectionSpecification + credentials = advanced_auth.predicate_key[0] + try: + one_of_default_method = dpath.util.get(spec_schema, f"/**/{credentials}/oneOf/0") + except KeyError as e: # Key Error when oneOf is not in credentials object + pytest.skip("Credentials object does not have oneOf option.") + + path_in_credentials = "/".join(advanced_auth.predicate_key[1:]) + auth_method_predicate_const = dpath.util.get(one_of_default_method, f"/**/{path_in_credentials}/const") + assert ( + auth_method_predicate_const == advanced_auth.predicate_value + ), f"Oauth method should be a default option. Current default method is {auth_method_predicate_const}." + @pytest.mark.default_timeout(ONE_MINUTE) @pytest.mark.backward_compatibility def test_backward_compatibility( @@ -939,6 +981,14 @@ def should_validate_schema_fixture(self, inputs: BasicReadTestConfig, test_stric else: return inputs.validate_schema + @pytest.fixture(name="should_validate_stream_statuses") + def should_validate_stream_statuses_fixture(self, inputs: BasicReadTestConfig, is_connector_certified: bool): + if inputs.validate_stream_statuses is None and is_connector_certified: + return True + if not inputs.validate_stream_statuses and is_connector_certified: + pytest.fail("High strictness level error: validate_stream_statuses must be set to true in the basic read test configuration.") + return inputs.validate_stream_statuses + @pytest.fixture(name="should_fail_on_extra_columns") def should_fail_on_extra_columns_fixture(self, inputs: BasicReadTestConfig): # TODO (Ella): enforce this param once all connectors are passing @@ -981,23 +1031,31 @@ def configured_catalog_fixture( else: return build_configured_catalog_from_custom_catalog(configured_catalog_path, discovered_catalog) + _file_types: Set[str] = set() + async def test_read( self, - connector_config, - configured_catalog, + connector_config: SecretDict, + configured_catalog: ConfiguredAirbyteCatalog, expect_records_config: ExpectedRecordsConfig, should_validate_schema: Boolean, should_validate_data_points: Boolean, + should_validate_stream_statuses: Boolean, should_fail_on_extra_columns: Boolean, empty_streams: Set[EmptyStreamConfiguration], ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]], expected_records_by_stream: MutableMapping[str, List[MutableMapping]], docker_runner: ConnectorRunner, - detailed_logger, + detailed_logger: Logger, + certified_file_based_connector: bool, ): output = await docker_runner.call_read(connector_config, configured_catalog) + records = [message.record for message in filter_output(output, Type.RECORD)] + if certified_file_based_connector: + self._file_types.update(self._get_actual_file_types(records)) + assert records, "At least one record should be read using provided catalog" if should_validate_schema: @@ -1025,6 +1083,14 @@ async def test_read( detailed_logger=detailed_logger, ) + if should_validate_stream_statuses: + all_statuses = [ + message.trace.stream_status + for message in filter_output(output, Type.TRACE) + if message.trace.type == TraceType.STREAM_STATUS + ] + self._validate_stream_statuses(configured_catalog=configured_catalog, statuses=all_statuses) + async def test_airbyte_trace_message_on_failure(self, connector_config, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): if not inputs.expect_trace_message_on_failure: pytest.skip("Skipping `test_airbyte_trace_message_on_failure` because `inputs.expect_trace_message_on_failure=False`") @@ -1141,3 +1207,365 @@ def group_by_stream(records: List[AirbyteRecordMessage]) -> MutableMapping[str, result[record.stream].append(record.data) return result + + @pytest.fixture(name="certified_file_based_connector") + def is_certified_file_based_connector(self, connector_metadata: Dict[str, Any], is_connector_certified: bool) -> bool: + metadata = connector_metadata.get("data", {}) + + # connector subtype is specified in data.connectorSubtype field + file_based_connector = metadata.get("connectorSubtype") == "file" + + return file_based_connector and is_connector_certified + + @staticmethod + def _get_file_extension(file_name: str) -> str: + _, file_extension = splitext(file_name) + return file_extension.casefold() + + def _get_actual_file_types(self, records: List[AirbyteRecordMessage]) -> Set[str]: + return {self._get_file_extension(record.data.get("_ab_source_file_url", "")) for record in records} + + @staticmethod + def _get_unsupported_file_types(config: List[UnsupportedFileTypeConfig]) -> Set[str]: + return {t.extension.casefold() for t in config} + + async def test_all_supported_file_types_present(self, certified_file_based_connector: bool, inputs: BasicReadTestConfig): + if not certified_file_based_connector or inputs.file_types.skip_test: + reason = ( + "Skipping the test for supported file types" + f"{' as it is only applicable for certified file-based connectors' if not certified_file_based_connector else ''}." + ) + pytest.skip(reason) + + structured_types = {".avro", ".csv", ".jsonl", ".parquet"} + unstructured_types = {".pdf", ".doc", ".docx", ".ppt", ".pptx", ".md"} + + if inputs.file_types.unsupported_types: + unsupported_file_types = self._get_unsupported_file_types(inputs.file_types.unsupported_types) + structured_types.difference_update(unsupported_file_types) + unstructured_types.difference_update(unsupported_file_types) + + missing_structured_types = structured_types - self._file_types + missing_unstructured_types = unstructured_types - self._file_types + + # all structured and at least one of unstructured supported file types should be present + assert not missing_structured_types and len(missing_unstructured_types) != len(unstructured_types), ( + f"Please make sure you added files with the following supported structured types {missing_structured_types} " + f"and at least one with unstructured type {unstructured_types} to the test account " + "or add them to the `file_types -> unsupported_types` list in config." + ) + + @staticmethod + def _validate_stream_statuses(configured_catalog: ConfiguredAirbyteCatalog, statuses: List[AirbyteStreamStatusTraceMessage]): + """Validate all statuses for all streams in the catalogs were emitted in correct order: + 1. STARTED + 2. RUNNING (can be >1) + 3. COMPLETE + """ + stream_statuses = defaultdict(list) + for status in statuses: + stream_statuses[f"{status.stream_descriptor.namespace}-{status.stream_descriptor.name}"].append(status.status) + + assert set(f"{x.stream.namespace}-{x.stream.name}" for x in configured_catalog.streams) == set( + stream_statuses + ), "All stream must emit status" + + for stream_name, status_list in stream_statuses.items(): + assert ( + len(status_list) >= 3 + ), f"Stream `{stream_name}` statuses should be emitted in the next order: `STARTED`, `RUNNING`,... `COMPLETE`" + assert status_list[0] == AirbyteStreamStatus.STARTED + assert status_list[-1] == AirbyteStreamStatus.COMPLETE + assert all(x == AirbyteStreamStatus.RUNNING for x in status_list[1:-1]) + + +@pytest.mark.default_timeout(TEN_MINUTES) +class TestConnectorAttributes(BaseTest): + # Overide from BaseTest! + # Used so that this is not part of the mandatory high strictness test suite yet + MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [] + + @pytest.fixture(name="operational_certification_test") + async def operational_certification_test_fixture(self, is_connector_certified: bool) -> bool: + """ + Fixture that is used to skip a test that is reserved only for connectors that are supposed to be tested + against operational certification criteria + """ + + if not is_connector_certified: + pytest.skip("Skipping operational connector certification test for uncertified connector") + return True + + @pytest.fixture(name="streams_without_primary_key") + def streams_without_primary_key_fixture(self, inputs: ConnectorAttributesConfig) -> List[NoPrimaryKeyConfiguration]: + return inputs.streams_without_primary_key or [] + + async def test_streams_define_primary_key( + self, operational_certification_test, streams_without_primary_key, connector_config, docker_runner: ConnectorRunner + ) -> None: + output = await docker_runner.call_discover(config=connector_config) + catalog_messages = filter_output(output, Type.CATALOG) + streams = catalog_messages[0].catalog.streams + discovered_streams_without_primary_key = {stream.name for stream in streams if not stream.source_defined_primary_key} + missing_primary_keys = discovered_streams_without_primary_key - {stream.name for stream in streams_without_primary_key} + + quoted_missing_primary_keys = {f"'{primary_key}'" for primary_key in missing_primary_keys} + assert not missing_primary_keys, f"The following streams {', '.join(quoted_missing_primary_keys)} do not define a primary_key" + + @pytest.fixture(name="allowed_hosts_test") + def allowed_hosts_fixture_test(self, inputs: ConnectorAttributesConfig) -> bool: + allowed_hosts = inputs.allowed_hosts + bypass_reason = allowed_hosts.bypass_reason if allowed_hosts else None + if bypass_reason: + pytest.skip(f"Skipping `metadata.allowedHosts` checks. Reason: {bypass_reason}") + return True + + async def test_certified_connector_has_allowed_hosts( + self, operational_certification_test, allowed_hosts_test, connector_metadata: dict + ) -> None: + """ + Checks whether or not the connector has `allowedHosts` and it's components defined in `metadata.yaml`. + Suitable for certified connectors starting `ql` >= 400. + + Arguments: + :: operational_certification_test -- pytest.fixure defines the connector is suitable for this test or not. + :: connector_metadata -- `metadata.yaml` file content + """ + metadata = connector_metadata.get("data", {}) + + has_allowed_hosts_property = "allowedHosts" in metadata.keys() + assert has_allowed_hosts_property, f"The `allowedHosts` property is missing in `metadata.data` for `metadata.yaml`." + + allowed_hosts = metadata.get("allowedHosts", {}) + has_hosts_property = "hosts" in allowed_hosts.keys() if allowed_hosts else False + assert has_hosts_property, f"The `hosts` property is missing in `metadata.data.allowedHosts` for `metadata.yaml`." + + hosts = allowed_hosts.get("hosts", []) + has_assigned_hosts = len(hosts) > 0 if hosts else False + assert ( + has_assigned_hosts + ), f"The `hosts` empty list is not allowed for `metadata.data.allowedHosts` for certified connectors. Please add `hosts` or define the `allowed_hosts.bypass_reason` in `acceptance-test-config.yaml`." + + @pytest.fixture(name="suggested_streams_test") + def suggested_streams_fixture_test(self, inputs: ConnectorAttributesConfig) -> bool: + suggested_streams = inputs.suggested_streams + bypass_reason = suggested_streams.bypass_reason if suggested_streams else None + if bypass_reason: + pytest.skip(f"Skipping `metadata.suggestedStreams` checks. Reason: {bypass_reason}") + return True + + async def test_certified_connector_has_suggested_streams( + self, operational_certification_test, suggested_streams_test, connector_metadata: dict + ) -> None: + """ + Checks whether or not the connector has `suggestedStreams` and it's components defined in `metadata.yaml`. + Suitable for certified connectors starting `ql` >= 400. + + Arguments: + :: operational_certification_test -- pytest.fixure defines the connector is suitable for this test or not. + :: connector_metadata -- `metadata.yaml` file content + """ + + metadata = connector_metadata.get("data", {}) + + has_suggested_streams_property = "suggestedStreams" in metadata.keys() + assert has_suggested_streams_property, f"The `suggestedStreams` property is missing in `metadata.data` for `metadata.yaml`." + + suggested_streams = metadata.get("suggestedStreams", {}) + has_streams_property = "streams" in suggested_streams.keys() if suggested_streams else False + assert has_streams_property, f"The `streams` property is missing in `metadata.data.suggestedStreams` for `metadata.yaml`." + + streams = suggested_streams.get("streams", []) + has_assigned_suggested_streams = len(streams) > 0 if streams else False + assert ( + has_assigned_suggested_streams + ), f"The `streams` empty list is not allowed for `metadata.data.suggestedStreams` for certified connectors." + + +class TestConnectorDocumentation(BaseTest): + MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [] # Used so that this is not part of the mandatory high strictness test suite yet + + PREREQUISITES = "Prerequisites" + HEADING = "heading" + CREDENTIALS_KEYWORDS = ["account", "auth", "credentials", "access"] + CONNECTOR_SPECIFIC_HEADINGS = "" + + @pytest.fixture(name="operational_certification_test") + async def operational_certification_test_fixture(self, is_connector_certified: bool) -> bool: + """ + Fixture that is used to skip a test that is reserved only for connectors that are supposed to be tested + against operational certification criteria + """ + if not is_connector_certified: + pytest.skip("Skipping testing source connector documentation due to low ql.") + return True + + def _get_template_headings(self, connector_name: str) -> tuple[tuple[str], tuple[str]]: + """ + https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw - standard template + Headings in order to docs structure. + """ + all_headings = ( + connector_name, + "Prerequisites", + "Setup guide", + f"Set up {connector_name}", + "For Airbyte Cloud:", + "For Airbyte Open Source:", + f"Set up the {connector_name} connector in Airbyte", + "For Airbyte Cloud:", + "For Airbyte Open Source:", + "Supported sync modes", + "Supported Streams", + self.CONNECTOR_SPECIFIC_HEADINGS, + "Performance considerations", + "Data type map", + "Troubleshooting", + "Tutorials", + "Changelog", + ) + not_required_heading = ( + f"Set up the {connector_name} connector in Airbyte", + "For Airbyte Cloud:", + "For Airbyte Open Source:", + self.CONNECTOR_SPECIFIC_HEADINGS, + "Performance considerations", + "Data type map", + "Troubleshooting", + "Tutorials", + ) + return all_headings, not_required_heading + + def _headings_description(self, connector_name: str) -> dict[str:Path]: + """ + Headings with path to file with template description + """ + descriptions_paths = { + connector_name: Path(__file__).parent / "doc_templates/source.txt", + "For Airbyte Cloud:": Path(__file__).parent / "doc_templates/for_airbyte_cloud.txt", + "For Airbyte Open Source:": Path(__file__).parent / "doc_templates/for_airbyte_open_source.txt", + "Supported sync modes": Path(__file__).parent / "doc_templates/supported_sync_modes.txt", + "Tutorials": Path(__file__).parent / "doc_templates/tutorials.txt", + } + return descriptions_paths + + def test_prerequisites_content( + self, operational_certification_test, actual_connector_spec: ConnectorSpecification, connector_documentation: str, docs_path: str + ): + node = docs_utils.documentation_node(connector_documentation) + header_line_map = {docs_utils.header_name(n): n.map[1] for n in node if n.type == self.HEADING} + headings = tuple(header_line_map.keys()) + + if not header_line_map.get(self.PREREQUISITES): + pytest.fail(f"Documentation does not have {self.PREREQUISITES} section.") + + prereq_start_line = header_line_map[self.PREREQUISITES] + prereq_end_line = docs_utils.description_end_line_index(self.PREREQUISITES, headings, header_line_map) + + with open(docs_path, "r") as docs_file: + prereq_content_lines = docs_file.readlines()[prereq_start_line:prereq_end_line] + # adding real character to avoid accidentally joining lines into a wanted title. + prereq_content = "|".join(prereq_content_lines).lower() + required_titles, has_credentials = docs_utils.required_titles_from_spec(actual_connector_spec.connectionSpecification) + + for title in required_titles: + assert title in prereq_content, ( + f"Required '{title}' field is not in {self.PREREQUISITES} section " f"or title in spec doesn't match name in the docs." + ) + + if has_credentials: + # credentials has specific check for keywords as we have a lot of way how to describe this step + credentials_validation = [k in prereq_content for k in self.CREDENTIALS_KEYWORDS] + assert True in credentials_validation, f"Required 'credentials' field is not in {self.PREREQUISITES} section." + + def test_docs_structure(self, operational_certification_test, connector_documentation: str, connector_metadata: dict): + """ + test_docs_structure gets all top-level headers from source documentation file and check that the order is correct. + The order of the headers should follow our standard template https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw. + _get_template_headings returns tuple of headers as in standard template and non-required headers that might nor be in the source docs. + CONNECTOR_SPECIFIC_HEADINGS value in list of required headers that shows a place where should be a connector specific headers, + which can be skipped as out of standard template and depend of connector. + """ + + heading_names = docs_utils.prepare_headers(connector_documentation) + template_headings, non_required_heading = self._get_template_headings(connector_metadata["data"]["name"]) + + heading_names_len, template_headings_len = len(heading_names), len(template_headings) + heading_names_index, template_headings_index = 0, 0 + + while heading_names_index < heading_names_len and template_headings_index < template_headings_len: + heading_names_value = heading_names[heading_names_index] + template_headings_value = template_headings[template_headings_index] + # check that template header is specific for connector and actual header should not be validated + if template_headings_value == self.CONNECTOR_SPECIFIC_HEADINGS: + # check that actual header is not in required headers, as required headers should be on a right place and order + if heading_names_value not in template_headings: + heading_names_index += 1 # go to the next actual header as CONNECTOR_SPECIFIC_HEADINGS can be more than one + continue + else: + # if actual header is required go to the next template header to validate actual header order + template_headings_index += 1 + continue + # strict check that actual header equals template header + if heading_names_value == template_headings_value: + # found expected header, go to the next header in template and actual headers + heading_names_index += 1 + template_headings_index += 1 + continue + # actual header != template header means that template value is not required and can be skipped + if template_headings_value in non_required_heading: + # found non-required header, go to the next template header to validate actual header + template_headings_index += 1 + continue + # any check is True, indexes didn't move to the next step + pytest.fail(docs_utils.reason_titles_not_match(heading_names_value, template_headings_value, template_headings)) + # indexes didn't move to the last required one, so some headers are missed + if template_headings_index != template_headings_len: + pytest.fail(docs_utils.reason_missing_titles(template_headings_index, template_headings)) + + def test_docs_descriptions( + self, operational_certification_test, docs_path: str, connector_documentation: str, connector_metadata: dict + ): + connector_name = connector_metadata["data"]["name"] + template_descriptions = self._headings_description(connector_name) + + node = docs_utils.documentation_node(connector_documentation) + header_line_map = {docs_utils.header_name(n): n.map[1] for n in node if n.type == self.HEADING} + actual_headings = tuple(header_line_map.keys()) + + for heading, description in template_descriptions.items(): + if heading in actual_headings: + + description_start_line = header_line_map[heading] + description_end_line = docs_utils.description_end_line_index(heading, actual_headings, header_line_map) + + with open(docs_path, "r") as docs_file, open(description, "r") as template_file: + + docs_description_content = docs_file.readlines()[description_start_line:description_end_line] + template_description_content = template_file.readlines() + + for d, t in zip(docs_description_content, template_description_content): + d, t = docs_utils.prepare_lines_to_compare(connector_name, d, t) + assert d == t, f"Description for '{heading}' does not follow structure.\nExpected: {t} Actual: {d}" + + def test_validate_links(self, operational_certification_test, connector_documentation: str): + valid_status_codes = [200, 403, 401, 405] # we skip 4xx due to needed access + links = re.findall("(https?://[^\s)]+)", connector_documentation) + invalid_links = [] + threads = [] + + def validate_docs_links(docs_link): + response = requests.get(docs_link) + if response.status_code not in valid_status_codes: + invalid_links.append(docs_link) + + for link in links: + process = Thread(target=validate_docs_links, args=[link]) + process.start() + threads.append(process) + + for process in threads: + process.join(timeout=30) # 30s timeout for process else link will be skipped + process.is_alive() + + assert not invalid_links, f"{len(invalid_links)} invalid links were found in the connector documentation: {invalid_links}." diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py index 89680d1745da..8362aac10053 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py @@ -8,7 +8,6 @@ from collections import defaultdict from typing import Any, Dict, List, Mapping -import dpath.util import pendulum from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog from jsonschema import Draft7Validator, FormatChecker, FormatError, ValidationError, validators @@ -26,6 +25,40 @@ Draft7ValidatorWithStrictInteger = validators.extend(Draft7Validator, type_checker=strict_integer_type_checker) +class NoAdditionalPropertiesValidator(Draft7Validator): + def __init__(self, schema, **kwargs): + schema = self._enforce_false_additional_properties(schema) + super().__init__(schema, **kwargs) + + @staticmethod + def _enforce_false_additional_properties(json_schema: Dict[str, Any]) -> Dict[str, Any]: + """Create a copy of the schema in which `additionalProperties` is set to False for all non-null object properties. + + This method will override the value of `additionalProperties` if it is set, + or will create the property and set it to False if it does not exist. + """ + new_schema = copy.deepcopy(json_schema) + new_schema["additionalProperties"] = False + + def add_properties(properties): + for prop_name, prop_value in properties.items(): + if "type" in prop_value and "object" in prop_value["type"] and len(prop_value.get("properties", [])): + prop_value["additionalProperties"] = False + add_properties(prop_value.get("properties", {})) + elif "type" in prop_value and "array" in prop_value["type"]: + if ( + prop_value.get("items") + and "object" in prop_value.get("items", {}).get("type", []) + and len(prop_value.get("items", {}).get("properties", [])) + ): + prop_value["items"]["additionalProperties"] = False + if prop_value.get("items", {}).get("properties"): + add_properties(prop_value["items"]["properties"]) + + add_properties(new_schema.get("properties", {})) + return new_schema + + class CustomFormatChecker(FormatChecker): @staticmethod def check_datetime(value: str) -> bool: @@ -46,17 +79,6 @@ def check(self, instance, format): return super().check(instance, format) -def _enforce_no_additional_top_level_properties(json_schema: Dict[str, Any]): - """Create a copy of the schema in which `additionalProperties` is set to False for the dict of top-level properties. - - This method will override the value of `additionalProperties` if it is set, - or will create the property and set it to False if it does not exist. - """ - enforced_schema = copy.deepcopy(json_schema) - dpath.util.new(enforced_schema, "additionalProperties", False) - return enforced_schema - - def verify_records_schema( records: List[AirbyteRecordMessage], catalog: ConfiguredAirbyteCatalog, fail_on_extra_columns: bool ) -> Mapping[str, Mapping[str, ValidationError]]: @@ -66,11 +88,8 @@ def verify_records_schema( stream_validators = {} for stream in catalog.streams: schema_to_validate_against = stream.stream.json_schema - if fail_on_extra_columns: - schema_to_validate_against = _enforce_no_additional_top_level_properties(schema_to_validate_against) - stream_validators[stream.stream.name] = Draft7ValidatorWithStrictInteger( - schema_to_validate_against, format_checker=CustomFormatChecker() - ) + validator = NoAdditionalPropertiesValidator if fail_on_extra_columns else Draft7ValidatorWithStrictInteger + stream_validators[stream.stream.name] = validator(schema_to_validate_against, format_checker=CustomFormatChecker()) stream_errors = defaultdict(dict) for record in records: validator = stream_validators.get(record.stream) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index af84c5f1eafa..0bf6810ea0ea 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -28,7 +28,7 @@ async def get_container_from_id(dagger_client: dagger.Client, container_id: str) dagger_client (dagger.Client): The dagger client to use to import the connector image """ try: - return await dagger_client.container(dagger.ContainerID(container_id)) + return await dagger_client.container(id=dagger.ContainerID(container_id)) except dagger.DaggerError as e: pytest.exit(f"Failed to load connector container: {e}") @@ -244,11 +244,11 @@ async def _run( if not enable_caching: container = container.with_env_variable("CAT_CACHEBUSTER", str(uuid.uuid4())) if config: - container = container.with_new_file(self.IN_CONTAINER_CONFIG_PATH, json.dumps(dict(config))) + container = container.with_new_file(self.IN_CONTAINER_CONFIG_PATH, contents=json.dumps(dict(config))) if state: - container = container.with_new_file(self.IN_CONTAINER_STATE_PATH, json.dumps(state)) + container = container.with_new_file(self.IN_CONTAINER_STATE_PATH, contents=json.dumps(state)) if catalog: - container = container.with_new_file(self.IN_CONTAINER_CATALOG_PATH, catalog.json()) + container = container.with_new_file(self.IN_CONTAINER_CATALOG_PATH, contents=catalog.json()) try: output = await self._read_output_from_stdout(airbyte_command, container) except dagger.QueryError as e: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/docs.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/docs.py new file mode 100644 index 000000000000..2b29a65ce5e4 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/docs.py @@ -0,0 +1,133 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import re +from difflib import get_close_matches +from typing import Any + +from markdown_it import MarkdownIt +from markdown_it.tree import SyntaxTreeNode + + +def remove_step_from_heading(heading: str) -> str: + if "Step 1: " in heading: + return heading.replace("Step 1: ", "") + if "Step 2: " in heading: + return heading.replace("Step 2: ", "") + return heading + + +def required_titles_from_spec(spec: dict[str, Any]) -> tuple[list[str], bool]: + has_credentials = False + spec_required = spec["required"] + spec_properties = spec["properties"].keys() + creds = ["credentials", "client_id", "client_secret", "access_token", "refresh_token"] + + if "credentials" in spec["required"] or "client_id" in spec["required"] or "client_secret" in spec_required: + has_credentials = True + if "credentials" in spec["required"] or "client_id" in spec["required"] or "client_secret" in spec_properties: + has_credentials = True + if has_credentials: + [spec_required.remove(cred) for cred in creds if cred in spec_required] + + titles = [spec["properties"][field]["title"].lower() for field in spec_required] + return titles, has_credentials + + +def documentation_node(connector_documentation: str) -> SyntaxTreeNode: + md = MarkdownIt("commonmark") + tokens = md.parse(connector_documentation) + return SyntaxTreeNode(tokens) + + +def header_name(n: SyntaxTreeNode) -> str: + return n.to_tokens()[1].children[0].content + + +def prepare_lines_to_compare(connector_name: str, docs_line: str, template_line: str) -> tuple[str]: + def _replace_link(docs_string: str, link_to_replace: str) -> str: + try: + docs_string = docs_string[: docs_string.index("(")] + link_to_replace + docs_string[docs_string.index(")") + 1 :] + return docs_string + except ValueError: # ValueError if actual docs doesn't have expected links + return docs_string + + connector_name_to_replace = "{connector_name}" + link_to_replace = "({docs_link})" + + template_line = ( + template_line.replace(connector_name_to_replace, connector_name) if connector_name_to_replace in template_line else template_line + ) + docs_line = _replace_link(docs_line, link_to_replace) if link_to_replace in template_line else docs_line + + return docs_line, template_line + + +def remove_not_required_step_headers(headers: tuple[str]) -> tuple[str]: + """ + Removes headers like Step 1.1 Step 3 Step 2.3 from actual headers, if they placed after Step 1: header. + from: "Connector name", "Prerequisites", "Setup guide", "Step 1: do something 1", "Step 1.11: do something 11", + "Step 2: do something 2", "Step 2.1: do something 2.1", "Changelog" + To: "Connector name", "Prerequisites", "Setup guide", "Step 1: do something 1", "Step 2: do something 2", "Changelog" + This is connector specific headers, so we can ignore them. + """ + step_one_index = None + for header in headers: + if re.search("Step 1: ", header): + step_one_index = headers.index(header) + if not step_one_index: # docs doesn't have Step 1 headers + return headers + + step_headers = headers[step_one_index:] + pattern = "Step \d+.?\d*: " + step = "Step 1: " + i = 0 + while i < len(step_headers): + if step in step_headers[i]: # if Step 1/2: is substring of current header + if i + 1 < len(step_headers) and re.match(pattern, step_headers[i + 1]): # check that header has Step x: + if "Step 2: " in step_headers[i + 1]: # found Step 2, it's required header, move to the next one + step = "Step 2: " + i += 1 + continue + else: + step_headers.remove(step_headers[i + 1]) # remove all other steps from headers + continue # move to the next header after Step 1/2 header + else: + break + break + + headers = headers[:step_one_index] + headers.extend(step_headers) + return headers + + +def reason_titles_not_match(heading_names_value: str, template_headings_value: str, template_headings: list[str]) -> str: + reason = ( + f"Documentation structure doesn't follow standard template. Heading '{heading_names_value}' is not in the right place, " + f"the name of heading is incorrect or the heading name is not expected.\n" + ) + close_titles = get_close_matches(heading_names_value, template_headings) + if close_titles and close_titles[0] != heading_names_value: + diff = f"Diff:\nActual Heading: '{heading_names_value}'. Possible correct heading: '{close_titles}'. Expected Heading: '{template_headings_value}'." + else: + diff = f"Diff:\nActual Heading: '{heading_names_value}'. Expected Heading: '{template_headings_value}'" + return reason + diff + + +def reason_missing_titles(template_headings_index: int, template_headings: list[str]) -> str: + return ( + f"Documentation structure doesn't follow standard template. docs is not full." + f"\nMissing headers: {template_headings[template_headings_index:]}" + ) + + +def description_end_line_index(heading: str, actual_headings: list[str], header_line_map: dict[str, int]) -> int: + if actual_headings.index(heading) + 1 == len(actual_headings): + return -1 + return header_line_map[actual_headings[actual_headings.index(heading) + 1]] + + +def prepare_headers(connector_documentation: dict) -> list[str]: + node = documentation_node(connector_documentation) + headers = [header_name(n) for n in node if n.type == "heading"] # find all headers + headers = remove_not_required_step_headers(headers) # remove Step 1.1 Step 3 ... headers + headers = tuple([remove_step_from_heading(h) for h in headers]) # remove Step 1 and Step 2 from header name + return headers diff --git a/airbyte-integrations/bases/connector-acceptance-test/poetry.lock b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock index 703b4b11f984..fff2f1561b59 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/poetry.lock +++ b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-protocol-models" -version = "0.4.1" +version = "0.5.3" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.4.1-py3-none-any.whl", hash = "sha256:95f1197c800d7867ba067f75770b83aeff4c2cec9b3d1def2dbf70261fee89ee"}, - {file = "airbyte_protocol_models-0.4.1.tar.gz", hash = "sha256:92602134eab4c921d1328fa4f24e9a810a679c117ccb352cf6b1521f95f0ed53"}, + {file = "airbyte_protocol_models-0.5.3-py3-none-any.whl", hash = "sha256:a913f1e86d5b2ae17d19e0135339e55fc25bb93bfc3f7ab38592677f29b56c57"}, + {file = "airbyte_protocol_models-0.5.3.tar.gz", hash = "sha256:a71bc0e98e0722d5cbd3122c40a59a7f9cbc91b6c934db7e768a57c40546f54b"}, ] [package.dependencies] @@ -47,21 +47,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -76,13 +77,13 @@ files = [ [[package]] name = "beartype" -version = "0.15.0" +version = "0.16.4" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.15.0-py3-none-any.whl", hash = "sha256:52cd2edea72fdd84e4e7f8011a9e3007bf0125c3d6d7219e937b9d8868169177"}, - {file = "beartype-0.15.0.tar.gz", hash = "sha256:2af6a8d8a7267ccf7d271e1a3bd908afbc025d2a09aa51123567d7d7b37438df"}, + {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, + {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, ] [package.extras] @@ -94,122 +95,137 @@ test-tox-coverage = ["coverage (>=5.5)"] [[package]] name = "cattrs" -version = "23.1.2" +version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, - {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, ] [package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] -bson = ["pymongo (>=4.2.0,<5.0.0)"] -cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] -msgpack = ["msgpack (>=1.0.2,<2.0.0)"] -orjson = ["orjson (>=3.5.2,<4.0.0)"] -pyyaml = ["PyYAML (>=6.0,<7.0)"] -tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] -ujson = ["ujson (>=5.4.0,<6.0.0)"] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -225,71 +241,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.7" +version = "7.4.0" description = "Code coverage measurement for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, - {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, - {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, - {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, - {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, - {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, - {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, - {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, - {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, - {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, - {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, - {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, - {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, - {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, - {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, - {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, - {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, - {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] [package.dependencies] @@ -300,13 +308,13 @@ toml = ["tomli"] [[package]] name = "dagger-io" -version = "0.6.4" +version = "0.9.6" description = "A client package for running Dagger pipelines in Python." optional = false python-versions = ">=3.10" files = [ - {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, - {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, ] [package.dependencies] @@ -317,11 +325,8 @@ gql = ">=3.4.0" graphql-core = ">=3.2.3" httpx = ">=0.23.1" platformdirs = ">=2.6.2" -typing-extensions = ">=4.4.0" - -[package.extras] -cli = ["typer[all] (>=0.6.1)"] -server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" [[package]] name = "deepdiff" @@ -374,13 +379,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -417,29 +422,31 @@ pyrepl = ">=0.8.2" [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" @@ -465,39 +472,40 @@ files = [ [[package]] name = "httpcore" -version = "0.17.3" +version = "1.0.2" description = "A minimal low-level HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, - {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, ] [package.dependencies] -anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" [package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.24.1" +version = "0.26.0" description = "The next generation HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, - {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [package.dependencies] +anyio = "*" certifi = "*" -httpcore = ">=0.15.0,<0.18.0" +httpcore = "==1.*" idna = "*" sniffio = "*" @@ -509,21 +517,22 @@ socks = ["socksio (==1.*)"] [[package]] name = "hypothesis" -version = "6.82.3" +version = "6.96.0" description = "A library for property-based testing" optional = false python-versions = ">=3.8" files = [ - {file = "hypothesis-6.82.3-py3-none-any.whl", hash = "sha256:7ff0f6a12d3cd9372e30f84d300e2468c3923e813198a93b9e479dda91858460"}, + {file = "hypothesis-6.96.0-py3-none-any.whl", hash = "sha256:ec8e0348844e1a9368aeaf85dbea1d247f93f5f865fdf65801bc578b4608cc08"}, + {file = "hypothesis-6.96.0.tar.gz", hash = "sha256:fec50dcbc54ec5884a4199d723543ba9408bbab940cc3ab849a92fe1fab97625"}, ] [package.dependencies] -attrs = ">=19.2.0" +attrs = ">=22.2.0" exceptiongroup = {version = ">=1.0.0", markers = "python_version < \"3.11\""} sortedcontainers = ">=2.1.0,<3.0.0" [package.extras] -all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.3)"] +all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.4)"] cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] codemods = ["libcst (>=0.3.16)"] dateutil = ["python-dateutil (>=1.4)"] @@ -536,22 +545,22 @@ pandas = ["pandas (>=1.1)"] pytest = ["pytest (>=4.6)"] pytz = ["pytz (>=2014.1)"] redis = ["redis (>=3.0.0)"] -zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] +zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.4)"] [[package]] name = "hypothesis-jsonschema" -version = "0.22.1" +version = "0.23.0" description = "Generate test data from JSON schemata with Hypothesis" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "hypothesis-jsonschema-0.22.1.tar.gz", hash = "sha256:5dd7449009f323e408a9aa64afb4d18bd1f60ea2eabf5bf152a510da728b34f2"}, - {file = "hypothesis_jsonschema-0.22.1-py3-none-any.whl", hash = "sha256:082968cb86a6aac2369627b08753cbf714c08054b1ebfce3588e3756e652cde6"}, + {file = "hypothesis-jsonschema-0.23.0.tar.gz", hash = "sha256:c3cc5ecddd78efcb5c10cc3fbcf06aa4d32d8300d0babb8c6f89485f7a503aef"}, + {file = "hypothesis_jsonschema-0.23.0-py3-none-any.whl", hash = "sha256:bbf13b49970216b69adfeab666e483bd83691573d9fee55f3c69adeefa978a09"}, ] [package.dependencies] -hypothesis = ">=6.31.6" -jsonschema = ">=4.0.0" +hypothesis = ">=6.84.3" +jsonschema = ">=4.18.0" [[package]] name = "icdiff" @@ -565,13 +574,13 @@ files = [ [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -609,13 +618,13 @@ files = [ [[package]] name = "jsonschema" -version = "4.19.0" +version = "4.21.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema-4.19.0-py3-none-any.whl", hash = "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb"}, - {file = "jsonschema-4.19.0.tar.gz", hash = "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"}, + {file = "jsonschema-4.21.0-py3-none-any.whl", hash = "sha256:70a09719d375c0a2874571b363c8a24be7df8071b80c9aa76bc4551e7297c63c"}, + {file = "jsonschema-4.21.0.tar.gz", hash = "sha256:3ba18e27f7491ea4a1b22edce00fb820eec968d397feb3f9cb61d5894bb38167"}, ] [package.dependencies] @@ -630,17 +639,52 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- [[package]] name = "jsonschema-specifications" -version = "2023.7.1" +version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, - {file = "jsonschema_specifications-2023.7.1.tar.gz", hash = "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"}, + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, ] [package.dependencies] -referencing = ">=0.28.0" +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] [[package]] name = "multidict" @@ -741,13 +785,13 @@ dev = ["black", "mypy", "pytest"] [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -772,47 +816,112 @@ testing = ["funcsigs", "pytest"] [[package]] name = "pendulum" -version = "2.1.2" +version = "3.0.0" description = "Python datetimes made easy" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.8" files = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, ] [package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] [[package]] name = "platformdirs" -version = "3.10.0" +version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, ] [package.extras] @@ -821,13 +930,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -858,50 +967,51 @@ files = [ [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false -python-versions = ">=3.6.1" -files = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -909,17 +1019,18 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyreadline" @@ -1035,13 +1146,13 @@ pytest = ">=3.6.0" [[package]] name = "pytest-xdist" -version = "3.3.1" +version = "3.5.0" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-xdist-3.3.1.tar.gz", hash = "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93"}, - {file = "pytest_xdist-3.3.1-py3-none-any.whl", hash = "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2"}, + {file = "pytest-xdist-3.5.0.tar.gz", hash = "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a"}, + {file = "pytest_xdist-3.5.0-py3-none-any.whl", hash = "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"}, ] [package.dependencies] @@ -1067,17 +1178,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, -] - [[package]] name = "pywin32" version = "306" @@ -1162,13 +1262,13 @@ files = [ [[package]] name = "referencing" -version = "0.30.2" +version = "0.32.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" files = [ - {file = "referencing-0.30.2-py3-none-any.whl", hash = "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf"}, - {file = "referencing-0.30.2.tar.gz", hash = "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"}, + {file = "referencing-0.32.1-py3-none-any.whl", hash = "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"}, + {file = "referencing-0.32.1.tar.gz", hash = "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3"}, ] [package.dependencies] @@ -1215,110 +1315,130 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] +[[package]] +name = "rich" +version = "13.7.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "rpds-py" -version = "0.9.2" +version = "0.17.1" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" files = [ - {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, - {file = "rpds_py-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6"}, - {file = "rpds_py-0.9.2-cp310-none-win32.whl", hash = "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764"}, - {file = "rpds_py-0.9.2-cp310-none-win_amd64.whl", hash = "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de"}, - {file = "rpds_py-0.9.2-cp311-none-win32.whl", hash = "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab"}, - {file = "rpds_py-0.9.2-cp311-none-win_amd64.whl", hash = "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e"}, - {file = "rpds_py-0.9.2-cp38-none-win32.whl", hash = "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920"}, - {file = "rpds_py-0.9.2-cp38-none-win_amd64.whl", hash = "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26"}, - {file = "rpds_py-0.9.2-cp39-none-win32.whl", hash = "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0"}, - {file = "rpds_py-0.9.2-cp39-none-win_amd64.whl", hash = "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe"}, - {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59"}, + {file = "rpds_py-0.17.1-cp310-none-win32.whl", hash = "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d"}, + {file = "rpds_py-0.17.1-cp310-none-win_amd64.whl", hash = "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea"}, + {file = "rpds_py-0.17.1-cp311-none-win32.whl", hash = "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518"}, + {file = "rpds_py-0.17.1-cp311-none-win_amd64.whl", hash = "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23"}, + {file = "rpds_py-0.17.1-cp312-none-win32.whl", hash = "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1"}, + {file = "rpds_py-0.17.1-cp312-none-win_amd64.whl", hash = "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6"}, + {file = "rpds_py-0.17.1-cp38-none-win32.whl", hash = "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a"}, + {file = "rpds_py-0.17.1-cp38-none-win_amd64.whl", hash = "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b"}, + {file = "rpds_py-0.17.1-cp39-none-win32.whl", hash = "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f"}, + {file = "rpds_py-0.17.1-cp39-none-win_amd64.whl", hash = "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68"}, + {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, ] [[package]] @@ -1356,13 +1476,13 @@ files = [ [[package]] name = "termcolor" -version = "2.3.0" +version = "2.4.0" description = "ANSI color formatting for output in terminal" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "termcolor-2.3.0-py3-none-any.whl", hash = "sha256:3afb05607b89aed0ffe25202399ee0867ad4d3cb4180d98aaf8eefa6a5f7d475"}, - {file = "termcolor-2.3.0.tar.gz", hash = "sha256:b5b08f68937f138fe92f6c089b99f1e2da0ae56c52b78bf7075fd95420fd9a5a"}, + {file = "termcolor-2.4.0-py3-none-any.whl", hash = "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63"}, + {file = "termcolor-2.4.0.tar.gz", hash = "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"}, ] [package.extras] @@ -1392,138 +1512,172 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "websocket-client" -version = "1.6.1" +version = "1.7.0" description = "WebSocket client for Python with low level API options" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, - {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, ] [package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] [[package]] name = "wmctrl" -version = "0.4" +version = "0.5" description = "A tool to programmatically control windows inside X" optional = false -python-versions = "*" +python-versions = ">=2.7" files = [ - {file = "wmctrl-0.4.tar.gz", hash = "sha256:66cbff72b0ca06a22ec3883ac3a4d7c41078bdae4fb7310f52951769b10e14e0"}, + {file = "wmctrl-0.5-py2.py3-none-any.whl", hash = "sha256:ae695c1863a314c899e7cf113f07c0da02a394b968c4772e1936219d9234ddd7"}, + {file = "wmctrl-0.5.tar.gz", hash = "sha256:7839a36b6fe9e2d6fd22304e5dc372dbced2116ba41283ea938b2da57f53e962"}, ] +[package.dependencies] +attrs = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -1533,4 +1687,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "15d79ace4da317a48b537ebaec3c1120e44e4a236ca6ab7050bc946d3a14a738" +content-hash = "1c468b66c56cfccd5e5bff7d9c69f01c729d828132a8a56a7089447f5da0f534" diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index bb6641847361..ed80ea830fd8 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "2.1.4" +version = "3.5.0" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT" @@ -13,7 +13,7 @@ homepage = "https://github.com/airbytehq/airbyte" [tool.poetry.dependencies] python = "^3.10" airbyte-protocol-models = "<1.0.0" -dagger-io = "==0.6.4" +dagger-io = "==0.9.6" PyYAML = "~=6.0" icdiff = "~=1.9" inflection = "~=0.5" @@ -41,4 +41,10 @@ urllib3 = "<2.0" requests = "<2.29.0" pytest-xdist = "^3.3.1" -[tool.poetry.dev-dependencies] +[tool.poe.tasks] +test = "pytest unit_tests" +ci = ["test"] + +[tool.airbyte_ci] +poe_tasks = ["test"] +mount_docker_socket = true diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct.md new file mode 100644 index 000000000000..6fdf34bba108 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct.md @@ -0,0 +1,305 @@ +# GitHub + + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + + +## Prerequisites + +- List of GitHub Repositories (and access for them in case they are private) + + +**For Airbyte Cloud:** + +- OAuth +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + + +**For Airbyte Open Source:** + +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + +## Setup guide + +### Step 1: Set up GitHub + +Create a [GitHub Account](https://github.com). + + +**Airbyte Open Source additional setup steps** + +Log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +### Step 2: Set up the GitHub connector in Airbyte + + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. +3. On the source selection page, select **GitHub** from the list of Sources. +4. Add a name for your GitHub connector. +5. To authenticate: + + + - **For Airbyte Cloud:** **Authenticate your GitHub account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. + + + + - **For Airbyte Open Source:** Authenticate with **Personal Access Token**. To generate a personal access token, log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). Enter your GitHub personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +6. **GitHub Repositories** - Enter a list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. + +:::caution +Repositories with the wrong name or repositories that do not exist or have the wrong name format will be skipped with `WARN` message in the logs. +::: + +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. For streams which support this configuration, only data generated on or after the start date will be replicated. + +- These streams will only sync records generated on or after the **Start Date**: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. + +- The **Start Date** does not apply to the streams below and all data will be synced for these streams: `assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` + +8. **Branch (Optional)** - List of GitHub repository branches to pull commits from, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). +9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5,000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + + + +## Supported sync modes + +The GitHub source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) + +## Supported Streams + +This connector outputs the following full refresh streams: + +- [Assignees](https://docs.github.com/en/rest/issues/assignees?apiVersion=2022-11-28#list-assignees) +- [Branches](https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches) +- [Contributor Activity](https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity) +- [Collaborators](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators) +- [Issue labels](https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#list-labels-for-a-repository) +- [Organizations](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations) +- [Pull request commits](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-commits-on-a-pull-request) +- [Tags](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-tags) +- [TeamMembers](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#list-team-members) +- [TeamMemberships](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#get-team-membership-for-a-user) +- [Teams](https://docs.github.com/en/rest/teams/teams?apiVersion=2022-11-28#list-teams) +- [Users](https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members) +- [Issue timeline events](https://docs.github.com/en/rest/issues/timeline?apiVersion=2022-11-28#list-timeline-events-for-an-issue) + +This connector outputs the following incremental streams: + +- [Comments](https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments-for-a-repository) +- [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions?apiVersion=2022-11-28#list-reactions-for-a-commit-comment) +- [Commit comments](https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository) +- [Commits](https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits) +- [Deployments](https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments) +- [Events](https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events) +- [Issue comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue-comment) +- [Issue events](https://docs.github.com/en/rest/issues/events?apiVersion=2022-11-28#list-issue-events-for-a-repository) +- [Issue milestones](https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#list-milestones) +- [Issue reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue) +- [Issues](https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues) +- [Project (Classic) cards](https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28#list-project-cards) +- [Project (Classic) columns](https://docs.github.com/en/rest/projects/columns?apiVersion=2022-11-28#list-project-columns) +- [Projects (Classic)](https://docs.github.com/en/rest/projects/projects?apiVersion=2022-11-28#list-repository-projects) +- [ProjectsV2](https://docs.github.com/en/graphql/reference/objects#projectv2) +- [Pull request comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-a-pull-request-review-comment) +- [Pull request stats](https://docs.github.com/en/graphql/reference/objects#pullrequest) +- [Pull requests](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests) +- [Releases](https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases) +- [Repositories](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories) +- [Review comments](https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository) +- [Reviews](https://docs.github.com/en/rest/pulls/reviews?apiVersion=2022-11-28#list-reviews-for-a-pull-request) +- [Stargazers](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) +- [WorkflowJobs](https://docs.github.com/pt/rest/actions/workflow-jobs?apiVersion=2022-11-28#list-jobs-for-a-workflow-run) +- [WorkflowRuns](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository) +- [Workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows) + +### Notes + +1. Only 4 streams \(`comments`, `commits`, `issues` and `review comments`\) from the listed above streams are pure incremental meaning that they: + + - read only new records; + - output only new records. + +2. Streams `workflow_runs` and `worflow_jobs` is almost pure incremental: + + - read new records and some portion of old records (in past 30 days) [docs](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs); + - the `workflow_jobs` depends on the `workflow_runs` to read the data, so they both follow the same logic [docs](https://docs.github.com/pt/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run); + - output only new records. + +3. Other 19 incremental streams are also incremental but with one difference, they: + + - read all records; + - output only new records. + Please, consider this behaviour when using those 19 incremental streams because it may affect you API call limits. + +4. Sometimes for large streams specifying very distant `start_date` in the past may result in keep on getting error from GitHub instead of records \(respective `WARN` log message will be outputted\). In this case Specifying more recent `start_date` may help. + **The "Start date" configuration option does not apply to the streams below, because the GitHub API does not include dates which can be used for filtering:** + +- `assignees` +- `branches` +- `collaborators` +- `issue_labels` +- `organizations` +- `pull_request_commits` +- `pull_request_stats` +- `repositories` +- `tags` +- `teams` +- `users` + +## Limitations & Troubleshooting + +
      + +Expand to see details about GitHub connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +#### Permissions and scopes + +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. + +Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: + +- For syncing Collaborators, the user which generates the personal access token must be a collaborator. To become a collaborator, they must be invited by an owner. If there are no collaborators, no records will be synced. Read more about access permissions [here](https://docs.github.com/en/get-started/learning-about-github/access-permissions-on-github). +- Syncing [Teams](https://docs.github.com/en/organizations/organizing-members-into-teams/about-teams) is only available to authenticated members of a team's [organization](https://docs.github.com/en/rest/orgs). [Personal user accounts](https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts) and repositories belonging to them don't have access to Teams features. In this case no records will be synced. +- To sync the Projects stream, the repository must have the Projects feature enabled. + +### Troubleshooting + +* Check out common troubleshooting issues for the GitHub source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) + +
      + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | +| 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | +| 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response | +| 1.5.1 | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` | +| 1.5.0 | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields | +| 1.4.6 | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \ | +| 1.4.5 | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` | +| 1.4.4 | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. | +| 1.4.3 | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` | +| 1.4.2 | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages | +| 1.4.1 | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version | +| 1.4.0 | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` | +| 1.3.1 | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` | +| 1.3.0 | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` | +| 1.2.1 | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` | +| 1.2.0 | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances | +| 1.1.1 | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages | +| 1.1.0 | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` | +| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter | +| 1.0.1 | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter | +| 1.0.0 | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI | +| 0.5.0 | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests | +| 0.4.11 | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025) | Added more transparent depiction of the personal access token expired | +| 0.4.10 | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075) | Add more specific error message description for no repos case. | +| 0.4.9 | 2023-05-01 | [24523](https://github.com/airbytehq/airbyte/pull/24523) | Add undeclared columns to spec | +| 0.4.8 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/25312) | Fix repo name validation | +| 0.4.7 | 2023-03-24 | [24457](https://github.com/airbytehq/airbyte/pull/24457) | Add validation and transformation for repositories config | +| 0.4.6 | 2023-03-24 | [24398](https://github.com/airbytehq/airbyte/pull/24398) | Fix caching for `get_starting_point` in stream "Commits" | +| 0.4.5 | 2023-03-23 | [24417](https://github.com/airbytehq/airbyte/pull/24417) | Add pattern_descriptors to fields with an expected format | +| 0.4.4 | 2023-03-17 | [24255](https://github.com/airbytehq/airbyte/pull/24255) | Add field groups and titles to improve display of connector setup form | +| 0.4.3 | 2023-03-04 | [22993](https://github.com/airbytehq/airbyte/pull/22993) | Specified date formatting in specification | +| 0.4.2 | 2023-03-03 | [23467](https://github.com/airbytehq/airbyte/pull/23467) | added user friendly messages, added AirbyteTracedException config_error, updated SAT | +| 0.4.1 | 2023-01-27 | [22039](https://github.com/airbytehq/airbyte/pull/22039) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.4.0 | 2023-01-20 | [21457](https://github.com/airbytehq/airbyte/pull/21457) | Use GraphQL for `issue_reactions` stream | +| 0.3.12 | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481) | Handle 502 Bad Gateway error with proper log message | +| 0.3.11 | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084) | Raise Error if no organizations or repos are available during read | +| 0.3.10 | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523) | Revert changes from 0.3.9 | +| 0.3.9 | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978) | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes | +| 0.3.8 | 2022-11-10 | [19299](https://github.com/airbytehq/airbyte/pull/19299) | Fix events and workflow_runs datetimes | +| 0.3.7 | 2022-10-20 | [18213](https://github.com/airbytehq/airbyte/pull/18213) | Skip retry on HTTP 200 | +| 0.3.6 | 2022-10-11 | [17852](https://github.com/airbytehq/airbyte/pull/17852) | Use default behaviour, retry on 429 and all 5XX errors | +| 0.3.5 | 2022-10-07 | [17715](https://github.com/airbytehq/airbyte/pull/17715) | Improve 502 handling for `comments` stream | +| 0.3.4 | 2022-10-04 | [17555](https://github.com/airbytehq/airbyte/pull/17555) | Skip repository if got HTTP 500 for WorkflowRuns stream | +| 0.3.3 | 2022-09-28 | [17287](https://github.com/airbytehq/airbyte/pull/17287) | Fix problem with "null" `cursor_field` for WorkflowJobs stream | +| 0.3.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.3.1 | 2022-09-21 | [16947](https://github.com/airbytehq/airbyte/pull/16947) | Improve error logging when handling HTTP 500 error | +| 0.3.0 | 2022-09-09 | [16534](https://github.com/airbytehq/airbyte/pull/16534) | Add new stream `WorkflowJobs` | +| 0.2.46 | 2022-08-17 | [15730](https://github.com/airbytehq/airbyte/pull/15730) | Validate input organizations and repositories | +| 0.2.45 | 2022-08-11 | [15420](https://github.com/airbytehq/airbyte/pull/15420) | "User" object can be "null" | +| 0.2.44 | 2022-08-01 | [14795](https://github.com/airbytehq/airbyte/pull/14795) | Use GraphQL for `pull_request_comment_reactions` stream | +| 0.2.43 | 2022-07-26 | [15049](https://github.com/airbytehq/airbyte/pull/15049) | Bugfix schemas for streams `deployments`, `workflow_runs`, `teams` | +| 0.2.42 | 2022-07-12 | [14613](https://github.com/airbytehq/airbyte/pull/14613) | Improve schema for stream `pull_request_commits` added "null" | +| 0.2.41 | 2022-07-03 | [14376](https://github.com/airbytehq/airbyte/pull/14376) | Add Retry for GraphQL API Resource limitations | +| 0.2.40 | 2022-07-01 | [14338](https://github.com/airbytehq/airbyte/pull/14338) | Revert: "Rename field `mergeable` to `is_mergeable`" | +| 0.2.39 | 2022-06-30 | [14274](https://github.com/airbytehq/airbyte/pull/14274) | Rename field `mergeable` to `is_mergeable` | +| 0.2.38 | 2022-06-27 | [13989](https://github.com/airbytehq/airbyte/pull/13989) | Use GraphQL for `reviews` stream | +| 0.2.37 | 2022-06-21 | [13955](https://github.com/airbytehq/airbyte/pull/13955) | Fix "secondary rate limit" not retrying | +| 0.2.36 | 2022-06-20 | [13926](https://github.com/airbytehq/airbyte/pull/13926) | Break point added for `workflows_runs` stream | +| 0.2.35 | 2022-06-16 | [13763](https://github.com/airbytehq/airbyte/pull/13763) | Use GraphQL for `pull_request_stats` stream | +| 0.2.34 | 2022-06-14 | [13707](https://github.com/airbytehq/airbyte/pull/13707) | Fix API sorting, fix `get_starting_point` caching | +| 0.2.33 | 2022-06-08 | [13558](https://github.com/airbytehq/airbyte/pull/13558) | Enable caching only for parent streams | +| 0.2.32 | 2022-06-07 | [13531](https://github.com/airbytehq/airbyte/pull/13531) | Fix different result from `get_starting_point` when reading by pages | +| 0.2.31 | 2022-05-24 | [13115](https://github.com/airbytehq/airbyte/pull/13115) | Add incremental support for streams `WorkflowRuns` | +| 0.2.30 | 2022-05-09 | [12294](https://github.com/airbytehq/airbyte/pull/12294) | Add incremental support for streams `CommitCommentReactions`, `IssueCommentReactions`, `IssueReactions`, `PullRequestCommentReactions`, `Repositories`, `Workflows` | +| 0.2.29 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | +| 0.2.28 | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships` | +| 0.2.27 | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec | +| 0.2.26 | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream | +| 0.2.25 | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling | +| 0.2.24 | 2022-03-30 | [9251](https://github.com/airbytehq/airbyte/pull/9251) | Add Streams Workflow and WorkflowRuns | +| 0.2.23 | 2022-03-17 | [11212](https://github.com/airbytehq/airbyte/pull/11212) | Improve documentation and spec for Beta | +| 0.2.22 | 2022-03-10 | [10878](https://github.com/airbytehq/airbyte/pull/10878) | Fix error handling for unavailable streams with 404 status code | +| 0.2.21 | 2022-03-04 | [10749](https://github.com/airbytehq/airbyte/pull/10749) | Add new stream `ProjectCards` | +| 0.2.20 | 2022-02-16 | [10385](https://github.com/airbytehq/airbyte/pull/10385) | Add new stream `Deployments`, `ProjectColumns`, `PullRequestCommits` | +| 0.2.19 | 2022-02-07 | [10211](https://github.com/airbytehq/airbyte/pull/10211) | Add human-readable error in case of incorrect organization or repo name | +| 0.2.18 | 2021-02-09 | [10193](https://github.com/airbytehq/airbyte/pull/10193) | Add handling secondary rate limits | +| 0.2.17 | 2021-02-02 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Remove BAD_GATEWAY code from backoff_time | +| 0.2.16 | 2021-02-02 | [9868](https://github.com/airbytehq/airbyte/pull/9868) | Add log message for streams that are restricted for OAuth. Update oauth scopes. | +| 0.2.15 | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream | +| 0.2.14 | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams | +| 0.2.13 | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry` | +| 0.2.11 | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response | +| 0.2.10 | 2021-01-03 | [7250](https://github.com/airbytehq/airbyte/pull/7250) | Use CDK caching and convert PR-related streams to incremental | +| 0.2.9 | 2021-12-29 | [9179](https://github.com/airbytehq/airbyte/pull/9179) | Use default retry delays on server error responses | +| 0.2.8 | 2021-12-07 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.2.7 | 2021-12-06 | [8518](https://github.com/airbytehq/airbyte/pull/8518) | Add connection retry with GitHub | +| 0.2.6 | 2021-11-24 | [8030](https://github.com/airbytehq/airbyte/pull/8030) | Support start date property for PullRequestStats and Reviews streams | +| 0.2.5 | 2021-11-21 | [8170](https://github.com/airbytehq/airbyte/pull/8170) | Fix slow check connection for organizations with a lot of repos | +| 0.2.4 | 2021-11-11 | [7856](https://github.com/airbytehq/airbyte/pull/7856) | Resolve $ref fields in some stream schemas | +| 0.2.3 | 2021-10-06 | [6833](https://github.com/airbytehq/airbyte/pull/6833) | Fix config backward compatability | +| 0.2.2 | 2021-10-05 | [6761](https://github.com/airbytehq/airbyte/pull/6761) | Add oauth worflow specification | +| 0.2.1 | 2021-09-22 | [6223](https://github.com/airbytehq/airbyte/pull/6223) | Add option to pull commits from user-specified branches | +| 0.2.0 | 2021-09-19 | [5898](https://github.com/airbytehq/airbyte/pull/5898) and [6227](https://github.com/airbytehq/airbyte/pull/6227) | Don't minimize any output fields & add better error handling | +| 0.1.11 | 2021-09-15 | [5949](https://github.com/airbytehq/airbyte/pull/5949) | Add caching for all streams | +| 0.1.10 | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams | +| 0.1.9 | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream | +| 0.1.8 | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams | +| 0.1.7 | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values | +| 0.1.6 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator | +| 0.1.5 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation | +| 0.1.4 | 2021-08-13 | [5136](https://github.com/airbytehq/airbyte/pull/5136) | Support syncing multiple repositories/organizations | +| 0.1.3 | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams | +| 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting | +| 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | + +       \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct_all_description_exist.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct_all_description_exist.md new file mode 100644 index 000000000000..43110297ed9d --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/correct_all_description_exist.md @@ -0,0 +1,38 @@ +# GitHub + + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + + +## For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. Click Sources and then click + New source/destination. +3. On the Set up the source page, select GitHub from the Source type dropdown. +4. Enter a name for the GitHub connector. +5. Add list of GitHub repositories you want to sync. +6. Add Start Date from with data will be replicated. + +## For Airbyte Open Source: + +1. Navigate to the Airbyte Open Source dashboard. +2. Click Sources and then click + New source/destination. +3. On the Set up the source page, select GitHub from the Source type dropdown. +4. Enter a name for the GitHub connector. +5. Add list of GitHub repositories you want to sync. +6. Add Start Date from with data will be replicated. + +## Supported sync modes + +The GitHub source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): + +- Full Refresh +- Incremental + +## Tutorials + +Now that you have set up the GitHub source connector, check out the following GitHub tutorials: + +- [Creating PAT](https://docs.github.com/en/enterprise-server@3.9/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_header_order.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_header_order.md new file mode 100644 index 000000000000..3feb237735a7 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_header_order.md @@ -0,0 +1,306 @@ +## Prerequisites + +- List of GitHub Repositories (and access for them in case they are private) +- +# GitHub + + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + + + + +**For Airbyte Cloud:** + +- OAuth +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + + +**For Airbyte Open Source:** + +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + +### Step 1: Set up GitHub + +## Setup guide + +Create a [GitHub Account](https://github.com). + + +**Airbyte Open Source additional setup steps** + +Log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +### Step 2: Set up the GitHub connector in Airbyte + + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. +3. On the source selection page, select **GitHub** from the list of Sources. +4. Add a name for your GitHub connector. +5. To authenticate: + + + - **For Airbyte Cloud:** **Authenticate your GitHub account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. + + + + - **For Airbyte Open Source:** Authenticate with **Personal Access Token**. To generate a personal access token, log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). Enter your GitHub personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +6. **GitHub Repositories** - Enter a list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. + +:::caution +Repositories with the wrong name or repositories that do not exist or have the wrong name format will be skipped with `WARN` message in the logs. +::: + +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. For streams which support this configuration, only data generated on or after the start date will be replicated. + +- These streams will only sync records generated on or after the **Start Date**: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. + +- The **Start Date** does not apply to the streams below and all data will be synced for these streams: `assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` + +8. **Branch (Optional)** - List of GitHub repository branches to pull commits from, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). +9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5,000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + + + +## Supported sync modes + +The GitHub source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) + +## Supported Streams + +This connector outputs the following full refresh streams: + +- [Assignees](https://docs.github.com/en/rest/issues/assignees?apiVersion=2022-11-28#list-assignees) +- [Branches](https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches) +- [Contributor Activity](https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity) +- [Collaborators](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators) +- [Issue labels](https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#list-labels-for-a-repository) +- [Organizations](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations) +- [Pull request commits](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-commits-on-a-pull-request) +- [Tags](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-tags) +- [TeamMembers](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#list-team-members) +- [TeamMemberships](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#get-team-membership-for-a-user) +- [Teams](https://docs.github.com/en/rest/teams/teams?apiVersion=2022-11-28#list-teams) +- [Users](https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members) +- [Issue timeline events](https://docs.github.com/en/rest/issues/timeline?apiVersion=2022-11-28#list-timeline-events-for-an-issue) + +This connector outputs the following incremental streams: + +- [Comments](https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments-for-a-repository) +- [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions?apiVersion=2022-11-28#list-reactions-for-a-commit-comment) +- [Commit comments](https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository) +- [Commits](https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits) +- [Deployments](https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments) +- [Events](https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events) +- [Issue comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue-comment) +- [Issue events](https://docs.github.com/en/rest/issues/events?apiVersion=2022-11-28#list-issue-events-for-a-repository) +- [Issue milestones](https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#list-milestones) +- [Issue reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue) +- [Issues](https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues) +- [Project (Classic) cards](https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28#list-project-cards) +- [Project (Classic) columns](https://docs.github.com/en/rest/projects/columns?apiVersion=2022-11-28#list-project-columns) +- [Projects (Classic)](https://docs.github.com/en/rest/projects/projects?apiVersion=2022-11-28#list-repository-projects) +- [ProjectsV2](https://docs.github.com/en/graphql/reference/objects#projectv2) +- [Pull request comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-a-pull-request-review-comment) +- [Pull request stats](https://docs.github.com/en/graphql/reference/objects#pullrequest) +- [Pull requests](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests) +- [Releases](https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases) +- [Repositories](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories) +- [Review comments](https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository) +- [Reviews](https://docs.github.com/en/rest/pulls/reviews?apiVersion=2022-11-28#list-reviews-for-a-pull-request) +- [Stargazers](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) +- [WorkflowJobs](https://docs.github.com/pt/rest/actions/workflow-jobs?apiVersion=2022-11-28#list-jobs-for-a-workflow-run) +- [WorkflowRuns](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository) +- [Workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows) + +### Notes + +1. Only 4 streams \(`comments`, `commits`, `issues` and `review comments`\) from the listed above streams are pure incremental meaning that they: + + - read only new records; + - output only new records. + +2. Streams `workflow_runs` and `worflow_jobs` is almost pure incremental: + + - read new records and some portion of old records (in past 30 days) [docs](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs); + - the `workflow_jobs` depends on the `workflow_runs` to read the data, so they both follow the same logic [docs](https://docs.github.com/pt/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run); + - output only new records. + +3. Other 19 incremental streams are also incremental but with one difference, they: + + - read all records; + - output only new records. + Please, consider this behaviour when using those 19 incremental streams because it may affect you API call limits. + +4. Sometimes for large streams specifying very distant `start_date` in the past may result in keep on getting error from GitHub instead of records \(respective `WARN` log message will be outputted\). In this case Specifying more recent `start_date` may help. + **The "Start date" configuration option does not apply to the streams below, because the GitHub API does not include dates which can be used for filtering:** + +- `assignees` +- `branches` +- `collaborators` +- `issue_labels` +- `organizations` +- `pull_request_commits` +- `pull_request_stats` +- `repositories` +- `tags` +- `teams` +- `users` + +## Limitations & Troubleshooting + +
      + +Expand to see details about GitHub connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +#### Permissions and scopes + +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. + +Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: + +- For syncing Collaborators, the user which generates the personal access token must be a collaborator. To become a collaborator, they must be invited by an owner. If there are no collaborators, no records will be synced. Read more about access permissions [here](https://docs.github.com/en/get-started/learning-about-github/access-permissions-on-github). +- Syncing [Teams](https://docs.github.com/en/organizations/organizing-members-into-teams/about-teams) is only available to authenticated members of a team's [organization](https://docs.github.com/en/rest/orgs). [Personal user accounts](https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts) and repositories belonging to them don't have access to Teams features. In this case no records will be synced. +- To sync the Projects stream, the repository must have the Projects feature enabled. + +### Troubleshooting + +* Check out common troubleshooting issues for the GitHub source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) + +
      + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | +| 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | +| 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response | +| 1.5.1 | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` | +| 1.5.0 | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields | +| 1.4.6 | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \ | +| 1.4.5 | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` | +| 1.4.4 | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. | +| 1.4.3 | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` | +| 1.4.2 | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages | +| 1.4.1 | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version | +| 1.4.0 | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` | +| 1.3.1 | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` | +| 1.3.0 | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` | +| 1.2.1 | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` | +| 1.2.0 | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances | +| 1.1.1 | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages | +| 1.1.0 | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` | +| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter | +| 1.0.1 | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter | +| 1.0.0 | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI | +| 0.5.0 | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests | +| 0.4.11 | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025) | Added more transparent depiction of the personal access token expired | +| 0.4.10 | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075) | Add more specific error message description for no repos case. | +| 0.4.9 | 2023-05-01 | [24523](https://github.com/airbytehq/airbyte/pull/24523) | Add undeclared columns to spec | +| 0.4.8 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/25312) | Fix repo name validation | +| 0.4.7 | 2023-03-24 | [24457](https://github.com/airbytehq/airbyte/pull/24457) | Add validation and transformation for repositories config | +| 0.4.6 | 2023-03-24 | [24398](https://github.com/airbytehq/airbyte/pull/24398) | Fix caching for `get_starting_point` in stream "Commits" | +| 0.4.5 | 2023-03-23 | [24417](https://github.com/airbytehq/airbyte/pull/24417) | Add pattern_descriptors to fields with an expected format | +| 0.4.4 | 2023-03-17 | [24255](https://github.com/airbytehq/airbyte/pull/24255) | Add field groups and titles to improve display of connector setup form | +| 0.4.3 | 2023-03-04 | [22993](https://github.com/airbytehq/airbyte/pull/22993) | Specified date formatting in specification | +| 0.4.2 | 2023-03-03 | [23467](https://github.com/airbytehq/airbyte/pull/23467) | added user friendly messages, added AirbyteTracedException config_error, updated SAT | +| 0.4.1 | 2023-01-27 | [22039](https://github.com/airbytehq/airbyte/pull/22039) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.4.0 | 2023-01-20 | [21457](https://github.com/airbytehq/airbyte/pull/21457) | Use GraphQL for `issue_reactions` stream | +| 0.3.12 | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481) | Handle 502 Bad Gateway error with proper log message | +| 0.3.11 | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084) | Raise Error if no organizations or repos are available during read | +| 0.3.10 | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523) | Revert changes from 0.3.9 | +| 0.3.9 | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978) | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes | +| 0.3.8 | 2022-11-10 | [19299](https://github.com/airbytehq/airbyte/pull/19299) | Fix events and workflow_runs datetimes | +| 0.3.7 | 2022-10-20 | [18213](https://github.com/airbytehq/airbyte/pull/18213) | Skip retry on HTTP 200 | +| 0.3.6 | 2022-10-11 | [17852](https://github.com/airbytehq/airbyte/pull/17852) | Use default behaviour, retry on 429 and all 5XX errors | +| 0.3.5 | 2022-10-07 | [17715](https://github.com/airbytehq/airbyte/pull/17715) | Improve 502 handling for `comments` stream | +| 0.3.4 | 2022-10-04 | [17555](https://github.com/airbytehq/airbyte/pull/17555) | Skip repository if got HTTP 500 for WorkflowRuns stream | +| 0.3.3 | 2022-09-28 | [17287](https://github.com/airbytehq/airbyte/pull/17287) | Fix problem with "null" `cursor_field` for WorkflowJobs stream | +| 0.3.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.3.1 | 2022-09-21 | [16947](https://github.com/airbytehq/airbyte/pull/16947) | Improve error logging when handling HTTP 500 error | +| 0.3.0 | 2022-09-09 | [16534](https://github.com/airbytehq/airbyte/pull/16534) | Add new stream `WorkflowJobs` | +| 0.2.46 | 2022-08-17 | [15730](https://github.com/airbytehq/airbyte/pull/15730) | Validate input organizations and repositories | +| 0.2.45 | 2022-08-11 | [15420](https://github.com/airbytehq/airbyte/pull/15420) | "User" object can be "null" | +| 0.2.44 | 2022-08-01 | [14795](https://github.com/airbytehq/airbyte/pull/14795) | Use GraphQL for `pull_request_comment_reactions` stream | +| 0.2.43 | 2022-07-26 | [15049](https://github.com/airbytehq/airbyte/pull/15049) | Bugfix schemas for streams `deployments`, `workflow_runs`, `teams` | +| 0.2.42 | 2022-07-12 | [14613](https://github.com/airbytehq/airbyte/pull/14613) | Improve schema for stream `pull_request_commits` added "null" | +| 0.2.41 | 2022-07-03 | [14376](https://github.com/airbytehq/airbyte/pull/14376) | Add Retry for GraphQL API Resource limitations | +| 0.2.40 | 2022-07-01 | [14338](https://github.com/airbytehq/airbyte/pull/14338) | Revert: "Rename field `mergeable` to `is_mergeable`" | +| 0.2.39 | 2022-06-30 | [14274](https://github.com/airbytehq/airbyte/pull/14274) | Rename field `mergeable` to `is_mergeable` | +| 0.2.38 | 2022-06-27 | [13989](https://github.com/airbytehq/airbyte/pull/13989) | Use GraphQL for `reviews` stream | +| 0.2.37 | 2022-06-21 | [13955](https://github.com/airbytehq/airbyte/pull/13955) | Fix "secondary rate limit" not retrying | +| 0.2.36 | 2022-06-20 | [13926](https://github.com/airbytehq/airbyte/pull/13926) | Break point added for `workflows_runs` stream | +| 0.2.35 | 2022-06-16 | [13763](https://github.com/airbytehq/airbyte/pull/13763) | Use GraphQL for `pull_request_stats` stream | +| 0.2.34 | 2022-06-14 | [13707](https://github.com/airbytehq/airbyte/pull/13707) | Fix API sorting, fix `get_starting_point` caching | +| 0.2.33 | 2022-06-08 | [13558](https://github.com/airbytehq/airbyte/pull/13558) | Enable caching only for parent streams | +| 0.2.32 | 2022-06-07 | [13531](https://github.com/airbytehq/airbyte/pull/13531) | Fix different result from `get_starting_point` when reading by pages | +| 0.2.31 | 2022-05-24 | [13115](https://github.com/airbytehq/airbyte/pull/13115) | Add incremental support for streams `WorkflowRuns` | +| 0.2.30 | 2022-05-09 | [12294](https://github.com/airbytehq/airbyte/pull/12294) | Add incremental support for streams `CommitCommentReactions`, `IssueCommentReactions`, `IssueReactions`, `PullRequestCommentReactions`, `Repositories`, `Workflows` | +| 0.2.29 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | +| 0.2.28 | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships` | +| 0.2.27 | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec | +| 0.2.26 | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream | +| 0.2.25 | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling | +| 0.2.24 | 2022-03-30 | [9251](https://github.com/airbytehq/airbyte/pull/9251) | Add Streams Workflow and WorkflowRuns | +| 0.2.23 | 2022-03-17 | [11212](https://github.com/airbytehq/airbyte/pull/11212) | Improve documentation and spec for Beta | +| 0.2.22 | 2022-03-10 | [10878](https://github.com/airbytehq/airbyte/pull/10878) | Fix error handling for unavailable streams with 404 status code | +| 0.2.21 | 2022-03-04 | [10749](https://github.com/airbytehq/airbyte/pull/10749) | Add new stream `ProjectCards` | +| 0.2.20 | 2022-02-16 | [10385](https://github.com/airbytehq/airbyte/pull/10385) | Add new stream `Deployments`, `ProjectColumns`, `PullRequestCommits` | +| 0.2.19 | 2022-02-07 | [10211](https://github.com/airbytehq/airbyte/pull/10211) | Add human-readable error in case of incorrect organization or repo name | +| 0.2.18 | 2021-02-09 | [10193](https://github.com/airbytehq/airbyte/pull/10193) | Add handling secondary rate limits | +| 0.2.17 | 2021-02-02 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Remove BAD_GATEWAY code from backoff_time | +| 0.2.16 | 2021-02-02 | [9868](https://github.com/airbytehq/airbyte/pull/9868) | Add log message for streams that are restricted for OAuth. Update oauth scopes. | +| 0.2.15 | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream | +| 0.2.14 | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams | +| 0.2.13 | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry` | +| 0.2.11 | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response | +| 0.2.10 | 2021-01-03 | [7250](https://github.com/airbytehq/airbyte/pull/7250) | Use CDK caching and convert PR-related streams to incremental | +| 0.2.9 | 2021-12-29 | [9179](https://github.com/airbytehq/airbyte/pull/9179) | Use default retry delays on server error responses | +| 0.2.8 | 2021-12-07 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.2.7 | 2021-12-06 | [8518](https://github.com/airbytehq/airbyte/pull/8518) | Add connection retry with GitHub | +| 0.2.6 | 2021-11-24 | [8030](https://github.com/airbytehq/airbyte/pull/8030) | Support start date property for PullRequestStats and Reviews streams | +| 0.2.5 | 2021-11-21 | [8170](https://github.com/airbytehq/airbyte/pull/8170) | Fix slow check connection for organizations with a lot of repos | +| 0.2.4 | 2021-11-11 | [7856](https://github.com/airbytehq/airbyte/pull/7856) | Resolve $ref fields in some stream schemas | +| 0.2.3 | 2021-10-06 | [6833](https://github.com/airbytehq/airbyte/pull/6833) | Fix config backward compatability | +| 0.2.2 | 2021-10-05 | [6761](https://github.com/airbytehq/airbyte/pull/6761) | Add oauth worflow specification | +| 0.2.1 | 2021-09-22 | [6223](https://github.com/airbytehq/airbyte/pull/6223) | Add option to pull commits from user-specified branches | +| 0.2.0 | 2021-09-19 | [5898](https://github.com/airbytehq/airbyte/pull/5898) and [6227](https://github.com/airbytehq/airbyte/pull/6227) | Don't minimize any output fields & add better error handling | +| 0.1.11 | 2021-09-15 | [5949](https://github.com/airbytehq/airbyte/pull/5949) | Add caching for all streams | +| 0.1.10 | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams | +| 0.1.9 | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream | +| 0.1.8 | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams | +| 0.1.7 | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values | +| 0.1.6 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator | +| 0.1.5 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation | +| 0.1.4 | 2021-08-13 | [5136](https://github.com/airbytehq/airbyte/pull/5136) | Support syncing multiple repositories/organizations | +| 0.1.3 | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams | +| 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting | +| 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | + +       \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_not_all_structure.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_not_all_structure.md new file mode 100644 index 000000000000..b041e8cc78d4 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/incorrect_not_all_structure.md @@ -0,0 +1,6 @@ +## GitHub + +## Prerequisites + +- Start Date - the start date to replicate your date. + diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/invalid_links.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/invalid_links.md new file mode 100644 index 000000000000..1efb99d700d7 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/invalid_links.md @@ -0,0 +1,305 @@ +# GitHub + + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + + +## Prerequisites + +- List of GitHub Repositories (and access for them in case they are private) + + +**For Airbyte Cloud:** + +- OAuth +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + + +**For Airbyte Open Source:** + +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) + + +## Setup guide + +### Step 1: Set up GitHub + +Create a [GitHub Account](https://github.com). + + +**Airbyte Open Source additional setup steps** + +Log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens-that_do_not_exist). To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +### Step 2: Set up the GitHub connector in Airbyte + + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. +3. On the source selection page, select **GitHub** from the list of Sources. +4. Add a name for your GitHub connector. +5. To authenticate: + + + - **For Airbyte Cloud:** **Authenticate your GitHub account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. + + + + - **For Airbyte Open Source:** Authenticate with **Personal Access Token**. To generate a personal access token, log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). Enter your GitHub personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +6. **GitHub Repositories** - Enter a list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. + +:::caution +Repositories with the wrong name or repositories that do not exist or have the wrong name format will be skipped with `WARN` message in the logs. +::: + +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. For streams which support this configuration, only data generated on or after the start date will be replicated. + +- These streams will only sync records generated on or after the **Start Date**: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. + +- The **Start Date** does not apply to the streams below and all data will be synced for these streams: `assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` + +8. **Branch (Optional)** - List of GitHub repository branches to pull commits from, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). +9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5,000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + + + +## Supported sync modes + +The GitHub source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) + +## Supported Streams + +This connector outputs the following full refresh streams: + +- [Assignees](https://docs.github.com/en/rest/issues/assignees?apiVersion=2022-11-28#list-assignees) +- [Branches](https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches) +- [Contributor Activity](https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity) +- [Collaborators](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators) +- [Issue labels](https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#list-labels-for-a-repository) +- [Organizations](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations) +- [Pull request commits](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-commits-on-a-pull-request) +- [Tags](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-tags) +- [TeamMembers](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#list-team-members) +- [TeamMemberships](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#get-team-membership-for-a-user) +- [Teams](https://docs.github.com/en/rest/teams/teams?apiVersion=2022-11-28#list-teams) +- [Users](https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members) +- [Issue timeline events](https://docs.github.com/en/rest/issues/timeline?apiVersion=2022-11-28#list-timeline-events-for-an-issue) + +This connector outputs the following incremental streams: + +- [Comments](https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments-for-a-repository) +- [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions?apiVersion=2022-11-28#list-reactions-for-a-commit-comment) +- [Commit comments](https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository) +- [Commits](https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits) +- [Deployments](https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments) +- [Events](https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events) +- [Issue comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue-comment) +- [Issue events](https://docs.github.com/en/rest/issues/events?apiVersion=2022-11-28#list-issue-events-for-a-repository) +- [Issue milestones](https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#list-milestones) +- [Issue reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue) +- [Issues](https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues) +- [Project (Classic) cards](https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28#list-project-cards) +- [Project (Classic) columns](https://docs.github.com/en/rest/projects/columns?apiVersion=2022-11-28#list-project-columns) +- [Projects (Classic)](https://docs.github.com/en/rest/projects/projects?apiVersion=2022-11-28#list-repository-projects) +- [ProjectsV2](https://docs.github.com/en/graphql/reference/objects#projectv2) +- [Pull request comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-a-pull-request-review-comment) +- [Pull request stats](https://docs.github.com/en/graphql/reference/objects#pullrequest) +- [Pull requests](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests) +- [Releases](https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases) +- [Repositories](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories) +- [Review comments](https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository) +- [Reviews](https://docs.github.com/en/rest/pulls/reviews?apiVersion=2022-11-28#list-reviews-for-a-pull-request) +- [Stargazers](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) +- [WorkflowJobs](https://docs.github.com/pt/rest/actions/workflow-jobs?apiVersion=2022-11-28#list-jobs-for-a-workflow-run) +- [WorkflowRuns](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository) +- [Workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows) + +### Notes + +1. Only 4 streams \(`comments`, `commits`, `issues` and `review comments`\) from the listed above streams are pure incremental meaning that they: + + - read only new records; + - output only new records. + +2. Streams `workflow_runs` and `worflow_jobs` is almost pure incremental: + + - read new records and some portion of old records (in past 30 days) [docs](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs); + - the `workflow_jobs` depends on the `workflow_runs` to read the data, so they both follow the same logic [docs](https://docs.github.com/pt/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run); + - output only new records. + +3. Other 19 incremental streams are also incremental but with one difference, they: + + - read all records; + - output only new records. + Please, consider this behaviour when using those 19 incremental streams because it may affect you API call limits. + +4. Sometimes for large streams specifying very distant `start_date` in the past may result in keep on getting error from GitHub instead of records \(respective `WARN` log message will be outputted\). In this case Specifying more recent `start_date` may help. + **The "Start date" configuration option does not apply to the streams below, because the GitHub API does not include dates which can be used for filtering:** + +- `assignees` +- `branches` +- `collaborators` +- `issue_labels` +- `organizations` +- `pull_request_commits` +- `pull_request_stats` +- `repositories` +- `tags` +- `teams` +- `users` + +## Limitations & Troubleshooting + +
      + +Expand to see details about GitHub connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +#### Permissions and scopes + +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. + +Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: + +- For syncing Collaborators, the user which generates the personal access token must be a collaborator. To become a collaborator, they must be invited by an owner. If there are no collaborators, no records will be synced. Read more about access permissions [here](https://docs.github.com/en/get-started/learning-about-github/access-permissions-on-github). +- Syncing [Teams](https://docs.github.com/en/organizations/organizing-members-into-teams/about-teams_do_not_exists) is only available to authenticated members of a team's [organization](https://docs.github.com/en/rest/orgs). [Personal user accounts](https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts) and repositories belonging to them don't have access to Teams features. In this case no records will be synced. +- To sync the Projects stream, the repository must have the Projects feature enabled. + +### Troubleshooting + +* Check out common troubleshooting issues for the GitHub source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) + +
      + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | +| 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | +| 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response | +| 1.5.1 | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` | +| 1.5.0 | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields | +| 1.4.6 | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \ | +| 1.4.5 | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` | +| 1.4.4 | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. | +| 1.4.3 | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` | +| 1.4.2 | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages | +| 1.4.1 | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version | +| 1.4.0 | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` | +| 1.3.1 | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` | +| 1.3.0 | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` | +| 1.2.1 | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` | +| 1.2.0 | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances | +| 1.1.1 | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages | +| 1.1.0 | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` | +| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter | +| 1.0.1 | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter | +| 1.0.0 | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI | +| 0.5.0 | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests | +| 0.4.11 | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025) | Added more transparent depiction of the personal access token expired | +| 0.4.10 | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075) | Add more specific error message description for no repos case. | +| 0.4.9 | 2023-05-01 | [24523](https://github.com/airbytehq/airbyte/pull/24523) | Add undeclared columns to spec | +| 0.4.8 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/25312) | Fix repo name validation | +| 0.4.7 | 2023-03-24 | [24457](https://github.com/airbytehq/airbyte/pull/24457) | Add validation and transformation for repositories config | +| 0.4.6 | 2023-03-24 | [24398](https://github.com/airbytehq/airbyte/pull/24398) | Fix caching for `get_starting_point` in stream "Commits" | +| 0.4.5 | 2023-03-23 | [24417](https://github.com/airbytehq/airbyte/pull/24417) | Add pattern_descriptors to fields with an expected format | +| 0.4.4 | 2023-03-17 | [24255](https://github.com/airbytehq/airbyte/pull/24255) | Add field groups and titles to improve display of connector setup form | +| 0.4.3 | 2023-03-04 | [22993](https://github.com/airbytehq/airbyte/pull/22993) | Specified date formatting in specification | +| 0.4.2 | 2023-03-03 | [23467](https://github.com/airbytehq/airbyte/pull/23467) | added user friendly messages, added AirbyteTracedException config_error, updated SAT | +| 0.4.1 | 2023-01-27 | [22039](https://github.com/airbytehq/airbyte/pull/22039) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.4.0 | 2023-01-20 | [21457](https://github.com/airbytehq/airbyte/pull/21457) | Use GraphQL for `issue_reactions` stream | +| 0.3.12 | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481) | Handle 502 Bad Gateway error with proper log message | +| 0.3.11 | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084) | Raise Error if no organizations or repos are available during read | +| 0.3.10 | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523) | Revert changes from 0.3.9 | +| 0.3.9 | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978) | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes | +| 0.3.8 | 2022-11-10 | [19299](https://github.com/airbytehq/airbyte/pull/19299) | Fix events and workflow_runs datetimes | +| 0.3.7 | 2022-10-20 | [18213](https://github.com/airbytehq/airbyte/pull/18213) | Skip retry on HTTP 200 | +| 0.3.6 | 2022-10-11 | [17852](https://github.com/airbytehq/airbyte/pull/17852) | Use default behaviour, retry on 429 and all 5XX errors | +| 0.3.5 | 2022-10-07 | [17715](https://github.com/airbytehq/airbyte/pull/17715) | Improve 502 handling for `comments` stream | +| 0.3.4 | 2022-10-04 | [17555](https://github.com/airbytehq/airbyte/pull/17555) | Skip repository if got HTTP 500 for WorkflowRuns stream | +| 0.3.3 | 2022-09-28 | [17287](https://github.com/airbytehq/airbyte/pull/17287) | Fix problem with "null" `cursor_field` for WorkflowJobs stream | +| 0.3.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.3.1 | 2022-09-21 | [16947](https://github.com/airbytehq/airbyte/pull/16947) | Improve error logging when handling HTTP 500 error | +| 0.3.0 | 2022-09-09 | [16534](https://github.com/airbytehq/airbyte/pull/16534) | Add new stream `WorkflowJobs` | +| 0.2.46 | 2022-08-17 | [15730](https://github.com/airbytehq/airbyte/pull/15730) | Validate input organizations and repositories | +| 0.2.45 | 2022-08-11 | [15420](https://github.com/airbytehq/airbyte/pull/15420) | "User" object can be "null" | +| 0.2.44 | 2022-08-01 | [14795](https://github.com/airbytehq/airbyte/pull/14795) | Use GraphQL for `pull_request_comment_reactions` stream | +| 0.2.43 | 2022-07-26 | [15049](https://github.com/airbytehq/airbyte/pull/15049) | Bugfix schemas for streams `deployments`, `workflow_runs`, `teams` | +| 0.2.42 | 2022-07-12 | [14613](https://github.com/airbytehq/airbyte/pull/14613) | Improve schema for stream `pull_request_commits` added "null" | +| 0.2.41 | 2022-07-03 | [14376](https://github.com/airbytehq/airbyte/pull/14376) | Add Retry for GraphQL API Resource limitations | +| 0.2.40 | 2022-07-01 | [14338](https://github.com/airbytehq/airbyte/pull/14338) | Revert: "Rename field `mergeable` to `is_mergeable`" | +| 0.2.39 | 2022-06-30 | [14274](https://github.com/airbytehq/airbyte/pull/14274) | Rename field `mergeable` to `is_mergeable` | +| 0.2.38 | 2022-06-27 | [13989](https://github.com/airbytehq/airbyte/pull/13989) | Use GraphQL for `reviews` stream | +| 0.2.37 | 2022-06-21 | [13955](https://github.com/airbytehq/airbyte/pull/13955) | Fix "secondary rate limit" not retrying | +| 0.2.36 | 2022-06-20 | [13926](https://github.com/airbytehq/airbyte/pull/13926) | Break point added for `workflows_runs` stream | +| 0.2.35 | 2022-06-16 | [13763](https://github.com/airbytehq/airbyte/pull/13763) | Use GraphQL for `pull_request_stats` stream | +| 0.2.34 | 2022-06-14 | [13707](https://github.com/airbytehq/airbyte/pull/13707) | Fix API sorting, fix `get_starting_point` caching | +| 0.2.33 | 2022-06-08 | [13558](https://github.com/airbytehq/airbyte/pull/13558) | Enable caching only for parent streams | +| 0.2.32 | 2022-06-07 | [13531](https://github.com/airbytehq/airbyte/pull/13531) | Fix different result from `get_starting_point` when reading by pages | +| 0.2.31 | 2022-05-24 | [13115](https://github.com/airbytehq/airbyte/pull/13115) | Add incremental support for streams `WorkflowRuns` | +| 0.2.30 | 2022-05-09 | [12294](https://github.com/airbytehq/airbyte/pull/12294) | Add incremental support for streams `CommitCommentReactions`, `IssueCommentReactions`, `IssueReactions`, `PullRequestCommentReactions`, `Repositories`, `Workflows` | +| 0.2.29 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | +| 0.2.28 | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships` | +| 0.2.27 | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec | +| 0.2.26 | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream | +| 0.2.25 | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling | +| 0.2.24 | 2022-03-30 | [9251](https://github.com/airbytehq/airbyte/pull/9251) | Add Streams Workflow and WorkflowRuns | +| 0.2.23 | 2022-03-17 | [11212](https://github.com/airbytehq/airbyte/pull/11212) | Improve documentation and spec for Beta | +| 0.2.22 | 2022-03-10 | [10878](https://github.com/airbytehq/airbyte/pull/10878) | Fix error handling for unavailable streams with 404 status code | +| 0.2.21 | 2022-03-04 | [10749](https://github.com/airbytehq/airbyte/pull/10749) | Add new stream `ProjectCards` | +| 0.2.20 | 2022-02-16 | [10385](https://github.com/airbytehq/airbyte/pull/10385) | Add new stream `Deployments`, `ProjectColumns`, `PullRequestCommits` | +| 0.2.19 | 2022-02-07 | [10211](https://github.com/airbytehq/airbyte/pull/10211) | Add human-readable error in case of incorrect organization or repo name | +| 0.2.18 | 2021-02-09 | [10193](https://github.com/airbytehq/airbyte/pull/10193) | Add handling secondary rate limits | +| 0.2.17 | 2021-02-02 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Remove BAD_GATEWAY code from backoff_time | +| 0.2.16 | 2021-02-02 | [9868](https://github.com/airbytehq/airbyte/pull/9868) | Add log message for streams that are restricted for OAuth. Update oauth scopes. | +| 0.2.15 | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream | +| 0.2.14 | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams | +| 0.2.13 | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry` | +| 0.2.11 | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response | +| 0.2.10 | 2021-01-03 | [7250](https://github.com/airbytehq/airbyte/pull/7250) | Use CDK caching and convert PR-related streams to incremental | +| 0.2.9 | 2021-12-29 | [9179](https://github.com/airbytehq/airbyte/pull/9179) | Use default retry delays on server error responses | +| 0.2.8 | 2021-12-07 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.2.7 | 2021-12-06 | [8518](https://github.com/airbytehq/airbyte/pull/8518) | Add connection retry with GitHub | +| 0.2.6 | 2021-11-24 | [8030](https://github.com/airbytehq/airbyte/pull/8030) | Support start date property for PullRequestStats and Reviews streams | +| 0.2.5 | 2021-11-21 | [8170](https://github.com/airbytehq/airbyte/pull/8170) | Fix slow check connection for organizations with a lot of repos | +| 0.2.4 | 2021-11-11 | [7856](https://github.com/airbytehq/airbyte/pull/7856) | Resolve $ref fields in some stream schemas | +| 0.2.3 | 2021-10-06 | [6833](https://github.com/airbytehq/airbyte/pull/6833) | Fix config backward compatability | +| 0.2.2 | 2021-10-05 | [6761](https://github.com/airbytehq/airbyte/pull/6761) | Add oauth worflow specification | +| 0.2.1 | 2021-09-22 | [6223](https://github.com/airbytehq/airbyte/pull/6223) | Add option to pull commits from user-specified branches | +| 0.2.0 | 2021-09-19 | [5898](https://github.com/airbytehq/airbyte/pull/5898) and [6227](https://github.com/airbytehq/airbyte/pull/6227) | Don't minimize any output fields & add better error handling | +| 0.1.11 | 2021-09-15 | [5949](https://github.com/airbytehq/airbyte/pull/5949) | Add caching for all streams | +| 0.1.10 | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams | +| 0.1.9 | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream | +| 0.1.8 | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams | +| 0.1.7 | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values | +| 0.1.6 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator | +| 0.1.5 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation | +| 0.1.4 | 2021-08-13 | [5136](https://github.com/airbytehq/airbyte/pull/5136) | Support syncing multiple repositories/organizations | +| 0.1.3 | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams | +| 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting | +| 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | + +       \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/with_not_required_steps.md b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/with_not_required_steps.md new file mode 100644 index 000000000000..942837b08a73 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/data/docs/with_not_required_steps.md @@ -0,0 +1,123 @@ +# Oracle Netsuite + +One unified business management suite, encompassing ERP/Financials, CRM and ecommerce for more than 31,000 customers. + +This connector implements the [SuiteTalk REST Web Services](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/chapter_1540391670.html) and uses REST API to fetch the customers data. + +## Prerequisites +* Oracle NetSuite [account](https://system.netsuite.com/pages/customerlogin.jsp?country=US) +* Allowed access to all Account permissions options + +## Setup guide +### Step 1: Create Oracle NetSuite account + +1. Create [account](https://system.netsuite.com/pages/customerlogin.jsp?country=US) on Oracle NetSuite +2. Confirm your Email + +### Step 2: Setup NetSuite account +#### Step 2.1: Obtain Realm info +1. Login into your NetSuite [account](https://system.netsuite.com/pages/customerlogin.jsp?country=US) +2. Go to **Setup** » **Company** » **Company Information** +3. Copy your Account ID (Realm). It should look like **1234567** for the `Production` env. or **1234567_SB2** - for a `Sandbox` +#### Step 2.2: Enable features +1. Go to **Setup** » **Company** » **Enable Features** +2. Click on **SuiteCloud** tab +3. Scroll down to **SuiteScript** section +4. Enable checkbox for `CLIENT SUITESCRIPT` and `SERVER SUITESCRIPT` +5. Scroll down to **Manage Authentication** section +6. Enable checkbox `TOKEN-BASED AUTHENTICATION` +7. Scroll down to **SuiteTalk (Web Services)** +8. Enable checkbox `REST WEB SERVISES` +9. Save the changes +#### Step 2.3: Create Integration (obtain Consumer Key and Consumer Secret) +1. Go to **Setup** » **Integration** » **Manage Integrations** » **New** +2. Fill the **Name** field (we recommend to put `airbyte-rest-integration` for a name) +3. Make sure the **State** is `enabled` +4. Enable checkbox `Token-Based Authentication` in **Authentication** section +5. Save changes +6. After that, **Consumer Key** and **Consumer Secret** will be showed once (copy them to the safe place) +#### Step 2.4: Setup Role +1. Go to **Setup** » **Users/Roles** » **Manage Roles** » **New** +2. Fill the **Name** field (we recommend to put `airbyte-integration-role` for a name) +3. Scroll down to **Permissions** tab +4. (REQUIRED) Click on `Transactions` and manually `add` all the dropdown entities with either `full` or `view` access level. +5. (REQUIRED) Click on `Reports` and manually `add` all the dropdown entities with either `full` or `view` access level. +6. (REQUIRED) Click on `Lists` and manually `add` all the dropdown entities with either `full` or `view` access level. +7. (REQUIRED) Click on `Setup` and manually `add` all the dropdown entities with either `full` or `view` access level. +* Make sure you've done all `REQUIRED` steps correctly, to avoid sync issues in the future. +* Please edit these params again when you `rename` or `customise` any `Object` in Netsuite for `airbyte-integration-role` to reflect such changes. + +#### Step 2.5: Setup User +1. Go to **Setup** » **Users/Roles** » **Manage Users** +2. In column `Name` click on the user’s name you want to give access to the `airbyte-integration-role` +3. Then click on **Edit** button under the user’s name +4. Scroll down to **Access** tab at the bottom +5. Select from dropdown list the `airbyte-integration-role` role which you created in step 2.4 +6. Save changes + +#### Step 2.6: Create Access Token for role +1. Go to **Setup** » **Users/Roles** » **Access Tokens** » **New** +2. Select an **Application Name** +3. Under **User** select the user you assigned the `airbyte-integration-role` in the step **2.4** +4. Inside **Role** select the one you gave to the user in the step **2.5** +5. Under **Token Name** you can give a descriptive name to the Token you are creating (we recommend to put `airbyte-rest-integration-token` for a name) +6. Save changes +7. After that, **Token ID** and **Token Secret** will be showed once (copy them to the safe place) + +#### Step 2.7: Summary +You have copied next parameters +* Realm (Account ID) +* Consumer Key +* Consumer Secret +* Token ID +* Token Secret +Also you have properly **Configured Account** with **Correct Permissions** and **Access Token** for User and Role you've created early. + +### Step 3: Set up the source connector in Airbyte +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **NetSuite** from the Source type dropdown and enter a name for this connector. +4. Add **Realm** +5. Add **Consumer Key** +6. Add **Consumer Secret** +7. Add **Token ID** +8. Add **Token Secret** +9. Click `Set up source`. + +### For Airbyte OSS: + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **NetSuite** from the Source type dropdown and enter a name for this connector. +4. Add **Realm** +5. Add **Consumer Key** +6. Add **Consumer Secret** +7. Add **Token ID** +8. Add **Token Secret** +9. Click `Set up source` + + +## Supported sync modes + +The NetSuite source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + - Incremental + +## Supported Streams + +- Streams are generated based on `ROLE` and `USER` access to them as well as `Account` settings, make sure you're using the correct role assigned in our case `airbyte-integration-role` or any other custom `ROLE` granted to the Access Token, having the access to the NetSuite objects for data sync, please refer to the **Setup guide** > **Step 2.4** and **Setup guide** > **Step 2.5** + + +## Performance considerations + +The connector is restricted by Netsuite [Concurrency Limit per Integration](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/bridgehead_156224824287.html). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.3 | 2023-01-20 | [21645](https://github.com/airbytehq/airbyte/pull/21645) | Minor issues fix, Setup Guide corrections for public docs | +| 0.1.1 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state | +| 0.1.0 | 2022-09-15 | [16093](https://github.com/airbytehq/airbyte/pull/16093) | Initial Alpha release | diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py index 0b78b93148c6..d0732458ec4d 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py @@ -87,6 +87,57 @@ def test_verify_records_schema(configured_catalog: ConfiguredAirbyteCatalog): ] +@pytest.mark.parametrize( + "json_schema, record, should_fail", + [ + ({"type": "object", "properties": {"a": {"type": "string"}}}, {"a": "str", "b": "extra_string"}, True), + ( + {"type": "object", "properties": {"a": {"type": "string"}, "some_obj": {"type": ["null", "object"]}}}, + {"a": "str", "some_obj": {"b": "extra_string"}}, + False, + ), + ( + { + "type": "object", + "properties": {"a": {"type": "string"}, "some_obj": {"type": ["null", "object"], "properties": {"a": {"type": "string"}}}}, + }, + {"a": "str", "some_obj": {"a": "str", "b": "extra_string"}}, + True, + ), + ( + {"type": "object", "properties": {"a": {"type": "string"}, "b": {"type": "array", "items": {"type": "object"}}}}, + {"a": "str", "b": [{"a": "extra_string"}]}, + False, + ), + ( + { + "type": "object", + "properties": { + "a": {"type": "string"}, + "b": {"type": "array", "items": {"type": "object", "properties": {"a": {"type": "string"}}}}, + }, + }, + {"a": "str", "b": [{"a": "string", "b": "extra_string"}]}, + True, + ), + ], + ids=[ + "simple_schema_and_record_with_extra_property", + "schema_with_object_without_properties_and_record_with_object_with_property", + "schema_with_object_with_properties_and_record_with_object_with_extra_property", + "schema_with_array_of_objects_without_properties_and_record_with_array_of_objects_with_property", + "schema_with_array_of_objects_with_properties_and_record_with_array_of_objects_with_extra_property", + ], +) +def test_verify_records_schema_with_fail_on_extra_columns(configured_catalog: ConfiguredAirbyteCatalog, json_schema, record, should_fail): + """Test that fail_on_extra_columns works correctly with nested objects, array of objects""" + configured_catalog.streams[0].stream.json_schema = json_schema + records = [AirbyteRecordMessage(stream="my_stream", data=record, emitted_at=0)] + streams_with_errors = verify_records_schema(records, configured_catalog, fail_on_extra_columns=True) + errors = [error.message for error in streams_with_errors["my_stream"].values()] + assert errors if should_fail else not errors + + @pytest.mark.parametrize( "record, configured_catalog, valid", [ diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py index 2687bfaf5101..5a4e62e55520 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py @@ -212,3 +212,33 @@ class TestExpectedRecordsConfig: def test_bypass_reason_behavior(self, path, bypass_reason, expectation): with expectation: config.ExpectedRecordsConfig(path=path, bypass_reason=bypass_reason) + + +class TestFileTypesConfig: + @pytest.mark.parametrize( + ("skip_test", "bypass_reason", "unsupported_types", "expectation"), + ( + (True, None, None, does_not_raise()), + (True, None, [config.UnsupportedFileTypeConfig(extension=".csv")], pytest.raises(ValidationError)), + (False, None, None, does_not_raise()), + (False, "bypass_reason", None, pytest.raises(ValidationError)), + (False, "", None, pytest.raises(ValidationError)), + (False, None, [config.UnsupportedFileTypeConfig(extension=".csv")], does_not_raise()), + ), + ) + def test_skip_test_behavior(self, skip_test, bypass_reason, unsupported_types, expectation): + with expectation: + config.FileTypesConfig(skip_test=skip_test, bypass_reason=bypass_reason, unsupported_types=unsupported_types) + + @pytest.mark.parametrize( + ("extension", "expectation"), + ( + (".csv", does_not_raise()), + ("csv", pytest.raises(ValidationError)), + (".", pytest.raises(ValidationError)), + ("", pytest.raises(ValidationError)), + ), + ) + def test_extension_validation(self, extension, expectation): + with expectation: + config.UnsupportedFileTypeConfig(extension=extension) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py new file mode 100644 index 000000000000..7435915090c7 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -0,0 +1,177 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type +from connector_acceptance_test.config import NoPrimaryKeyConfiguration +from connector_acceptance_test.tests import test_core + +pytestmark = pytest.mark.anyio + + +@pytest.mark.parametrize( + "stream_configs, excluded_streams, expected_error_streams", + [ + pytest.param([{"name": "stream_with_primary_key", "primary_key": [["id"]]}], [], None, id="test_stream_with_primary_key_succeeds"), + pytest.param([{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails"), + pytest.param([{"name": "report_stream"}], ["report_stream"], None, id="test_primary_key_excluded_from_test"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], [], ["himmel", "heiter"], id="test_multiple_streams_that_are_missing_primary_key"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], ["himmel", "heiter"], None, id="test_multiple_streams_that_exclude_primary_key"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], ["heiter"], ["himmel"], id="test_multiple_streams_missing_primary_key_or_excluded"), + ], +) +async def test_streams_define_primary_key(mocker, stream_configs, excluded_streams, expected_error_streams): + t = test_core.TestConnectorAttributes() + + streams = [AirbyteStream.parse_obj({ + "name": stream_config.get("name"), + "json_schema": {}, + "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": stream_config.get("primary_key"), + }) for stream_config in stream_configs] + + streams_without_primary_key = [NoPrimaryKeyConfiguration(name=stream, bypass_reason="") for stream in excluded_streams] + + docker_runner_mock = mocker.MagicMock( + call_discover=mocker.AsyncMock( + return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))] + ) + ) + + if expected_error_streams: + with pytest.raises(AssertionError) as e: + await t.test_streams_define_primary_key( + operational_certification_test=True, + streams_without_primary_key=streams_without_primary_key, + connector_config={}, + docker_runner=docker_runner_mock + ) + streams_in_error_message = [stream_name for stream_name in expected_error_streams if stream_name in e.value.args[0]] + assert streams_in_error_message == expected_error_streams + else: + await t.test_streams_define_primary_key( + operational_certification_test=True, + streams_without_primary_key=streams_without_primary_key, + connector_config={}, + docker_runner=docker_runner_mock + ) + + +@pytest.mark.parametrize( + "metadata_yaml, should_raise_assert_error, expected_error", + [ + pytest.param( + {"data": {"ab_internal": {"ql": 400}}}, + True, + "The `allowedHosts` property is missing in `metadata.data` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {}}}, + True, + "The `hosts` property is missing in `metadata.data.allowedHosts` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {"hosts": []}}}, + True, + "'The `hosts` empty list is not allowed for `metadata.data.allowedHosts` for certified connectors", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "allowedHosts": {"hosts": ["*.test.com"]}}}, + False, + None, + ), + ], + ids=[ + "No `allowedHosts`", + "Has `allowdHosts` but no `hosts`", + "Has `hosts` but it's empty list", + "Has non-empty `hosts`", + ] +) +async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise_assert_error, expected_error) -> None: + t = test_core.TestConnectorAttributes() + + if should_raise_assert_error: + with pytest.raises(AssertionError) as e: + await t.test_certified_connector_has_allowed_hosts( + operational_certification_test=True, + allowed_hosts_test=True, + connector_metadata=metadata_yaml + ) + assert expected_error in repr(e.value) + else: + await t.test_certified_connector_has_allowed_hosts( + operational_certification_test=True, + allowed_hosts_test=True, + connector_metadata=metadata_yaml + ) + + +@pytest.mark.parametrize( + "metadata_yaml, should_raise_assert_error, expected_error", + [ + pytest.param( + {"data": {"ab_internal": {"ql": 400}}}, + True, + "The `suggestedStreams` property is missing in `metadata.data` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {}}}, + True, + "The `streams` property is missing in `metadata.data.suggestedStreams` for `metadata.yaml`", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {"streams": []}}}, + True, + "'The `streams` empty list is not allowed for `metadata.data.suggestedStreams` for certified connectors", + ), + pytest.param( + {"data": {"ab_internal": {"ql": 400}, "suggestedStreams": {"streams": ["stream_1", "stream_2"]}}}, + False, + None, + ), + ], + ids=[ + "No `suggestedStreams`", + "Has `suggestedStreams` but no `streams`", + "Has `streams` but it's empty list", + "Has non-empty `streams`", + ] +) +async def test_certified_connector_has_suggested_streams(metadata_yaml, should_raise_assert_error, expected_error) -> None: + t = test_core.TestConnectorAttributes() + + if should_raise_assert_error: + with pytest.raises(AssertionError) as e: + await t.test_certified_connector_has_suggested_streams( + operational_certification_test=True, + suggested_streams_test=True, + connector_metadata=metadata_yaml + ) + assert expected_error in repr(e.value) + else: + await t.test_certified_connector_has_suggested_streams( + operational_certification_test=True, + suggested_streams_test=True, + connector_metadata=metadata_yaml + ) \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index ee017ba36c18..3e5dbda69f3e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -13,15 +13,25 @@ AirbyteMessage, AirbyteRecordMessage, AirbyteStream, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Level, + StreamDescriptor, SyncMode, TraceType, Type, ) -from connector_acceptance_test.config import BasicReadTestConfig, Config, ExpectedRecordsConfig, IgnoredFieldsConfiguration +from connector_acceptance_test.config import ( + BasicReadTestConfig, + Config, + ExpectedRecordsConfig, + FileTypesConfig, + IgnoredFieldsConfiguration, + UnsupportedFileTypeConfig, +) from connector_acceptance_test.tests import test_core from jsonschema.exceptions import SchemaError @@ -681,12 +691,14 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor expect_records_config=expect_records_config, should_validate_schema=True, should_validate_data_points=False, + should_validate_stream_statuses=False, should_fail_on_extra_columns=False, empty_streams=set(), expected_records_by_stream=expected_records_by_stream, docker_runner=docker_runner_mock, ignored_fields=ignored_fields, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) @@ -735,12 +747,14 @@ async def test_fail_on_extra_columns( expect_records_config=ExpectedRecordsConfig(path="foobar"), should_validate_schema=True, should_validate_data_points=False, + should_validate_stream_statuses=False, should_fail_on_extra_columns=config_fail_on_extra_columns, empty_streams=set(), expected_records_by_stream={}, docker_runner=docker_runner_mock, ignored_fields=None, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) else: t.test_read( @@ -749,12 +763,14 @@ async def test_fail_on_extra_columns( expect_records_config=ExpectedRecordsConfig(path="foobar"), should_validate_schema=True, should_validate_data_points=False, + should_validate_stream_statuses=False, should_fail_on_extra_columns=config_fail_on_extra_columns, empty_streams=set(), expected_records_by_stream={}, docker_runner=docker_runner_mock, ignored_fields=None, detailed_logger=MagicMock(), + certified_file_based_connector=False, ) @@ -1319,3 +1335,348 @@ def test_validate_field_appears_at_least_once(records, configured_catalog, expec t._validate_field_appears_at_least_once(records=records, configured_catalog=configured_catalog) else: t._validate_field_appears_at_least_once(records=records, configured_catalog=configured_catalog) + + +async def test_read_validate_async_output_stream_statuses(mocker): + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream.parse_obj({"name": f"test_stream_{x}", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), + sync_mode="full_refresh", + destination_sync_mode="overwrite", + ) + for x in range(3) + ] + ) + async_stream_output = [ + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.STARTED + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_2"), status=AirbyteStreamStatus.STARTED + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_1"), status=AirbyteStreamStatus.STARTED + ), + ), + ), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_1", data={"a": 1}, emitted_at=112)), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_2", data={"a": 1}, emitted_at=113)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=114, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_1"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=114, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_2"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=114, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=115, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_1"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=115, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_2"), status=AirbyteStreamStatus.COMPLETE + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=116, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_1"), status=AirbyteStreamStatus.COMPLETE + ), + ), + ), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=120, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.COMPLETE + ), + ), + ), + ] + docker_runner_mock = mocker.MagicMock(call_read=mocker.AsyncMock(return_value=async_stream_output)) + + t = test_core.TestBasicRead() + await t.test_read( + connector_config=None, + configured_catalog=configured_catalog, + expect_records_config=ExpectedRecordsConfig(path="foobar"), + should_validate_schema=False, + should_validate_data_points=False, + should_validate_stream_statuses=True, + should_fail_on_extra_columns=False, + empty_streams=set(), + expected_records_by_stream={}, + docker_runner=docker_runner_mock, + ignored_fields=None, + detailed_logger=MagicMock(), + certified_file_based_connector=False, + ) + + +@pytest.mark.parametrize( + "output", + [ + (AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)),), + ( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.STARTED + ), + ), + ), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)), + ), + ( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.STARTED + ), + ), + ), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=2, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + ), + ( + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=2, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.RUNNING + ), + ), + ), + ), + ( + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream_0", data={"a": 1}, emitted_at=111)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=2, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.COMPLETE + ), + ), + ), + ), + ], + ids=["no_statuses", "only_started_present", "only_started_and_running_present", "only_running", "only_complete"], +) +async def test_read_validate_stream_statuses_exceptions(mocker, output): + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream.parse_obj({"name": f"test_stream_0", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), + sync_mode="full_refresh", + destination_sync_mode="overwrite", + ) + ] + ) + docker_runner_mock = mocker.MagicMock(call_read=mocker.AsyncMock(return_value=output)) + + t = test_core.TestBasicRead() + with pytest.raises(AssertionError): + await t.test_read( + connector_config=None, + configured_catalog=configured_catalog, + expect_records_config=ExpectedRecordsConfig(path="foobar"), + should_validate_schema=False, + should_validate_data_points=False, + should_validate_stream_statuses=True, + should_fail_on_extra_columns=False, + empty_streams=set(), + expected_records_by_stream={}, + docker_runner=docker_runner_mock, + ignored_fields=None, + detailed_logger=MagicMock(), + certified_file_based_connector=False, + ) + + +@pytest.mark.parametrize( + "metadata, expected_file_based, is_connector_certified", + [ + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 400}}}, True, True), + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 500}}}, True, True), + ({}, False, False), + ({"data": {"ab_internal": {}}}, False, False), + ({"data": {"ab_internal": {"ql": 400}}}, False, False), + ({"data": {"connectorSubtype": "file"}}, False, False), + ({"data": {"connectorSubtype": "file", "ab_internal": {"ql": 200}}}, False, False), + ({"data": {"connectorSubtype": "not_file", "ab_internal": {"ql": 400}}}, False, False), + ], +) +def test_is_certified_file_based_connector(metadata, is_connector_certified, expected_file_based): + t = test_core.TestBasicRead() + assert test_core.TestBasicRead.is_certified_file_based_connector.__wrapped__(t, metadata, is_connector_certified) is expected_file_based + + +@pytest.mark.parametrize( + ("file_name", "expected_extension"), + ( + ("test.csv", ".csv"), + ("test/directory/test.csv", ".csv"), + ("test/directory/test.CSV", ".csv"), + ("test/directory/", ""), + (".bashrc", ""), + ("", ""), + ), +) +def test_get_file_extension(file_name, expected_extension): + t = test_core.TestBasicRead() + assert t._get_file_extension(file_name) == expected_extension + + +@pytest.mark.parametrize( + ("records", "expected_file_types"), + ( + ([], set()), + ( + [ + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test.csv"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_2.pdf"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_3.pdf"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": "test_3.CSV"}, emitted_at=111), + ], + {".csv", ".pdf"}, + ), + ( + [ + AirbyteRecordMessage(stream="stream", data={"field": "value"}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": ""}, emitted_at=111), + AirbyteRecordMessage(stream="stream", data={"field": "value", "_ab_source_file_url": ".bashrc"}, emitted_at=111), + ], + {""}, + ), + ), +) +def test_get_actual_file_types(records, expected_file_types): + t = test_core.TestBasicRead() + assert t._get_actual_file_types(records) == expected_file_types + + +@pytest.mark.parametrize( + ("config", "expected_file_types"), + ( + ([], set()), + ([UnsupportedFileTypeConfig(extension=".csv"), UnsupportedFileTypeConfig(extension=".pdf")], {".csv", ".pdf"}), + ([UnsupportedFileTypeConfig(extension=".CSV")], {".csv"}), + ), +) +def test_get_unsupported_file_types(config, expected_file_types): + t = test_core.TestBasicRead() + assert t._get_unsupported_file_types(config) == expected_file_types + + +@pytest.mark.parametrize( + ("is_file_based_connector", "skip_test"), + ((False, True), (False, False), (True, True)), +) +async def test_all_supported_file_types_present_skipped(mocker, is_file_based_connector, skip_test): + mocker.patch.object(test_core.pytest, "skip") + mocker.patch.object(test_core.TestBasicRead, "_file_types", {".avro", ".csv", ".jsonl", ".parquet", ".pdf"}) + + t = test_core.TestBasicRead() + config = BasicReadTestConfig(config_path="config_path", file_types=FileTypesConfig(skip_test=skip_test)) + await t.test_all_supported_file_types_present(is_file_based_connector, config) + test_core.pytest.skip.assert_called_once() + + +@pytest.mark.parametrize( + ("file_types_found", "should_fail"), + ( + ({".avro", ".csv", ".jsonl", ".parquet", ".pdf"}, False), + ({".csv", ".jsonl", ".parquet", ".pdf"}, True), + ({".avro", ".csv", ".jsonl", ".parquet"}, True), + ), +) +async def test_all_supported_file_types_present(mocker, file_types_found, should_fail): + mocker.patch.object(test_core.TestBasicRead, "_file_types", file_types_found) + t = test_core.TestBasicRead() + config = BasicReadTestConfig(config_path="config_path", file_types=FileTypesConfig(skip_test=False)) + + if should_fail: + with pytest.raises(AssertionError) as e: + await t.test_all_supported_file_types_present(certified_file_based_connector=True, inputs=config) + else: + await t.test_all_supported_file_types_present(certified_file_based_connector=True, inputs=config) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py new file mode 100644 index 000000000000..5a602e85a2e7 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py @@ -0,0 +1,155 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pathlib import Path + +import pytest +from airbyte_protocol.models import ConnectorSpecification +from connector_acceptance_test import conftest +from connector_acceptance_test.tests.test_core import TestConnectorDocumentation as _TestConnectorDocumentation + + +@pytest.mark.parametrize( + "connector_spec, docs_path, should_fail", + ( + # SUCCESS: required field from spec exists in Prerequisites section + ( + {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, + "data/docs/incorrect_not_all_structure.md", + False + ), + # FAIL: required field from spec does not exist in Prerequisites section + ( + {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, + "data/docs/incorrect_not_all_structure.md", + True + ) + ) +) +def test_documentation_prerequisites_section(connector_spec, docs_path, should_fail): + t = _TestConnectorDocumentation() + docs_path = Path(__file__).parent / docs_path + with open(docs_path, "r") as f: + documentation = f.read().rstrip() + + if should_fail is True: + with pytest.raises(AssertionError): + t.test_prerequisites_content(True, ConnectorSpecification(connectionSpecification=connector_spec), documentation, docs_path) + else: + t.test_prerequisites_content(True, ConnectorSpecification(connectionSpecification=connector_spec), documentation, docs_path) + + +@pytest.mark.parametrize( + "metadata, docs_path, should_fail, failure", + ( + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + "Missing headers:", + ), + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "Oracle Netsuite"}}, + "data/docs/with_not_required_steps.md", + True, + "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", + ), + # # SUCCESS: Docs follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + "", + ), + # Fail: Incorrect header order + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_header_order.md", + True, + "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", + ), + ) +) +def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, failure): + t = _TestConnectorDocumentation() + + docs_path = Path(__file__).parent / docs_path + with open(docs_path, "r") as f: + documentation = f.read().rstrip() + + if should_fail: + with pytest.raises(BaseException) as e: + t.test_docs_structure(True, documentation, metadata) + assert e.match(failure) + else: + t.test_docs_structure(True, documentation, metadata) + + +@pytest.mark.parametrize( + "metadata, docs_path, should_fail", + ( + # FAIL: Prerequisites section does not follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct_all_description_exist.md", + False, + ), + ) +) +def test_docs_description(mocker, metadata, docs_path, should_fail): + mocker.patch.object(conftest.pytest, "fail") + + t = _TestConnectorDocumentation() + + docs_path = Path(__file__).parent / docs_path + with open(docs_path, "r") as f: + documentation = f.read().rstrip() + + if should_fail is True: + with pytest.raises(AssertionError): + t.test_docs_descriptions(True, docs_path, documentation, metadata) + else: + t.test_docs_descriptions(True, docs_path, documentation, metadata) + + +@pytest.mark.parametrize( + ("docs_path", "should_fail"), + ( + ( + "data/docs/correct_all_description_exist.md", + False, + ), + ( + "data/docs/invalid_links.md", + True, + ), + ( + "data/docs/correct.md", + False, + ), + ) +) +def test_docs_urls(docs_path, should_fail): + t = _TestConnectorDocumentation() + docs_path = Path(__file__).parent / docs_path + with open(docs_path, "r") as f: + documentation = f.read().rstrip() + + if should_fail is True: + with pytest.raises(AssertionError): + t.test_validate_links(True, documentation) + else: + t.test_validate_links(True, documentation) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 3303c2650d1e..4b4b1d456e75 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -681,7 +681,7 @@ def test_enum_usage(connector_spec, should_fail): }, ), "", - ), + ) ], ) def test_validate_oauth_flow(connector_spec, expected_error): @@ -693,6 +693,369 @@ def test_validate_oauth_flow(connector_spec, expected_error): t.test_oauth_flow_parameters(connector_spec) +@pytest.mark.parametrize( + "connector_spec, expected_error", + [ + # FAIL: OAuth is not default + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + ] + } + } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "path_in_connector_config": ["api_url"] + } + } + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + }, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } + ), "Oauth method should be a default option. Current default method is access_token." + ), + # SUCCESS: Oauth is default + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + } + ] + } + } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "path_in_connector_config": ["api_url"] + } + } + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + }, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } + ), "" + ), + # SUCCESS: no advancedAuth specified + (ConnectorSpecification(connectionSpecification={}), ""), + # SUCCESS: only OAuth option to auth + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, + }, + }, + }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "", + ), + # SUCCESS: Credentials object does not have oneOf option. + ( + ConnectorSpecification( + connectionSpecification={"type": "object"}, + advanced_auth={ + "auth_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + }, + ), + "Credentials object does not have oneOf option.", + ), + # SUCCESS: Skipped: no predicate key. + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, + }, + }, + }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "Advanced Auth object does not have predicate_key, only one option to authenticate.", + ) + ] +) +def test_validate_auth_default_method(connector_spec, expected_error): + t = _TestSpec() + if expected_error: + with pytest.raises(AssertionError, match=expected_error): + t.test_oauth_is_default_method(skip_oauth_default_method_test=False, actual_connector_spec=connector_spec) + else: + t.test_oauth_is_default_method(skip_oauth_default_method_test=False, actual_connector_spec=connector_spec) + + @pytest.mark.parametrize( "connector_spec, expectation", [ @@ -740,6 +1103,73 @@ def test_additional_properties_is_true(connector_spec, expectation): ({"type": "object", "properties": {"refresh_token": {"type": "boolean", "airbyte_secret": True}}}, True), ({"type": "object", "properties": {"refresh_token": {"type": ["null", "string"]}}}, True), ({"type": "object", "properties": {"credentials": {"type": "array", "items": {"type": "string"}}}}, True), + ( + { + "type": "object", + "properties": {"credentials": {"type": "object", "properties": { + "client_secret": {"type": "string"}, + "access_token": {"type": "string", "airbyte_secret": True}}}} + }, + True + ), + ( + { + "type": "object", + "properties": {"credentials": {"type": "object", "properties": { + "client_secret": {"type": "string", "airbyte_secret": True}, + "access_token": {"type": "string", "airbyte_secret": True}}}} + }, + False + ), + ( + { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + } + } + }, + { + "type": "object", + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "access_token": { + "type": "string" + }, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + } + } + }, + ] + } + } + }, + True + ), ({"type": "object", "properties": {"auth": {"oneOf": [{"api_token": {"type": "string"}}]}}}, True), ( { @@ -757,7 +1187,7 @@ def test_airbyte_secret(mocker, connector_spec, should_fail): t = _TestSpec() logger = mocker.Mock() t.test_secret_is_properly_marked( - {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials") + {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret") ) if should_fail: conftest.pytest.fail.assert_called_once() diff --git a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs index fdd5c3deb969..5f67617a6fe2 100644 --- a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs @@ -4,7 +4,7 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: database diff --git a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs index 9f24ee5cdea5..170fee939b3f 100644 --- a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs @@ -4,7 +4,7 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorBuildOptions: diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index ab1d2a03a6bf..f1a97cc3f942 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -24,7 +24,6 @@ ${additionalMessage || ""} module.exports = function (plop) { const docRoot = '../../../docs/integrations'; - const definitionRoot = '../../../airbyte-config-oss/init-oss/src/main/resources'; const connectorAcceptanceTestFilesInputRoot = '../connector_acceptance_test_files'; @@ -296,12 +295,6 @@ module.exports = function (plop) { templateFile: `${javaDestinationInput}/.dockerignore.hbs`, path: `${javaDestinationOutputRoot}/.dockerignore` }, - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/Dockerfile.hbs`, - path: `${javaDestinationOutputRoot}/Dockerfile` - }, // Java { type: 'add', diff --git a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs index f162a414b44d..5ce3f8817b95 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs @@ -4,9 +4,13 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-{{dashCase name}} connectorBuildOptions: # Please update to the latest version of the connector base image. # https://hub.docker.com/r/airbyte/python-connector-base @@ -26,5 +30,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs index 4f8331bb191c..59f8bcf9b580 100644 --- a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs @@ -4,7 +4,7 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: api diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs index fa29cee2516d..5e325ba27e88 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs @@ -4,7 +4,7 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: database diff --git a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs index dc8ed8df1dc9..202f3973567d 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs index fdc68039f864..8d952455ab14 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs @@ -4,9 +4,13 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-{{dashCase name}} connectorBuildOptions: # Please update to the latest version of the connector base image. # https://hub.docker.com/r/airbyte/python-connector-base diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index 667a27713662..8f3eebe3cef1 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 000000000000..25c9400301f9 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +def run(): + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-python/main.py.hbs b/airbyte-integrations/connector-templates/source-python/main.py.hbs index dc8ed8df1dc9..202f3973567d 100644 --- a/airbyte-integrations/connector-templates/source-python/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs index fdc68039f864..ba39befbfce3 100644 --- a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs @@ -2,9 +2,13 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-{{dashCase name}} registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorBuildOptions: diff --git a/airbyte-integrations/connector-templates/source-python/setup.py.hbs b/airbyte-integrations/connector-templates/source-python/setup.py.hbs index 563d13c3708c..b16123258acb 100644 --- a/airbyte-integrations/connector-templates/source-python/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 000000000000..25c9400301f9 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +def run(): + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs index 629b17607a6b..f20c375e6065 100644 --- a/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs @@ -4,7 +4,7 @@ data: - TODO # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: api diff --git a/airbyte-integrations/connectors-performance/destination-harness/build.gradle b/airbyte-integrations/connectors-performance/destination-harness/build.gradle index 25332d02ca27..fd6bb54d0aed 100644 --- a/airbyte-integrations/connectors-performance/destination-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/destination-harness/build.gradle @@ -1,6 +1,5 @@ plugins { id 'application' - id 'airbyte-docker-legacy' } application { @@ -9,7 +8,7 @@ application { } dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' @@ -18,7 +17,6 @@ dependencies { implementation 'junit:junit:4.13.1' implementation 'org.testng:testng:7.1.0' implementation 'org.junit.jupiter:junit-jupiter:5.8.1' - } //This is only needed because we're using some very old libraries from airbyte-commons that were not packaged correctly diff --git a/airbyte-integrations/connectors-performance/source-harness/build.gradle b/airbyte-integrations/connectors-performance/source-harness/build.gradle index 2cdfcc461d3e..667e5af0a9a5 100644 --- a/airbyte-integrations/connectors-performance/source-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/source-harness/build.gradle @@ -1,6 +1,5 @@ plugins { id 'application' - id 'airbyte-docker-legacy' } application { @@ -9,7 +8,7 @@ application { } dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-integrations/connectors/build.gradle b/airbyte-integrations/connectors/build.gradle new file mode 100644 index 000000000000..9c9abf6535f8 --- /dev/null +++ b/airbyte-integrations/connectors/build.gradle @@ -0,0 +1,104 @@ +plugins { + id 'base' + id 'ru.vyarus.use-python' version '2.3.0' +} + +// Pyenv support. +try { + def pyenvRoot = "pyenv root".execute() + if (pyenvRoot.waitFor() == 0) { + ext.pyenvRoot = pyenvRoot.text.trim() + } +} catch (IOException _) { + // Swallow exception if pyenv is not installed. +} + +def pythonBin = layout.buildDirectory.file('.venv/bin/python').get().asFile.absolutePath + +// python is required by the connectors project to run airbyte-ci from source to build connector images. +python { + envPath = layout.buildDirectory.file('.venv').get().asFile + minPythonVersion = '3.10' // should be 3.10 for local development + + // Pyenv support. + try { + def pyenvRoot = "pyenv root".execute() + def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() + // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. + pythonBinary "python" + if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { + pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" + } + } catch (IOException _) { + // Swallow exception if pyenv is not installed. + } + + scope = 'VIRTUALENV' + installVirtualenv = true + // poetry is required for installing and running airbyte-ci + pip 'poetry:1.5.1' +} + +def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) { + workingDir rootProject.file('airbyte-ci/connectors/pipelines') + commandLine pythonBin + args "-m", "poetry", "install", "--no-cache" +} +poetryInstallAirbyteCI.configure { + dependsOn tasks.named('pipInstall') +} +def poetryCleanVirtualenv = tasks.register('cleanPoetryVirtualenv', Exec) { + workingDir rootProject.file('airbyte-ci/connectors/pipelines') + commandLine pythonBin + args "-m", "poetry", "env", "remove", "--all" + onlyIf { + layout.buildDirectory.file('venv/bin/python').get().asFile.exists() + } +} +tasks.named('clean').configure { + dependsOn poetryCleanVirtualenv +} + +allprojects { + // Evaluate CDK project before evaluating the connector. + evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk') + + // Adds airbyte-ci task. + def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs -> + def task = tasks.register(taskName, Exec) { + workingDir rootDir + environment "CI", "1" // set to use more suitable logging format + commandLine pythonBin + args "-m", "poetry" + args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}" + args "run" + args "airbyte-ci", "connectors", "--name=${project.name}" + args connectorsArgs + // Forbid these kinds of tasks from running concurrently. + // We can induce serial execution by giving them all a common output directory. + outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock") + outputs.upToDateWhen { false } + } + task.configure { dependsOn poetryInstallAirbyteCI } + return task + } + + // Build connector image as part of 'assemble' task. + // This is required for local 'integrationTest' execution. + def buildConnectorImage = airbyteCIConnectorsTask( + 'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar') + buildConnectorImage.configure { + // Images for java projects always rely on the distribution tarball. + dependsOn tasks.matching { it.name == 'distTar' } + // Ensure that all files exist beforehand. + dependsOn tasks.matching { it.name == 'generate' } + } + tasks.named('assemble').configure { + // We may revisit the dependency on assemble but the dependency should always be on a base task. + dependsOn buildConnectorImage + } + + // Convenience tasks for local airbyte-ci execution. + airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build') + airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test') +} diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile b/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile index 50e7598d2c3e..9861de2b6843 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile +++ b/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile @@ -34,5 +34,5 @@ COPY destination_amazon_sqs ./destination_amazon_sqs ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-amazon-sqs diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md b/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md index 37c5e934fad2..ce91ec1ef142 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md +++ b/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md @@ -11,7 +11,7 @@ have performance implications if sending high volumes of messages. #### Message Body By default, the SQS Message body is built using the AirbyteMessageRecord's 'data' property. -If the **message_body_key** config item is set, we use the value as a key within the the AirbyteMessageRecord's 'data' property. This could be +If the **message_body_key** config item is set, we use the value as a key within the AirbyteMessageRecord's 'data' property. This could be improved to handle nested keys by using JSONPath syntax to lookup values. For example, given the input Record: @@ -56,4 +56,4 @@ to use as a dedupe ID. ### Credentials Requires an AWS IAM Access Key ID and Secret Key. -This could be improved to add support for configured AWS profiles, env vars etc. \ No newline at end of file +This could be improved to add support for configured AWS profiles, env vars etc. diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json index ee0e2be338d5..f94d7d023e81 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json +++ b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json @@ -23,31 +23,39 @@ "description": "AWS Region of the SQS Queue", "type": "string", "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 1 }, diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml index 832eb8ca9454..8b6fa7635281 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: destination definitionId: 0eeee7fb-518f-4045-bacc-9619e31c43ea - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/destination-amazon-sqs githubIssueLabel: destination-amazon-sqs icon: awssqs.svg diff --git a/airbyte-integrations/connectors/destination-astra/README.md b/airbyte-integrations/connectors/destination-astra/README.md new file mode 100644 index 000000000000..2fa995b22593 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/README.md @@ -0,0 +1,159 @@ +# Astra Destination + +This is the repository for the Astra destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/astra). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/astra) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_astra/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination astra test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name destination-astra build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-astra:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-astra:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/destination-astra:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-astra:dev spec +```` +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-astra:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-astra:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-astra:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using `airbyte-ci` to run tests +See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command) + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml new file mode 100644 index 000000000000..75ab930dc09e --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/acceptance-test-config.yml @@ -0,0 +1,5 @@ +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json +connector_image: airbyte/destination-astra:dev diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py b/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py new file mode 100644 index 000000000000..1f125a4276a5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationAstra + +__all__ = ["DestinationAstra"] diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py b/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py new file mode 100644 index 000000000000..527c8345daa0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/astra_client.py @@ -0,0 +1,152 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from typing import Dict, List, Optional + +import requests +import urllib3 + + +class AstraClient: + def __init__( + self, + astra_endpoint: str, + astra_application_token: str, + keyspace_name: str, + embedding_dim: int, + similarity_function: str, + ): + self.astra_endpoint = astra_endpoint + self.astra_application_token = astra_application_token + self.keyspace_name = keyspace_name + self.embedding_dim = embedding_dim + self.similarity_function = similarity_function + + self.request_base_url = f"{self.astra_endpoint}/api/json/v1/{self.keyspace_name}" + self.request_header = { + "x-cassandra-token": self.astra_application_token, + "Content-Type": "application/json", + } + + def _run_query(self, request_url: str, query: Dict): + try: + response = requests.request("POST", request_url, headers=self.request_header, data=json.dumps(query)) + if response.status_code == 200: + response_dict = json.loads(response.text) + if "errors" in response_dict: + raise Exception(f"Astra DB request error - {response_dict['errors']}") + else: + return response_dict + else: + raise urllib3.exceptions.HTTPError(f"Astra DB not available. Status code: {response.status_code}, {response.text}") + except Exception: + raise + + def find_collections(self, include_detail: bool = True): + query = {"findCollections": {"options": {"explain": include_detail}}} + result = self._run_query(self.request_base_url, query) + + return result["status"]["collections"] + + def find_collection(self, collection_name: str): + collections = self.find_collections(False) + return collection_name in collections + + def create_collection(self, collection_name: str, embedding_dim: Optional[int] = None, similarity_function: Optional[str] = None): + query = { + "createCollection": { + "name": collection_name, + "options": { + "vector": { + "dimension": embedding_dim if embedding_dim is not None else self.embedding_dim, + "metric": similarity_function if similarity_function is not None else self.similarity_function, + } + }, + } + } + result = self._run_query(self.request_base_url, query) + + return True if result["status"]["ok"] == 1 else False + + def delete_collection(self, collection_name: str): + query = {"deleteCollection": {"name": collection_name}} + result = self._run_query(self.request_base_url, query) + + return True if result["status"]["ok"] == 1 else False + + def _build_collection_query(self, collection_name: str): + return f"{self.request_base_url}/{collection_name}" + + def find_documents( + self, + collection_name: str, + filter: Optional[Dict] = None, + vector: Optional[List[float]] = None, + limit: Optional[int] = None, + include_vector: Optional[bool] = None, + include_similarity: Optional[bool] = None, + ) -> List[Dict]: + find_query = {} + + if filter is not None: + find_query["filter"] = filter + + if vector is not None: + find_query["sort"] = {"$vector": vector} + + if include_vector is not None and include_vector == False: + find_query["projection"] = {"$vector": 0} + + if limit is not None: + find_query["options"] = {"limit": limit} + + if include_similarity is not None: + if "options" in find_query: + find_query["options"]["includeSimilarity"] = int(include_similarity) + else: + find_query["options"] = {"includeSimilarity": int(include_similarity)} + + query = {"find": find_query} + result = self._run_query(self._build_collection_query(collection_name), query) + return result["data"]["documents"] + + def insert_document(self, collection_name: str, document: Dict) -> str: + query = {"insertOne": {"document": document}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["insertedIds"][0] + + def insert_documents(self, collection_name: str, documents: List[Dict]) -> List[str]: + query = {"insertMany": {"documents": documents}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["insertedIds"] + + def update_document(self, collection_name: str, filter: Dict, update: Dict, upsert: bool = True) -> Dict: + query = {"findOneAndUpdate": {"filter": filter, "update": update, "options": {"returnDocument": "after", "upsert": upsert}}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"] + + def update_documents(self, collection_name: str, filter: Dict, update: Dict): + query = { + "updateMany": { + "filter": filter, + "update": update, + } + } + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"] + + def count_documents(self, collection_name: str): + query = {"countDocuments": {}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["count"] + + def delete_documents(self, collection_name: str, filter: Dict) -> int: + query = {"deleteMany": {"filter": filter}} + result = self._run_query(self._build_collection_query(collection_name), query) + + return result["status"]["deletedCount"] diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py new file mode 100644 index 000000000000..01d805ecd782 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel +from pydantic import BaseModel, Field + + +class AstraIndexingModel(BaseModel): + astra_db_app_token: str = Field( + ..., + title="Astra DB Application Token", + airbyte_secret=True, + description="The application token authorizes a user to connect to a specific Astra DB database. It is created when the user clicks the Generate Token button on the Overview tab of the Database page in the Astra UI.", + ) + astra_db_endpoint: str = Field( + ..., + title="Astra DB Endpoint", + description="The endpoint specifies which Astra DB database queries are sent to. It can be copied from the Database Details section of the Overview tab of the Database page in the Astra UI.", + pattern="^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", + examples=["https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"], + ) + astra_db_keyspace: str = Field( + ..., + title="Astra DB Keyspace", + description="Keyspaces (or Namespaces) serve as containers for organizing data within a database. You can create a new keyspace uisng the Data Explorer tab in the Astra UI. The keyspace default_keyspace is created for you when you create a Vector Database in Astra DB.", + ) + collection: str = Field( + ..., + title="Astra DB collection", + description="Collections hold data. They are analagous to tables in traditional Cassandra terminology. This tool will create the collection with the provided name automatically if it does not already exist. Alternatively, you can create one thorugh the Data Explorer tab in the Astra UI.", + ) + + class Config: + title = "Indexing" + schema_extra = { + "description": "Astra DB gives developers the APIs, real-time data and ecosystem integrations to put accurate RAG and Gen AI apps with fewer hallucinations in production.", + "group": "indexing", + } + + +class ConfigModel(VectorDBConfigModel): + indexing: AstraIndexingModel diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py new file mode 100644 index 000000000000..6fa1bd7ade5b --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_astra.config import ConfigModel +from destination_astra.indexer import AstraIndexer + +BATCH_SIZE = 100 + + +class DestinationAstra(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = create_from_config(config.embedding, config.processing) + self.indexer = AstraIndexer(config.indexing, self.embedder.embedding_dimensions) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/astra", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py new file mode 100644 index 000000000000..ee936ae1f5b4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Optional + +import urllib3 +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception +from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog, DestinationSyncMode +from destination_astra.astra_client import AstraClient +from destination_astra.config import AstraIndexingModel + +# do not flood the server with too many connections in parallel +PARALLELISM_LIMIT = 20 + +MAX_METADATA_SIZE = 40_960 - 10_000 + +MAX_IDS_PER_DELETE = 1000 + + +class AstraIndexer(Indexer): + config: AstraIndexingModel + + def __init__(self, config: AstraIndexingModel, embedding_dimensions: int): + super().__init__(config) + + self.client = AstraClient( + config.astra_db_endpoint, config.astra_db_app_token, config.astra_db_keyspace, embedding_dimensions, "cosine" + ) + + self.embedding_dimensions = embedding_dimensions + + def _create_collection(self): + if self.client.find_collection(self.config.collection) is False: + self.client.create_collection(self.config.collection) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + self._create_collection() + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self.client.delete_documents( + collection_name=self.config.collection, filter={METADATA_STREAM_FIELD: create_stream_identifier(stream.stream)} + ) + + def index(self, document_chunks, namespace, stream): + docs = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + metadata = chunk.metadata + if chunk.page_content is not None: + metadata["text"] = chunk.page_content + doc = { + "_id": str(uuid.uuid4()), + "$vector": chunk.embedding, + **metadata, + } + docs.append(doc) + serial_batches = create_chunks(docs, batch_size=PARALLELISM_LIMIT) + + for batch in serial_batches: + results = [chunk for chunk in batch] + self.client.insert_documents(collection_name=self.config.collection, documents=results) + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + self.client.delete_documents(collection_name=self.config.collection, filter={METADATA_RECORD_ID_FIELD: {"$in": delete_ids}}) + + def check(self) -> Optional[str]: + try: + self._create_collection() + collections = self.client.find_collections() + collection = next(filter(lambda f: f["name"] == self.config.collection, collections), None) + if collection is None: + return f"{self.config.collection} collection does not exist." + + actual_dimension = collection["options"]["vector"]["dimension"] + if actual_dimension != self.embedding_dimensions: + return f"Your embedding configuration will produce vectors with dimension {self.embedding_dimensions:d}, but your collection is configured with dimension {actual_dimension:d}. Make sure embedding and indexing configurations match." + except Exception as e: + if isinstance(e, urllib3.exceptions.MaxRetryError): + if "Failed to resolve 'apps.astra.datastax.com'" in str(e.reason): + return "Failed to resolve environment, please check whether the credential is correct." + if isinstance(e, urllib3.exceptions.HTTPError): + return str(e) + + formatted_exception = format_exception(e) + return formatted_exception + return None diff --git a/airbyte-integrations/connectors/destination-astra/icon.svg b/airbyte-integrations/connectors/destination-astra/icon.svg new file mode 100644 index 000000000000..ecc353976f51 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/icon.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py new file mode 100644 index 000000000000..b9d1aac8ae3c --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging + +from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_astra.astra_client import AstraClient +from destination_astra.config import ConfigModel +from destination_astra.destination import DestinationAstra + + +class AstraIntegrationTest(BaseIntegrationTest): + + def test_check_valid_config(self): + outcome = DestinationAstra().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + invalid_config = self.config + + invalid_config["embedding"]["openai_key"] = 123 + + outcome = DestinationAstra().check( + logging.getLogger("airbyte"), invalid_config) + assert outcome.status == Status.FAILED + + def test_write(self): + db_config = ConfigModel.parse_obj(self.config) + embedder = create_from_config(db_config.embedding, db_config.processing) + db_creds = db_config.indexing + astra_client = AstraClient( + db_creds.astra_db_endpoint, + db_creds.astra_db_app_token, + db_creds.astra_db_keyspace, + embedder.embedding_dimensions, + "cosine" + ) + + astra_client.delete_documents(collection_name=db_creds.collection, filter={}) + assert astra_client.count_documents(db_creds.collection) == 0 + + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + + message1 = self._record("mystream", "text data 1", 1) + message2 = self._record("mystream", "text data 2", 2) + + outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) + assert astra_client.count_documents(db_creds.collection) == 2 + diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json new file mode 100644 index 000000000000..35951290a06c --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json @@ -0,0 +1,377 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/astra", + "connectionSpecification": { + "title": "Destination Config", + "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", + "type": "object", + "properties": { + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string" + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string" + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text." + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string" + } + }, + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"] + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string" + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": true, + "type": "string" + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string" + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string" + } + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string" + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": true, + "type": "string" + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string" + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text." + } + ] + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { + "type": "string" + } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"] + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"] + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + }, + "indexing": { + "title": "Indexing", + "type": "object", + "properties": { + "astra_db_app_token": { + "title": "Astra DB Application Token", + "description": "The application token authorizes a user to connect to a specific Astra DB database. It is created when the user clicks the Generate Token button on the Overview tab of the Database page in the Astra UI.", + "airbyte_secret": true, + "type": "string" + }, + "astra_db_endpoint": { + "title": "Astra DB Endpoint", + "description": "The endpoint specifies which Astra DB database queries are sent to. It can be copied from the Database Details section of the Overview tab of the Database page in the Astra UI.", + "pattern": "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", + "examples": [ + "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com" + ], + "type": "string" + }, + "astra_db_keyspace": { + "title": "Astra DB Keyspace", + "description": "Keyspaces (or Namespaces) serve as containers for organizing data within a database. You can create a new keyspace uisng the Data Explorer tab in the Astra UI. The keyspace default_keyspace is created for you when you create a Vector Database in Astra DB.", + "type": "string" + }, + "collection": { + "title": "Astra DB collection", + "description": "Collections hold data. They are analagous to tables in traditional Cassandra terminology. This tool will create the collection with the provided name automatically if it does not already exist. Alternatively, you can create one thorugh the Data Explorer tab in the Astra UI.", + "type": "string" + } + }, + "required": [ + "astra_db_app_token", + "astra_db_endpoint", + "astra_db_keyspace", + "collection" + ], + "description": "Astra DB gives developers the APIs, real-time data and ecosystem integrations to put accurate RAG and Gen AI apps with fewer hallucinations in production.", + "group": "indexing" + } + }, + "required": ["embedding", "processing", "indexing"], + "groups": [ + { + "id": "processing", + "title": "Processing" + }, + { + "id": "embedding", + "title": "Embedding" + }, + { + "id": "indexing", + "title": "Indexing" + }, + { + "id": "advanced", + "title": "Advanced" + } + ] + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] +} diff --git a/airbyte-integrations/connectors/destination-astra/main.py b/airbyte-integrations/connectors/destination-astra/main.py new file mode 100644 index 000000000000..53b96b2b39ec --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_astra import DestinationAstra + +if __name__ == "__main__": + DestinationAstra().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-astra/metadata.yaml b/airbyte-integrations/connectors/destination-astra/metadata.yaml new file mode 100644 index 000000000000..c675ed875fdd --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/metadata.yaml @@ -0,0 +1,30 @@ +data: + allowedHosts: + hosts: + - "*.apps.astra.datastax.com" + registries: + oss: + enabled: true + cloud: + enabled: true + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # Please use the full address with sha256 hash to guarantee build reproducibility. + # https://hub.docker.com/r/airbyte/python-connector-base + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 + connectorSubtype: database + connectorType: destination + definitionId: 042ce96f-1158-4662-9543-e2ff015be97a + dockerImageTag: 0.1.1 + dockerRepository: airbyte/destination-astra + githubIssueLabel: destination-astra + icon: astra.svg + license: MIT + name: Astra DB + releaseDate: 2024-01-10 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/astra + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-airtable/requirements.txt b/airbyte-integrations/connectors/destination-astra/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-airtable/requirements.txt rename to airbyte-integrations/connectors/destination-astra/requirements.txt diff --git a/airbyte-integrations/connectors/destination-astra/setup.py b/airbyte-integrations/connectors/destination-astra/setup.py new file mode 100644 index 000000000000..8bd1a185b52e --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0"] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_astra", + description="Destination implementation for Astra.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py new file mode 100644 index 000000000000..f7d1400df709 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_astra.config import ConfigModel +from destination_astra.destination import DestinationAstra + + +class TestDestinationAstra(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "astra_db_app_token": "mytoken", + "astra_db_endpoint": "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com", + "astra_db_keyspace": "mykeyspace", + "collection": "mycollection", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedAstraIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedAstraIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationAstra() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedAstraIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedAstraIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationAstra() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_astra.destination.Writer") + @patch("destination_astra.destination.AstraIndexer") + @patch("destination_astra.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedAstraIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + mock_writer = Mock() + + MockedAstraIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationAstra() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=100, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationAstra() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py new file mode 100644 index 000000000000..ebb1d41e230a --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -0,0 +1,171 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from unittest.mock import ANY, MagicMock, Mock, patch + +import pytest +import urllib3 +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_astra.config import AstraIndexingModel +from destination_astra.indexer import AstraIndexer + + +def create_astra_indexer(): + config = AstraIndexingModel( + astra_db_app_token="mytoken", + astra_db_endpoint="https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com", + astra_db_keyspace="mykeyspace", + collection="mycollection", + ) + indexer = AstraIndexer(config, 3) + + indexer.client.delete_documents = MagicMock() + indexer.client.insert_documents = MagicMock() + indexer.client.find_documents = MagicMock() + + return indexer + + +def create_index_description(collection_name, dimensions): + return {"name": collection_name, "options": {"vector": {"dimension": dimensions, "metric": "cosine"}}} + + +def test_astra_index_upsert_and_delete(): + indexer = create_astra_indexer() + indexer.index( + [ + Mock(page_content="test", metadata={"_ab_stream": "abc"}, embedding=[1, 2, 3]), + Mock(page_content="test2", metadata={"_ab_stream": "abc"}, embedding=[4, 5, 6]), + ], + "ns1", + "some_stream", + ) + indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") + indexer.client.delete_documents.assert_called_with( + collection_name="mycollection", filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}} + ) + indexer.client.insert_documents.assert_called_with( + collection_name="mycollection", + documents=[ + {"_id": ANY, "$vector": [1, 2, 3], "_ab_stream": "abc", "text": "test"}, + {"_id": ANY, "$vector": [4, 5, 6], "_ab_stream": "abc", "text": "test2"}, + ], + ) + + +def test_astra_index_empty_batch(): + indexer = create_astra_indexer() + indexer.index([], "ns1", "some_stream") + indexer.client.delete_documents.assert_not_called() + indexer.client.insert_documents.assert_not_called() + + +def test_astra_index_upsert_batching(): + indexer = create_astra_indexer() + indexer.index( + [Mock(page_content=f"test {i}", metadata={"_ab_stream": "abc"}, embedding=[i, i, i]) for i in range(50)], + "ns1", + "some_stream", + ) + assert indexer.client.insert_documents.call_count == 3 + for i in range(20): + assert indexer.client.insert_documents.call_args_list[0].kwargs.get("documents")[i] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + for i in range(20, 40): + assert indexer.client.insert_documents.call_args_list[1].kwargs.get("documents")[i - 20] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + for i in range(40, 50): + assert indexer.client.insert_documents.call_args_list[2].kwargs.get("documents")[i - 40] == { + "_id": ANY, + "$vector": [i, i, i], + "_ab_stream": "abc", + "text": f"test {i}", + } + + +def generate_catalog(): + return ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns1", + }, + "primary_key": [["_id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns2", + }, + "primary_key": [["_id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + ) + + +def test_astra_pre_sync(): + indexer = create_astra_indexer() + indexer.client.find_collection = MagicMock(collection_name="") + indexer.client.find_collection.return_value = True + + indexer.pre_sync(generate_catalog()) + indexer.client.delete_documents.assert_called_with(collection_name="mycollection", filter={"_ab_stream": "ns2_example_stream2"}) + + +@pytest.mark.parametrize( + "collection_name,describe_throws,reported_dimensions,check_succeeds,error_message", + [ + ("mycollection", None, 3, True, None), + ("other_collection", None, 3, False, "mycollection collection does not exist."), + ( + ["mycollection"], + urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.")), + 3, + False, + "Failed to resolve environment", + ), + ("mycollection", None, 4, False, "Make sure embedding and indexing configurations match."), + ("mycollection", Exception("describe failed"), 3, False, "describe failed"), + ("mycollection", Exception("describe failed"), 4, False, "describe failed"), + ], +) +def test_astra_check(collection_name, describe_throws, reported_dimensions, check_succeeds, error_message): + indexer = create_astra_indexer() + + indexer.client.create_collection = MagicMock() + indexer.client.find_collections = MagicMock() + indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + + if describe_throws: + indexer.client.find_collections.side_effect = describe_throws + else: + indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + + result = indexer.check() + if check_succeeds: + assert result is None + else: + assert error_message in result diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py new file mode 100644 index 000000000000..219ae0142c72 --- /dev/null +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index 6e3c7234dde5..73d0a933e1c5 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -13,5 +13,5 @@ COPY destination_aws_datalake ./destination_aws_datalake ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json index cdf4a1de08c1..a868e1b3e5f1 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json @@ -90,31 +90,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 3 }, diff --git a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml index 9240657289d7..032954e7f2b9 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 99878c90-0fbd-46d3-9d98-ffde879d17fc - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-aws-datalake githubIssueLabel: destination-aws-datalake icon: awsdatalake.svg diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index d52bd144f6f1..f5d1b05d4b54 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -1,15 +1,24 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.7.0' - features = ['db-destinations', 's3-destinations'] + cdkVersionRequired = '0.20.9' + features = [ + 'db-destinations', + 'datastore-bigquery', + 'typing-deduping', + 'gcs-destinations', + ] useLocalCdk = false } -airbyteJavaConnector.addCdkDependencies() +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-this-escape" + } +} application { mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' @@ -26,41 +35,7 @@ application { ] } -airbyteJavaConnector.addCdkDependencies() - dependencies { - implementation project(':airbyte-integrations:connectors:destination-gcs') - - implementation 'com.google.cloud:google-cloud-bigquery:2.31.1' - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'org.apache.commons:commons-csv:1.4' + implementation 'com.codepoetics:protonpack:1.13' implementation 'org.apache.commons:commons-text:1.10.0' - - implementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.4.5' - implementation group: 'com.codepoetics', name: 'protonpack', version: '1.13' - - implementation (libs.airbyte.protocol) { - exclude group: 'io.airbyte', module: 'airbyte-commons' - } - // implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery') - - // TODO: declare typing-deduping as a CDK feature instead of importing from source. - implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) - - // TODO: remove these dependencies (what's S3 doing here???) - implementation libs.aws.java.sdk.s3 - implementation libs.s3 -} - -configurations.all { - resolutionStrategy { - // at time of writing: deps.toml declares google-cloud-storage 2.17.2 - // which pulls in google-api-client:2.2.0 - // which conflicts with google-cloud-bigquery, which requires google-api-client:1.x - // google-cloud-storage is OK with downgrading to anything >=1.31.1. - force 'com.google.api-client:google-api-client:1.31.5' - } } diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 8ad18978bfda..e3e73d6a5c98 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.3.20 + dockerImageTag: 2.4.11 dockerRepository: airbyte/destination-bigquery documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery githubIssueLabel: destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 78e6e782f71e..b884fe5dbd23 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -25,6 +25,10 @@ import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.gcs.BaseGcsDestination; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsNameTransformer; +import io.airbyte.cdk.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; @@ -44,10 +48,6 @@ import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.integrations.destination.gcs.GcsDestination; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.GcsNameTransformer; -import io.airbyte.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -67,7 +67,6 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTime; @@ -87,7 +86,6 @@ public class BigQueryDestination extends BaseConnector implements Destination { "storage.objects.delete", "storage.objects.get", "storage.objects.list"); - private static final ConcurrentMap randomSuffixMap = new ConcurrentHashMap<>(); protected final BigQuerySQLNameTransformer namingResolver; public BigQueryDestination() { @@ -157,7 +155,7 @@ private AirbyteConnectionStatus checkGcsPermission(final JsonNode config) { .map(i -> REQUIRED_PERMISSIONS.get(Math.toIntExact(i.getIndex()))) .toList()); - final GcsDestination gcsDestination = new GcsDestination(); + final BaseGcsDestination gcsDestination = new BaseGcsDestination() {}; final JsonNode gcsJsonNodeConfig = BigQueryUtils.getGcsJsonNodeConfig(config); return gcsDestination.check(gcsJsonNodeConfig); } catch (final Exception e) { @@ -192,16 +190,19 @@ public static BigQuery getBigQuery(final JsonNode config) { } public static GoogleCredentials getServiceAccountCredentials(final JsonNode config) throws IOException { - if (!BigQueryUtils.isUsingJsonCredentials(config)) { + final JsonNode serviceAccountKey = config.get(BigQueryConsts.CONFIG_CREDS); + // Follows this order of resolution: + // https://cloud.google.com/java/docs/reference/google-auth-library/latest/com.google.auth.oauth2.GoogleCredentials#com_google_auth_oauth2_GoogleCredentials_getApplicationDefault + if (serviceAccountKey == null) { LOGGER.info("No service account key json is provided. It is required if you are using Airbyte cloud."); LOGGER.info("Using the default service account credential from environment."); return GoogleCredentials.getApplicationDefault(); } // The JSON credential can either be a raw JSON object, or a serialized JSON object. - final String credentialsString = config.get(BigQueryConsts.CONFIG_CREDS).isObject() - ? Jsons.serialize(config.get(BigQueryConsts.CONFIG_CREDS)) - : config.get(BigQueryConsts.CONFIG_CREDS).asText(); + final String credentialsString = serviceAccountKey.isObject() + ? Jsons.serialize(serviceAccountKey) + : serviceAccountKey.asText(); return GoogleCredentials.fromStream( new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8))); } @@ -238,15 +239,20 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(config); final JsonNode serviceAccountKey = config.get(BigQueryConsts.CONFIG_CREDS); - if (serviceAccountKey.isTextual()) { - // There are cases where we fail to deserialize the service account key. In these cases, we - // shouldn't do anything. - // Google's creds library is more lenient with JSON-parsing than Jackson, and I'd rather just let it - // go. - Jsons.tryDeserialize(serviceAccountKey.asText()) - .ifPresent(AirbyteExceptionHandler::addAllStringsInConfigForDeinterpolation); - } else { - AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(serviceAccountKey); + if (serviceAccountKey != null) { + // If the service account key is a non-null string, we will try to + // deserialize it. Otherwise, we will let the Google library find it in + // the environment during the client initialization. + if (serviceAccountKey.isTextual()) { + // There are cases where we fail to deserialize the service account key. In these cases, we + // shouldn't do anything. + // Google's creds library is more lenient with JSON-parsing than Jackson, and I'd rather just let it + // go. + Jsons.tryDeserialize(serviceAccountKey.asText()) + .ifPresent(AirbyteExceptionHandler::addAllStringsInConfigForDeinterpolation); + } else { + AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(serviceAccountKey); + } } if (uploadingMethod == UploadingMethod.STANDARD) { @@ -294,10 +300,6 @@ protected Supplier { + (hasFailed, streamSyncSummaries) -> { try { Thread.sleep(30 * 1000); - typerDeduper.typeAndDedupe(); + typerDeduper.typeAndDedupe(streamSyncSummaries); typerDeduper.commitFinalTables(); typerDeduper.cleanup(); } catch (final Exception e) { @@ -456,6 +457,11 @@ private TyperDeduper buildTyperDeduper(final BigQuerySqlGenerator sqlGenerator, } + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { AirbyteExceptionHandler.addThrowableForDeinterpolation(BigQueryException.class); final Destination destination = new BigQueryDestination(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index a6f4f9a8c0cf..6e88b5970b97 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -14,11 +14,11 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.GcsStorageOperations; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -114,7 +114,7 @@ public void createStageIfNotExists(final String datasetId, final String stream) public String uploadRecordsToStage(final String datasetId, final String stream, final SerializableBuffer writer) { final String objectPath = getStagingFullPath(datasetId, stream); LOGGER.info("Uploading records to staging for stream {} (dataset {}): {}", stream, datasetId, objectPath); - return gcsStorageOperations.uploadRecordsToBucket(writer, datasetId, getStagingRootPath(datasetId, stream), objectPath); + return gcsStorageOperations.uploadRecordsToBucket(writer, datasetId, objectPath); } /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java index 9045b353fcf0..3102c1da1089 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java @@ -41,6 +41,7 @@ public String getNamespace(final String input) { return normalizedName; } + @Deprecated public String getTmpTableName(final String streamName, final String randomSuffix) { return convertStreamName("_airbyte_tmp" + "_" + randomSuffix + "_" + streamName); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java index dfab0f81a24e..a929bfbf095f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java @@ -61,10 +61,10 @@ public SerializedAirbyteMessageConsumer createAsync( return new AsyncStreamConsumer( outputRecordCollector, onStartFunction(bigQueryGcsOperations, writeConfigsByDescriptor, typerDeduper), - (hasFailed) -> { + (hasFailed, recordCounts) -> { try { - onCloseFunction(bigQueryGcsOperations, writeConfigsByDescriptor, typerDeduper).accept(hasFailed); - } catch (Exception e) { + onCloseFunction(bigQueryGcsOperations, writeConfigsByDescriptor, typerDeduper).accept(hasFailed, recordCounts); + } catch (final Exception e) { throw new RuntimeException(e); } }, @@ -169,13 +169,13 @@ private OnStartFunction onStartFunction(final BigQueryStagingOperations bigQuery private OnCloseFunction onCloseFunction(final BigQueryStagingOperations bigQueryGcsOperations, final Map writeConfigs, final TyperDeduper typerDeduper) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { /* * Previously the hasFailed value was used to commit any remaining staged files into destination, * however, with the changes to checkpointing this will no longer be necessary since despite partial * successes, we'll be committing the target table (aka airbyte_raw) table throughout the sync */ - typerDeduper.typeAndDedupe(); + typerDeduper.typeAndDedupe(streamSyncSummaries); LOGGER.info("Cleaning up destination started for {} streams", writeConfigs.size()); for (final Map.Entry entry : writeConfigs.entrySet()) { bigQueryGcsOperations.dropStageIfExists(entry.getValue().datasetId(), entry.getValue().streamName()); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 9516f4befc61..82b3d67a5c7d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -19,6 +19,7 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobId; @@ -37,9 +38,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.time.Instant; import java.time.LocalDateTime; @@ -106,14 +107,12 @@ static Job waitForQuery(final Job queryJob) { public static void createSchemaAndTableIfNeeded(final BigQuery bigquery, final Set existingSchemas, final String schemaName, - final TableId tmpTableId, final String datasetLocation, final Schema schema) { if (!existingSchemas.contains(schemaName)) { getOrCreateDataset(bigquery, schemaName, datasetLocation); existingSchemas.add(schemaName); } - BigQueryUtils.createPartitionedTableIfNotExists(bigquery, tmpTableId, schema); } public static Dataset getOrCreateDataset(final BigQuery bigquery, final String datasetId, final String datasetLocation) { @@ -161,13 +160,15 @@ private static void attemptCreateTableAndTestInsert(final BigQuery bigquery, fin CHECK_TEST_TMP_TABLE_NAME, testTableSchema); // Try to make test (dummy records) insert to make sure that user has required permissions + // Use ids for BigQuery client to attempt idempotent retries. + // See https://github.com/airbytehq/airbyte/issues/33982 try { final InsertAllResponse response = bigquery.insertAll(InsertAllRequest .newBuilder(test_connection_table_name) - .addRow(Map.of("id", 1, "name", "James")) - .addRow(Map.of("id", 2, "name", "Eugene")) - .addRow(Map.of("id", 3, "name", "Angelina")) + .addRow(RowToInsert.of("1", ImmutableMap.of("id", 1, "name", "James"))) + .addRow(RowToInsert.of("2", ImmutableMap.of("id", 2, "name", "Eugene"))) + .addRow(RowToInsert.of("3", ImmutableMap.of("id", 3, "name", "Angelina"))) .build()); if (response.hasErrors()) { @@ -177,6 +178,7 @@ private static void attemptCreateTableAndTestInsert(final BigQuery bigquery, fin } } } catch (final BigQueryException e) { + LOGGER.error("Dummy inserts in check failed", e); throw new ConfigErrorException("Failed to check connection: \n" + e.getMessage()); } finally { test_connection_table_name.delete(); @@ -394,18 +396,6 @@ public static JobInfo.WriteDisposition getWriteDisposition(final DestinationSync } } - public static boolean isUsingJsonCredentials(final JsonNode config) { - if (!config.has(BigQueryConsts.CONFIG_CREDS)) { - return false; - } - final JsonNode json = config.get(BigQueryConsts.CONFIG_CREDS); - if (json.isTextual()) { - return !json.asText().isEmpty(); - } else { - return !Jsons.serialize(json).isEmpty(); - } - } - // https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html public static Integer getBigQueryClientChunkSize(final JsonNode config) { Integer chunkSizeFromConfig = null; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java index 8a19dcdc6d46..8199165d6527 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -17,14 +17,15 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableResult; import com.google.common.collect.Streams; import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.math.BigInteger; -import java.time.Instant; import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -52,57 +53,73 @@ public Optional findExistingTable(final StreamId id) { return Optional.ofNullable(table).map(Table::getDefinition); } + @Override + public LinkedHashMap findExistingFinalTables(List streamIds) throws Exception { + return null; + } + @Override public boolean isFinalTableEmpty(final StreamId id) { return BigInteger.ZERO.equals(bq.getTable(TableId.of(id.finalNamespace(), id.finalName())).getNumRows()); } @Override - public Optional getMinTimestampForSync(final StreamId id) throws Exception { + public InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { final Table rawTable = bq.getTable(TableId.of(id.rawNamespace(), id.rawName())); if (rawTable == null) { - return Optional.empty(); + // Table doesn't exist. There are no unprocessed records, and no timestamp. + return new InitialRawTableState(false, Optional.empty()); } - final TableResult queryResult = bq.query(QueryJobConfiguration.newBuilder(new StringSubstitutor(Map.of( + + final FieldValue unloadedRecordTimestamp = bq.query(QueryJobConfiguration.newBuilder(new StringSubstitutor(Map.of( "raw_table", id.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( // bigquery timestamps have microsecond precision - // and COALESCE short-circuits, so if the first subquery returns non-null, we don't - // evaluate the second query at all """ - SELECT COALESCE( - ( - SELECT TIMESTAMP_SUB(MIN(_airbyte_extracted_at), INTERVAL 1 MICROSECOND) - FROM ${raw_table} - WHERE _airbyte_loaded_at IS NULL - ), - ( - SELECT MAX(_airbyte_extracted_at) - FROM ${raw_table} - ) - ) + SELECT TIMESTAMP_SUB(MIN(_airbyte_extracted_at), INTERVAL 1 MICROSECOND) + FROM ${raw_table} + WHERE _airbyte_loaded_at IS NULL + """)) + .build()).iterateAll().iterator().next().get(0); + // If this value is null, then there are no records with null loaded_at. + // If it's not null, then we can return immediately - we've found some unprocessed records and their + // timestamp. + if (!unloadedRecordTimestamp.isNull()) { + return new InitialRawTableState(true, Optional.of(unloadedRecordTimestamp.getTimestampInstant())); + } + + final FieldValue loadedRecordTimestamp = bq.query(QueryJobConfiguration.newBuilder(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( + """ + SELECT MAX(_airbyte_extracted_at) + FROM ${raw_table} """)) - .build()); - final FieldValue value = queryResult.iterateAll().iterator().next().get(0); - if (value.isNull()) { - return Optional.empty(); + .build()).iterateAll().iterator().next().get(0); + // We know (from the previous query) that all records have been processed by T+D already. + // So we just need to get the timestamp of the most recent record. + if (loadedRecordTimestamp.isNull()) { + // Null timestamp because the table is empty. T+D can process the entire raw table during this sync. + return new InitialRawTableState(false, Optional.empty()); } else { - return Optional.ofNullable(value.getTimestampInstant()); + // The raw table already has some records. T+D can skip all records with timestamp <= this value. + return new InitialRawTableState(false, Optional.of(loadedRecordTimestamp.getTimestampInstant())); } } @Override - public void execute(final String sql) throws InterruptedException { - if ("".equals(sql)) { + public void execute(final Sql sql) throws InterruptedException { + final List transactions = sql.asSqlStrings("BEGIN TRANSACTION", "COMMIT TRANSACTION"); + if (transactions.isEmpty()) { return; } final UUID queryId = UUID.randomUUID(); - LOGGER.debug("Executing sql {}: {}", queryId, sql); + final String statement = String.join("\n", transactions); + LOGGER.debug("Executing sql {}: {}", queryId, statement); /* * If you run a query like CREATE SCHEMA ... OPTIONS(location=foo); CREATE TABLE ...;, bigquery * doesn't do a good job of inferring the query location. Pass it in explicitly. */ - Job job = bq.create(JobInfo.of(JobId.newBuilder().setLocation(datasetLocation).build(), QueryJobConfiguration.newBuilder(sql).build())); + Job job = bq.create(JobInfo.of(JobId.newBuilder().setLocation(datasetLocation).build(), QueryJobConfiguration.newBuilder(statement).build())); AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); // job.waitFor() gets stuck forever in some failure cases, so manually poll the job instead. while (!JobStatus.State.DONE.equals(job.getStatus().getState())) { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java index 8a0d85c71a59..c4370fc5dc0a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java @@ -7,6 +7,8 @@ import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsAllIgnoreCase; import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsIgnoreCase; import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.matchingKey; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.separately; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; import static io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction.SOFT_RESET_SUFFIX; import static java.util.stream.Collectors.joining; @@ -23,6 +25,7 @@ import io.airbyte.integrations.base.destination.typing_deduping.AlterTableReport; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; @@ -210,25 +213,21 @@ public StandardSQLTypeName toDialectType(final AirbyteProtocolType airbyteProtoc } @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { final String columnDeclarations = columnsAndTypes(stream); final String clusterConfig = clusteringColumns(stream).stream() .map(c -> StringUtils.wrap(c, QUOTE)) .collect(joining(", ")); final String forceCreateTable = force ? "OR REPLACE" : ""; - return new StringSubstitutor(Map.of( + return Sql.of(new StringSubstitutor(Map.of( "project_id", '`' + projectId + '`', "final_namespace", stream.id().finalNamespace(QUOTE), - "dataset_location", datasetLocation, "force_create_table", forceCreateTable, "final_table_id", stream.id().finalTableId(QUOTE, suffix), "column_declarations", columnDeclarations, "cluster_config", clusterConfig)).replace( """ - CREATE SCHEMA IF NOT EXISTS ${project_id}.${final_namespace} - OPTIONS(location="${dataset_location}"); - CREATE ${force_create_table} TABLE ${project_id}.${final_table_id} ( _airbyte_raw_id STRING NOT NULL, _airbyte_extracted_at TIMESTAMP NOT NULL, @@ -237,7 +236,7 @@ public String createTable(final StreamConfig stream, final String suffix, final ) PARTITION BY (DATE_TRUNC(_airbyte_extracted_at, DAY)) CLUSTER BY ${cluster_config}; - """); + """)); } private List clusteringColumns(final StreamConfig stream) { @@ -363,8 +362,9 @@ public static boolean schemaContainAllFinalTableV2AirbyteColumns(final Collectio } @Override - public String prepareTablesForSoftReset(final StreamConfig stream) { - return String.join("\n", List.of( + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + // Bigquery can't run DDL in a transaction, so these are separate transactions. + return Sql.concat( // If a previous sync failed to delete the soft reset temp table (unclear why this happens), // AND this sync is trying to change the clustering config, then we need to manually drop the soft // reset temp table. @@ -373,33 +373,33 @@ public String prepareTablesForSoftReset(final StreamConfig stream) { // So we explicitly drop the soft reset temp table first. dropTableIfExists(stream, SOFT_RESET_SUFFIX), createTable(stream, SOFT_RESET_SUFFIX, true), - clearLoadedAt(stream.id()))); + clearLoadedAt(stream.id())); } - public String dropTableIfExists(final StreamConfig stream, final String suffix) { - return new StringSubstitutor(Map.of( + public Sql dropTableIfExists(final StreamConfig stream, final String suffix) { + return Sql.of(new StringSubstitutor(Map.of( "project_id", '`' + projectId + '`', "table_id", stream.id().finalTableId(QUOTE, suffix))) .replace(""" DROP TABLE IF EXISTS ${project_id}.${table_id}; - """); + """)); } @Override - public String clearLoadedAt(final StreamId streamId) { - return new StringSubstitutor(Map.of( + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(new StringSubstitutor(Map.of( "project_id", '`' + projectId + '`', "raw_table_id", streamId.rawTableId(QUOTE))) .replace(""" UPDATE ${project_id}.${raw_table_id} SET _airbyte_loaded_at = NULL WHERE 1=1; - """); + """)); } @Override - public String updateTable(final StreamConfig stream, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { final String handleNewRecords; if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { handleNewRecords = upsertNewRecords(stream, finalSuffix, useExpensiveSaferCasting, minRawTimestamp); @@ -408,15 +408,7 @@ public String updateTable(final StreamConfig stream, } final String commitRawTable = commitRawTable(stream.id(), minRawTimestamp); - return new StringSubstitutor(Map.of( - "handleNewRecords", handleNewRecords, - "commit_raw_table", commitRawTable)).replace( - """ - BEGIN TRANSACTION; - ${handleNewRecords} - ${commit_raw_table} - COMMIT TRANSACTION; - """); + return transactionally(handleNewRecords, commitRawTable); } private String insertNewRecords(final StreamConfig stream, @@ -690,16 +682,15 @@ String commitRawTable(final StreamId id, final Optional minRawTimestamp } @Override - public String overwriteFinalTable(final StreamId streamId, final String finalSuffix) { - return new StringSubstitutor(Map.of( + public Sql overwriteFinalTable(final StreamId streamId, final String finalSuffix) { + final StringSubstitutor substitutor = new StringSubstitutor(Map.of( "project_id", '`' + projectId + '`', "final_table_id", streamId.finalTableId(QUOTE), "tmp_final_table", streamId.finalTableId(QUOTE, finalSuffix), - "real_final_table", streamId.finalName(QUOTE))).replace( - """ - DROP TABLE IF EXISTS ${project_id}.${final_table_id}; - ALTER TABLE ${project_id}.${tmp_final_table} RENAME TO ${real_final_table}; - """); + "real_final_table", streamId.finalName(QUOTE))); + return separately( + substitutor.replace("DROP TABLE IF EXISTS ${project_id}.${final_table_id};"), + substitutor.replace("ALTER TABLE ${project_id}.${tmp_final_table} RENAME TO ${real_final_table};")); } private String wrapAndQuote(final String namespace, final String tableName) { @@ -709,17 +700,20 @@ private String wrapAndQuote(final String namespace, final String tableName) { } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - return new StringSubstitutor(Map.of( + public Sql createSchema(final String schema) { + return Sql.of(new StringSubstitutor(Map.of("schema", StringUtils.wrap(schema, QUOTE), + "project_id", StringUtils.wrap(projectId, QUOTE), + "dataset_location", datasetLocation)) + .replace("CREATE SCHEMA IF NOT EXISTS ${project_id}.${schema} OPTIONS(location=\"${dataset_location}\");")); + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return Sql.of(new StringSubstitutor(Map.of( "project_id", '`' + projectId + '`', - "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), - "dataset_location", datasetLocation, "v2_raw_table", streamId.rawTableId(QUOTE), "v1_raw_table", wrapAndQuote(namespace, tableName))).replace( """ - CREATE SCHEMA IF NOT EXISTS ${project_id}.${raw_namespace} - OPTIONS(location="${dataset_location}"); - CREATE OR REPLACE TABLE ${project_id}.${v2_raw_table} ( _airbyte_raw_id STRING, _airbyte_data STRING, @@ -736,7 +730,7 @@ PARTITION BY DATE(_airbyte_extracted_at) CAST(NULL AS TIMESTAMP) AS _airbyte_loaded_at FROM ${project_id}.${v1_raw_table} ); - """); + """)); } /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java index 68eec640a2b3..34b425cae7f5 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java @@ -38,20 +38,17 @@ public abstract class AbstractBigQueryUploader { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractBigQueryUploader.class); protected final TableId table; - protected final TableId tmpTable; protected final WriteDisposition syncMode; protected final T writer; protected final BigQuery bigQuery; protected final BigQueryRecordFormatter recordFormatter; AbstractBigQueryUploader(final TableId table, - final TableId tmpTable, final T writer, final WriteDisposition syncMode, final BigQuery bigQuery, final BigQueryRecordFormatter recordFormatter) { this.table = table; - this.tmpTable = tmpTable; this.writer = writer; this.syncMode = syncMode; this.bigQuery = bigQuery; @@ -120,8 +117,6 @@ protected void uploadData(final Consumer outputRecordCollector, } catch (final Exception e) { LOGGER.error("Upload data is failed!"); throw e; - } finally { - dropTmpTable(); } } @@ -137,25 +132,6 @@ public void createRawTable() { } } - protected void dropTmpTable() { - try { - // clean up tmp tables; - LOGGER.info("Removing tmp tables..."); - bigQuery.delete(tmpTable); - LOGGER.info("Finishing destination process...completed"); - } catch (final Exception e) { - LOGGER.error("Fail to tmp table drop table: " + e.getMessage()); - } - } - - protected void uploadDataToTableFromTmpTable() { - LOGGER.info("Replication finished with no explicit errors. Copying data from tmp tables to permanent"); - if (syncMode.equals(JobInfo.WriteDisposition.WRITE_APPEND)) { - partitionIfUnpartitioned(bigQuery, recordFormatter.getBigQuerySchema(), table); - } - copyTable(bigQuery, tmpTable, table, syncMode); - } - /** * Creates a partitioned table if the table previously was not partitioned * @@ -265,7 +241,6 @@ private static String getCreatePartitionedTableFromSelectQuery(final Schema sche public String toString() { return "AbstractBigQueryUploader{" + "table=" + table.getTable() + - ", tmpTable=" + tmpTable.getTable() + ", syncMode=" + syncMode + ", writer=" + writer.getClass() + ", recordFormatter=" + recordFormatter.getClass() + diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java deleted file mode 100644 index 8fe7f721d983..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.uploader; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.JobInfo.WriteDisposition; -import com.google.cloud.bigquery.LoadJobConfiguration; -import com.google.cloud.bigquery.TableId; -import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.bigquery.BigQueryUtils; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import java.util.List; -import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractGscBigQueryUploader extends AbstractBigQueryUploader { - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractGscBigQueryUploader.class); - - private final boolean isKeepFilesInGcs; - protected final GcsDestinationConfig gcsDestinationConfig; - - AbstractGscBigQueryUploader(final TableId table, - final TableId tmpTable, - final T writer, - final WriteDisposition syncMode, - final GcsDestinationConfig gcsDestinationConfig, - final BigQuery bigQuery, - final boolean isKeepFilesInGcs, - final BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, bigQuery, recordFormatter); - this.isKeepFilesInGcs = isKeepFilesInGcs; - this.gcsDestinationConfig = gcsDestinationConfig; - } - - @Override - public void postProcessAction(final boolean hasFailed) { - if (!isKeepFilesInGcs) { - deleteGcsFiles(); - } - } - - @Override - protected void uploadData(final Consumer outputRecordCollector, final AirbyteMessage lastStateMessage) throws Exception { - LOGGER.info("Uploading data to the tmp table {}.", tmpTable.getTable()); - uploadDataFromFileToTmpTable(); - super.uploadData(outputRecordCollector, lastStateMessage); - } - - protected void uploadDataFromFileToTmpTable() { - try { - final String fileLocation = this.writer.getFileLocation(); - - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - LOGGER.info(String.format("Started copying data from %s GCS " + getFileTypeName() + " file to %s tmp BigQuery table with schema: \n %s", - fileLocation, tmpTable, recordFormatter.getBigQuerySchema())); - - final LoadJobConfiguration configuration = getLoadConfiguration(); - - // For more information on Job see: - // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html - // Load the table - final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); - LOGGER.info("Created a new job GCS " + getFileTypeName() + " file to tmp BigQuery table: " + loadJob); - - // Load data from a GCS parquet file into the table - // Blocks until this load table job completes its execution, either failing or succeeding. - BigQueryUtils.waitForJobFinish(loadJob); - - LOGGER.info("Table is successfully overwritten by file loaded from GCS: {}", getFileTypeName()); - } catch (final BigQueryException | InterruptedException e) { - LOGGER.error("Column not added during load append", e); - throw new RuntimeException("Column not added during load append \n" + e.toString()); - } - } - - abstract protected LoadJobConfiguration getLoadConfiguration(); - - private String getFileTypeName() { - return writer.getFileFormat().getFileExtension(); - } - - private void deleteGcsFiles() { - LOGGER.info("Deleting file {}", writer.getFileLocation()); - final GcsDestinationConfig gcsDestinationConfig = this.gcsDestinationConfig; - final AmazonS3 s3Client = gcsDestinationConfig.getS3Client(); - - final String gcsBucketName = gcsDestinationConfig.getBucketName(); - final String gcs_bucket_path = gcsDestinationConfig.getBucketPath(); - - final List objects = s3Client - .listObjects(gcsBucketName, gcs_bucket_path) - .getObjectSummaries(); - - objects.stream().filter(s3ObjectSummary -> s3ObjectSummary.getKey().equals(writer.getOutputPath())).forEach(s3ObjectSummary -> { - s3Client.deleteObject(gcsBucketName, new DeleteObjectsRequest.KeyVersion(s3ObjectSummary.getKey()).getKey()); - LOGGER.info("File is deleted : " + s3ObjectSummary.getKey()); - }); - s3Client.shutdown(); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java index 81a4641395ff..2a464e366645 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.bigquery.uploader; import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.TableId; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; @@ -16,12 +16,11 @@ public class BigQueryDirectUploader extends AbstractBigQueryUploader { public BigQueryDirectUploader(final TableId table, - final TableId tmpTable, final BigQueryTableWriter writer, - final JobInfo.WriteDisposition syncMode, + final WriteDisposition syncMode, final BigQuery bigQuery, final BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, bigQuery, recordFormatter); + super(table, writer, syncMode, bigQuery, recordFormatter); } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java index d2a1f84bdba5..6eca8c9f947e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java @@ -4,15 +4,13 @@ package io.airbyte.integrations.destination.bigquery.uploader; -import static software.amazon.awssdk.http.HttpStatusCode.FORBIDDEN; -import static software.amazon.awssdk.http.HttpStatusCode.NOT_FOUND; - import com.fasterxml.jackson.databind.JsonNode; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableDataWriteChannel; import com.google.cloud.bigquery.TableId; @@ -32,6 +30,9 @@ public class BigQueryUploaderFactory { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUploaderFactory.class); + private static final int HTTP_STATUS_CODE_FORBIDDEN = 403; + private static final int HTTP_STATUS_CODE_NOT_FOUND = 404; + private static final String CONFIG_ERROR_MSG = """ Failed to write to destination schema. @@ -56,13 +57,11 @@ public static AbstractBigQueryUploader getUploader(final UploaderConfig uploa final Schema bigQuerySchema = recordFormatter.getBigQuerySchema(); final TableId targetTable = TableId.of(dataset, uploaderConfig.getTargetTableName()); - final TableId tmpTable = TableId.of(dataset, uploaderConfig.getTmpTableName()); BigQueryUtils.createSchemaAndTableIfNeeded( uploaderConfig.getBigQuery(), existingDatasets, dataset, - tmpTable, datasetLocation, bigQuerySchema); @@ -72,7 +71,6 @@ public static AbstractBigQueryUploader getUploader(final UploaderConfig uploa return getBigQueryDirectUploader( uploaderConfig.getConfig(), targetTable, - tmpTable, uploaderConfig.getBigQuery(), syncMode, datasetLocation, @@ -82,9 +80,8 @@ public static AbstractBigQueryUploader getUploader(final UploaderConfig uploa private static BigQueryDirectUploader getBigQueryDirectUploader( final JsonNode config, final TableId targetTable, - final TableId tmpTable, final BigQuery bigQuery, - final JobInfo.WriteDisposition syncMode, + final WriteDisposition syncMode, final String datasetLocation, final BigQueryRecordFormatter formatter) { // https://cloud.google.com/bigquery/docs/loading-data-local#loading_data_from_a_local_data_source @@ -107,7 +104,7 @@ private static BigQueryDirectUploader getBigQueryDirectUploader( try { writer = bigQuery.writer(job, writeChannelConfiguration); } catch (final BigQueryException e) { - if (e.getCode() == FORBIDDEN || e.getCode() == NOT_FOUND) { + if (e.getCode() == HTTP_STATUS_CODE_FORBIDDEN || e.getCode() == HTTP_STATUS_CODE_NOT_FOUND) { throw new ConfigErrorException(CONFIG_ERROR_MSG + e); } else { throw new BigQueryException(e.getCode(), e.getMessage()); @@ -123,7 +120,6 @@ private static BigQueryDirectUploader getBigQueryDirectUploader( return new BigQueryDirectUploader( targetTable, - tmpTable, new BigQueryTableWriter(writer), syncMode, bigQuery, diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java deleted file mode 100644 index 9abbe21565b2..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.uploader; - -import static com.amazonaws.util.StringUtils.UTF8; - -import com.google.cloud.bigquery.*; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.csv.GcsCsvWriter; - -public class GcsCsvBigQueryUploader extends AbstractGscBigQueryUploader { - - public GcsCsvBigQueryUploader(TableId table, - TableId tmpTable, - GcsCsvWriter writer, - JobInfo.WriteDisposition syncMode, - GcsDestinationConfig gcsDestinationConfig, - BigQuery bigQuery, - boolean isKeepFilesInGcs, - BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, gcsDestinationConfig, bigQuery, isKeepFilesInGcs, recordFormatter); - } - - @Override - protected LoadJobConfiguration getLoadConfiguration() { - final var csvOptions = CsvOptions.newBuilder().setEncoding(UTF8).setSkipLeadingRows(1).build(); - - return LoadJobConfiguration.builder(tmpTable, writer.getFileLocation()) - .setFormatOptions(csvOptions) - .setSchema(recordFormatter.getBigQuerySchema()) - .setWriteDisposition(syncMode) - .build(); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java index a93123a54807..28e2f0ea3f24 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java @@ -31,7 +31,6 @@ public class UploaderConfig { */ private StreamConfig parsedStream; private String targetTableName; - private String tmpTableName; private BigQuery bigQuery; private Map formatterMap; private boolean isDefaultAirbyteTmpSchema; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index 775ecefcb43d..41dafa21cc72 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -1,7 +1,6 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/destinations/bigquery", "supportsIncremental": true, - "supportsNormalization": true, "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index 68f56c0fe41a..99c13c121a33 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -30,12 +30,12 @@ import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index f17b12320adc..a220a1127696 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -8,8 +8,8 @@ import com.amazonaws.services.s3.AmazonS3; import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import java.nio.file.Path; import java.util.HashSet; import org.junit.jupiter.api.Disabled; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java index 61c28f7b67fa..99ac8a8e75dd 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java @@ -31,6 +31,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.bigquery.BigQueryConsts; @@ -365,8 +366,9 @@ public void testCreateTableInOtherRegion() throws InterruptedException { final BigQueryDestinationHandler destinationHandler = new BigQueryDestinationHandler(bq, "asia-east1"); // We're creating the dataset in the wrong location in the @BeforeEach block. Explicitly delete it. bq.getDataset(namespace).delete(); - - destinationHandler.execute(new BigQuerySqlGenerator(projectId, "asia-east1").createTable(incrementalDedupStream, "", false)); + final var sqlGenerator = new BigQuerySqlGenerator(projectId, "asia-east1"); + destinationHandler.execute(sqlGenerator.createSchema(namespace)); + destinationHandler.execute(sqlGenerator.createTable(incrementalDedupStream, "", false)); // Empirically, it sometimes takes Bigquery nearly 30 seconds to propagate the dataset's existence. // Give ourselves 2 minutes just in case. @@ -411,7 +413,7 @@ public void testFailureOnReservedColumnNamePrefix(final String prefix) { }); - final String createTable = generator.createTable(stream, "", false); + final Sql createTable = generator.createTable(stream, "", false); assertThrows( BigQueryException.class, () -> destinationHandler.execute(createTable)); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl index e456f48d443a..b24f35cc66d4 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -2,3 +2,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl index 623527f41e75..a3f87b7fe513 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -3,3 +3,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} // Invalid columns are nulled out (i.e. SQL null, not JSON null) {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl index 569905e1f03d..9f98de58cc61 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -3,3 +3,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} // Invalid data is still allowed in the raw table. {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl index 1f4d620add7b..0b3e157bdefc 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -2,6 +2,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl index 10cd001e22f6..9ce69173315b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} // Delete Bob, keep Charlie {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl index 0f44480d1b5b..02468d97ab2c 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -3,6 +3,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} // And append the records from the second sync {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index 627521e4d958..e83d33307523 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -5,3 +5,4 @@ // Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. // But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.17411800000001}, "array": [67.17411800000001], "unknown": 67.17411800000001, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index 9f89442b914f..aad52eb2e525 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -3,3 +3,4 @@ {"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} {"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 8043726bda18..0f03515ee087 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -5,18 +5,13 @@ package io.airbyte.integrations.destination.bigquery; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import java.util.Collections; -import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -58,27 +53,6 @@ public void testGetDatasetIdFail(final String projectId, final String datasetId, assertEquals(expected, exception.getMessage()); } - @Test - public void testIsUsingJsonCredentials() { - // empty - final JsonNode emptyConfig = Jsons.jsonNode(Collections.emptyMap()); - assertFalse(BigQueryUtils.isUsingJsonCredentials(emptyConfig)); - - // empty text - final JsonNode emptyTextConfig = Jsons.jsonNode(Map.of(BigQueryConsts.CONFIG_CREDS, "")); - assertFalse(BigQueryUtils.isUsingJsonCredentials(emptyTextConfig)); - - // non-empty text - final JsonNode nonEmptyTextConfig = Jsons.jsonNode( - Map.of(BigQueryConsts.CONFIG_CREDS, "{ \"service_account\": \"test@airbyte.io\" }")); - assertTrue(BigQueryUtils.isUsingJsonCredentials(nonEmptyTextConfig)); - - // object - final JsonNode objectConfig = Jsons.jsonNode(Map.of( - BigQueryConsts.CONFIG_CREDS, Jsons.jsonNode(Map.of("service_account", "test@airbyte.io")))); - assertTrue(BigQueryUtils.isUsingJsonCredentials(objectConfig)); - } - private static Stream validBigQueryIdProvider() { return Stream.of( Arguments.arguments("my-project", "my_dataset", "my_dataset"), diff --git a/airbyte-integrations/connectors/destination-chroma/Dockerfile b/airbyte-integrations/connectors/destination-chroma/Dockerfile index dda5952e33b0..6eec4a792d2a 100644 --- a/airbyte-integrations/connectors/destination-chroma/Dockerfile +++ b/airbyte-integrations/connectors/destination-chroma/Dockerfile @@ -41,5 +41,5 @@ COPY destination_chroma ./destination_chroma ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.8 +LABEL io.airbyte.version=0.0.9 LABEL io.airbyte.name=airbyte/destination-chroma diff --git a/airbyte-integrations/connectors/destination-chroma/metadata.yaml b/airbyte-integrations/connectors/destination-chroma/metadata.yaml index 04fb729c5d42..8283e5453d0b 100644 --- a/airbyte-integrations/connectors/destination-chroma/metadata.yaml +++ b/airbyte-integrations/connectors/destination-chroma/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 0b75218b-f702-4a28-85ac-34d3d84c0fc2 - dockerImageTag: 0.0.8 + dockerImageTag: 0.0.9 dockerRepository: airbyte/destination-chroma githubIssueLabel: destination-chroma icon: chroma.svg diff --git a/airbyte-integrations/connectors/destination-chroma/setup.py b/airbyte-integrations/connectors/destination-chroma/setup.py index f39968fef18f..ae2f70163452 100644 --- a/airbyte-integrations/connectors/destination-chroma/setup.py +++ b/airbyte-integrations/connectors/destination-chroma/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[vector-db-based]==0.55.1", + "airbyte-cdk[vector-db-based]==0.57.0", "chromadb", ] diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index d1a316d740a4..584fd3a0dc72 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -4,8 +4,8 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.22.1' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml index b58fc5f5d3e5..c5023258510c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml @@ -7,16 +7,22 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/destination-clickhouse-strict-encrypt githubIssueLabel: destination-clickhouse icon: clickhouse.svg license: MIT name: Clickhouse - normalizationConfig: - normalizationIntegrationType: clickhouse - normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.4.1 + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2024-03-15" + message: > + This version removes the option to use "normalization" with clickhouse. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse supportsDbt: false diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java index 98d998a9e3ef..4efc4db3545c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java @@ -36,4 +36,9 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed destination: {}", ClickhouseDestinationStrictEncrypt.class); } + @Override + public boolean isV2Destination() { + return true; + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 6769060d4ff1..991ef0e2cde4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; @@ -139,7 +140,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -147,7 +148,9 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final JdbcDatabase jdbcDB = getDatabase(getConfig()); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final var nameTransformer = new StandardNameTransformer(); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 0386841d5f45..25cb081d4263 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,8 +4,8 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.22.1' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml index cf10ca7aa667..b6cc2b944a79 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml @@ -2,22 +2,28 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/destination-clickhouse githubIssueLabel: destination-clickhouse icon: clickhouse.svg license: MIT name: Clickhouse - normalizationConfig: - normalizationIntegrationType: clickhouse - normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.4.3 registries: cloud: dockerRepository: airbyte/destination-clickhouse-strict-encrypt enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2024-03-15" + message: > + This version removes the option to use "normalization" with clickhouse. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse supportsDbt: false diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 45a9d7cc8f08..77b3347944ca 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -16,6 +16,8 @@ import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; @@ -87,7 +89,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { final JdbcDatabase database = getDatabase(dataSource); final NamingConventionTransformer namingResolver = getNamingResolver(); final String outputSchema = namingResolver.getIdentifier(config.get(JdbcUtils.DATABASE_KEY).asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, namingResolver, getSqlOperations()); + attemptTableOperations(outputSchema, database, namingResolver, getSqlOperations(), false); return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (final Exception e) { LOGGER.error("Exception while checking connection: ", e); @@ -115,4 +117,19 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed destination: {}", ClickhouseDestination.class); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RawOnlySqlGenerator(new ClickhouseSQLNameTransformer()); + } + + @Override + public boolean isV2Destination() { + return true; + } + + @Override + protected String getConfigSchemaKey() { + return "database"; + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 76d2fa56af89..0d0acf62d5ee 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -10,7 +10,7 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -36,18 +36,22 @@ public boolean isSchemaRequired() { @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s String,\n" - + "%s String,\n" - + "%s DateTime64(3, 'GMT') DEFAULT now(),\n" - + "PRIMARY KEY(%s)\n" - + ")\n" - + "ENGINE = MergeTree;\n", + """ + CREATE TABLE IF NOT EXISTS `%s`.`%s` ( + %s String, + %s String, + %s DateTime64(3, 'GMT') DEFAULT now(), + %s DateTime64(3, 'GMT') NULL, + PRIMARY KEY(%s) + ) + ENGINE = MergeTree; + """, schemaName, tableName, - JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT, - JavaBaseConstants.COLUMN_NAME_AB_ID); + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID); } @Override @@ -60,7 +64,7 @@ public void executeTransaction(final JdbcDatabase database, final List q @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { @@ -102,4 +106,13 @@ public void insertRecordsInternal(final JdbcDatabase database, }); } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + insertRecordsInternal(database, records, schemaName, tableName); + } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index 4f3c51333f8a..a86a89f7f746 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -3,7 +3,7 @@ "supportsIncremental": true, "supportsNormalization": true, "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "ClickHouse Destination Spec", @@ -58,6 +58,12 @@ "type": "boolean", "default": false, "order": 6 + }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 7 } } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 5f5c3ae948fa..c7e7d7a5b6a6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.sql.SQLException; import java.time.Duration; import java.util.HashSet; @@ -111,7 +112,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -119,7 +120,9 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final JdbcDatabase jdbcDB = getDatabase(getConfig()); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final var nameTransformer = new StandardNameTransformer(); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index c82dfca207c1..163c9a6e36c6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -85,7 +86,7 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() throws Exception { return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME) - .put("schema", DB_NAME), false); + .put("schema", DB_NAME), true); } @Override @@ -109,7 +110,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -122,7 +123,8 @@ private List retrieveRecordsFromTable(final String tableName, final St JdbcUtils.PORT_LIST_KEY, mangledConfig -> { final JdbcDatabase database = getDatabase(mangledConfig); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, namingResolver.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return database.queryJsons(query); }); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 0b05cb932a8e..e414e428e63d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -13,11 +13,13 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -26,6 +28,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Comparator; import java.util.List; @@ -95,22 +98,26 @@ static void cleanUp() { @Test void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); - final AirbyteMessageConsumer consumer = dest.getConsumer(config, catalog, + DestinationConfig.initialize(config, dest.isV2Destination()); + final SerializedAirbyteMessageConsumer consumer = dest.getSerializedMessageConsumer(config, catalog, Destination::defaultOutputRecordCollector); final List expectedRecords = generateRecords(10); consumer.start(); expectedRecords.forEach(m -> { try { - consumer.accept(m); + final var strMessage = Jsons.jsonNode(m).toString(); + consumer.accept(strMessage, strMessage.getBytes(StandardCharsets.UTF_8).length); } catch (final Exception e) { throw new RuntimeException(e); } }); - consumer.accept(new AirbyteMessage() + final var abMessage = Jsons.jsonNode(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))))); + .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))))) + .toString(); + consumer.accept(abMessage, abMessage.getBytes(StandardCharsets.UTF_8).length); consumer.close(); final JdbcDatabase database = new DefaultJdbcDatabase( @@ -126,8 +133,8 @@ void sanityTest() throws Exception { final List actualRecords = database.bufferedResultSetQuery( connection -> connection.createStatement().executeQuery( - String.format("SELECT * FROM %s.%s;", DB_NAME, - namingResolver.getRawTableName(STREAM_NAME))), + String.format("SELECT * FROM %s.%s;", "airbyte_internal", + StreamId.concatenateRawTableName(DB_NAME, STREAM_NAME))), JdbcUtils.getDefaultSourceOperations()::rowToJson); assertEquals( diff --git a/airbyte-integrations/connectors/destination-databricks/metadata.yaml b/airbyte-integrations/connectors/destination-databricks/metadata.yaml index 0b707d0c110f..8d7eeeb33ee7 100644 --- a/airbyte-integrations/connectors/destination-databricks/metadata.yaml +++ b/airbyte-integrations/connectors/destination-databricks/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 072d5540-f236-4294-ba7c-ade8fd918496 - dockerImageTag: 1.1.0 + dockerImageTag: 1.1.1 dockerRepository: airbyte/destination-databricks githubIssueLabel: destination-databricks icon: databricks.svg @@ -11,8 +11,10 @@ data: registries: cloud: enabled: true + dockerImageTag: 1.1.0 # pinning due to CDK incompatibility, see https://github.com/airbytehq/alpha-beta-issues/issues/2596 oss: enabled: true + dockerImageTag: 1.1.0 # pinning due to CDK incompatibility, see https://github.com/airbytehq/alpha-beta-issues/issues/2596 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/databricks tags: diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json index 19b74c77a80f..5331b730b258 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json @@ -129,31 +129,40 @@ "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index e167b803db50..e4a9dfb1d487 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.6' features = ['db-destinations'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -25,6 +15,4 @@ application { dependencies { implementation project(':airbyte-integrations:connectors:destination-e2e-test') - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dev-null') } diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index ad5594d4b173..aea42c796b2a 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3 - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/destination-dev-null githubIssueLabel: destination-dev-null icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java index 2cc8a935d677..417bc0a2d699 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java @@ -58,4 +58,9 @@ protected void assertSameMessages(final List expected, assertEquals(0, actual.size()); } + @Override + public void testSyncNotFailsWithNewFields() { + // Skip because `retrieveRecords` returns an empty list at all times. + } + } diff --git a/airbyte-integrations/connectors/destination-duckdb/build.gradle b/airbyte-integrations/connectors/destination-duckdb/build.gradle deleted file mode 100644 index b623727f32cd..000000000000 --- a/airbyte-integrations/connectors/destination-duckdb/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// No-op Gradle build file. Required temporarily due to cross-project dependencies. -// TODO: Delete when no longer needed, per CI testing requirements. diff --git a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml index 961fa4be5cd3..c50ab11ae6b5 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/destination-dynamodb githubIssueLabel: destination-dynamodb icon: dynamodb.svg diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json index c77cd537ff98..0da5853a1910 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json @@ -36,31 +36,39 @@ "description": "The region of the DynamoDB.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ] }, "access_key_id": { diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index c8d98e1ddc2b..c1a95d154e3b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -1,25 +1,16 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.6' features = ['db-destinations'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.e2e_test.TestingDestinations' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml index 31b19e2f5b12..381d1b6b1ecc 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: unknown connectorType: destination definitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537 - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/destination-e2e-test githubIssueLabel: destination-e2e-test icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java index ba3921e87cda..2cbb95cf160b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java @@ -59,4 +59,9 @@ protected void assertSameMessages(final List expected, assertEquals(0, actual.size()); } + @Override + public void testSyncNotFailsWithNewFields() { + // Skip because `retrieveRecords` returns an empty list at all times. + } + } diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index a5cc5eb33d28..23f1896db8e8 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -1,51 +1,17 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.20.8' + features = ['db-destinations', 'gcs-destinations'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.gcs.GcsDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - - implementation platform('com.amazonaws:aws-java-sdk-bom:1.12.14') - implementation 'com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.1' - - // csv - implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - - // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' } diff --git a/airbyte-integrations/connectors/destination-gcs/metadata.yaml b/airbyte-integrations/connectors/destination-gcs/metadata.yaml index 3d904cb25b47..1f861a2d02a2 100644 --- a/airbyte-integrations/connectors/destination-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/destination-gcs/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: ca8f6566-e555-4b40-943a-545bf123117a - dockerImageTag: 0.4.4 + dockerImageTag: 0.4.6 dockerRepository: airbyte/destination-gcs githubIssueLabel: destination-gcs icon: googlecloudstorage.svg diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java index 3b209f6fe7c7..5c3a40487841 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java @@ -4,41 +4,11 @@ package io.airbyte.integrations.destination.gcs; -import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; - -import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.BaseConnector; -import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; -import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; -import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; -import io.airbyte.cdk.integrations.destination.s3.S3ConsumerFactory; -import io.airbyte.cdk.integrations.destination.s3.SerializedBufferFactory; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class GcsDestination extends BaseConnector implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(GcsDestination.class); - public static final String EXPECTED_ROLES = "storage.multipartUploads.abort, storage.multipartUploads.create, " - + "storage.objects.create, storage.objects.delete, storage.objects.get, storage.objects.list"; +import io.airbyte.cdk.integrations.destination.gcs.BaseGcsDestination; - private final NamingConventionTransformer nameTransformer; - - public GcsDestination() { - this.nameTransformer = new GcsNameTransformer(); - } +public class GcsDestination extends BaseGcsDestination { public static void main(final String[] args) throws Exception { System.setProperty(SkipMd5CheckStrategy.DISABLE_GET_OBJECT_MD5_VALIDATION_PROPERTY, "true"); @@ -46,48 +16,4 @@ public static void main(final String[] args) throws Exception { new IntegrationRunner(new GcsDestination()).run(args); } - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - try { - final GcsDestinationConfig destinationConfig = GcsDestinationConfig.getGcsDestinationConfig(config); - final AmazonS3 s3Client = destinationConfig.getS3Client(); - - // Test single upload (for small files) permissions - S3BaseChecks.testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); - - // Test multipart upload with stream transfer manager - S3BaseChecks.testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); - - return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); - } catch (final AmazonS3Exception e) { - LOGGER.error("Exception attempting to access the Gcs bucket", e); - final String message = getErrorMessage(e.getErrorCode(), 0, e.getMessage(), e); - AirbyteTraceMessageUtility.emitConfigErrorTrace(e, message); - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage(message); - } catch (final Exception e) { - LOGGER.error("Exception attempting to access the Gcs bucket: {}. Please make sure you account has all of these roles: " + EXPECTED_ROLES, e); - AirbyteTraceMessageUtility.emitConfigErrorTrace(e, e.getMessage()); - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) - .withMessage("Could not connect to the Gcs bucket with the provided configuration. \n" + e - .getMessage()); - } - } - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog configuredCatalog, - final Consumer outputRecordCollector) { - final GcsDestinationConfig gcsConfig = GcsDestinationConfig.getGcsDestinationConfig(config); - return new S3ConsumerFactory().create( - outputRecordCollector, - new GcsStorageOperations(nameTransformer, gcsConfig.getS3Client(), gcsConfig), - nameTransformer, - SerializedBufferFactory.getCreateFunction(gcsConfig, FileBuffer::new), - gcsConfig, - configuredCatalog); - } - } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java deleted file mode 100644 index c22b6069b619..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs.credential; - -public enum GcsCredentialType { - HMAC_KEY -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java new file mode 100644 index 000000000000..6f3f422c3548 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseAvroDestinationAcceptanceTest; + +public class GcsAvroDestinationAcceptanceTest extends GcsBaseAvroDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java new file mode 100644 index 000000000000..ee4088ecc7c8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseCsvDestinationAcceptanceTest; + +public class GcsCsvDestinationAcceptanceTest extends GcsBaseCsvDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java new file mode 100644 index 000000000000..e9f8bc9abc7e --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseCsvGzipDestinationAcceptanceTest; + +public class GcsCsvGzipDestinationAcceptanceTest extends GcsBaseCsvGzipDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java new file mode 100644 index 000000000000..70583bf73bd6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseJsonlDestinationAcceptanceTest; + +public class GcsJsonlDestinationAcceptanceTest extends GcsBaseJsonlDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java new file mode 100644 index 000000000000..958d7090253c --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseJsonlGzipDestinationAcceptanceTest; + +public class GcsJsonlGzipDestinationAcceptanceTest extends GcsBaseJsonlGzipDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java new file mode 100644 index 000000000000..12b65b6167db --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseParquetDestinationAcceptanceTest; + +public class GcsParquetDestinationAcceptanceTest extends GcsBaseParquetDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java deleted file mode 100644 index e19036bd5c47..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectReader; -import io.airbyte.cdk.integrations.destination.s3.S3Format; -import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.commons.json.Jsons; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.avro.Schema.Type; -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.SeekableByteArrayInput; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Record; -import org.apache.avro.generic.GenericDatumReader; - -public class GcsAvroDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { - - public GcsAvroDestinationAcceptanceTest() { - super(S3Format.AVRO); - } - - @Override - protected JsonNode getFormatConfig() { - return Jsons.deserialize("{\n" - + " \"format_type\": \"Avro\",\n" - + " \"compression_codec\": { \"codec\": \"no compression\", \"compression_level\": 5, \"include_checksum\": true }\n" - + "}"); - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new GcsAvroTestDataComparator(); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv testEnv, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - final JsonFieldNameUpdater nameUpdater = AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema); - - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - final List jsonRecords = new LinkedList<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - try (final DataFileReader dataFileReader = new DataFileReader<>( - new SeekableByteArrayInput(object.getObjectContent().readAllBytes()), - new GenericDatumReader<>())) { - final ObjectReader jsonReader = MAPPER.reader(); - while (dataFileReader.hasNext()) { - final GenericData.Record record = dataFileReader.next(); - final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); - JsonNode jsonRecord = jsonReader.readTree(jsonBytes); - jsonRecord = nameUpdater.getJsonWithOriginalFieldNames(jsonRecord); - jsonRecords.add(AvroRecordHelper.pruneAirbyteJson(jsonRecord)); - } - } - } - - return jsonRecords; - } - - @Override - protected Map> retrieveDataTypesFromPersistedFiles(final String streamName, final String namespace) throws Exception { - - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - final Map> resultDataTypes = new HashMap<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - try (final DataFileReader dataFileReader = new DataFileReader<>( - new SeekableByteArrayInput(object.getObjectContent().readAllBytes()), - new GenericDatumReader<>())) { - while (dataFileReader.hasNext()) { - final GenericData.Record record = dataFileReader.next(); - final Map> actualDataTypes = getTypes(record); - resultDataTypes.putAll(actualDataTypes); - } - } - } - return resultDataTypes; - } - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java deleted file mode 100644 index 7c7985498e0e..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.s3.S3Format; -import io.airbyte.cdk.integrations.destination.s3.util.Flattening; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.commons.json.Jsons; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.StreamSupport; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.csv.QuoteMode; - -public class GcsCsvDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { - - public GcsCsvDestinationAcceptanceTest() { - super(S3Format.CSV); - } - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - - @Override - protected JsonNode getFormatConfig() { - return Jsons.jsonNode(Map.of( - "format_type", outputFormat, - "flattening", Flattening.ROOT_LEVEL.getValue(), - "compression", Jsons.jsonNode(Map.of("compression_type", "No Compression")))); - } - - /** - * Convert json_schema to a map from field name to field types. - */ - private static Map getFieldTypes(final JsonNode streamSchema) { - final Map fieldTypes = new HashMap<>(); - final JsonNode fieldDefinitions = streamSchema.get("properties"); - final Iterator> iterator = fieldDefinitions.fields(); - while (iterator.hasNext()) { - final Map.Entry entry = iterator.next(); - JsonNode fieldValue = entry.getValue(); - JsonNode typeValue = fieldValue.get("type") == null ? fieldValue.get("$ref") : fieldValue.get("type"); - fieldTypes.put(entry.getKey(), typeValue.asText()); - } - return fieldTypes; - } - - private static JsonNode getJsonNode(final Map input, final Map fieldTypes) { - final ObjectNode json = MAPPER.createObjectNode(); - - if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) { - return Jsons.deserialize(input.get(JavaBaseConstants.COLUMN_NAME_DATA)); - } - - for (final Map.Entry entry : input.entrySet()) { - final String key = entry.getKey(); - if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key - .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { - continue; - } - final String value = entry.getValue(); - if (value == null || value.equals("")) { - continue; - } - final String type = fieldTypes.get(key); - switch (type) { - case "boolean" -> json.put(key, Boolean.valueOf(value)); - case "integer" -> json.put(key, Integer.valueOf(value)); - case "number" -> json.put(key, Double.valueOf(value)); - case "" -> addNoTypeValue(json, key, value); - default -> json.put(key, value); - } - } - return json; - } - - private static void addNoTypeValue(final ObjectNode json, final String key, final String value) { - if (value != null && (value.matches("^\\[.*\\]$")) || value.matches("^\\{.*\\}$")) { - final var newNode = Jsons.deserialize(value); - json.set(key, newNode); - } else { - json.put(key, value); - } - } - - @Override - protected List retrieveRecords(final TestDestinationEnv testEnv, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws IOException { - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - - final Map fieldTypes = getFieldTypes(streamSchema); - final List jsonRecords = new LinkedList<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - try (final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - final Reader in = getReader(object)) { - final Iterable records = CSVFormat.DEFAULT - .withQuoteMode(QuoteMode.NON_NUMERIC) - .withFirstRecordAsHeader() - .parse(in); - StreamSupport.stream(records.spliterator(), false) - .forEach(r -> jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))); - } - } - - return jsonRecords; - } - - protected Reader getReader(final S3Object s3Object) throws IOException { - return new InputStreamReader(s3Object.getObjectContent(), StandardCharsets.UTF_8); - } - -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java deleted file mode 100644 index 075b9532a54f..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.destination.s3.util.Flattening; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.commons.json.Jsons; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.zip.GZIPInputStream; - -public class GcsCsvGzipDestinationAcceptanceTest extends GcsCsvDestinationAcceptanceTest { - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - - @Override - protected JsonNode getFormatConfig() { - // config without compression defaults to GZIP - return Jsons.jsonNode(Map.of( - "format_type", outputFormat, - "flattening", Flattening.ROOT_LEVEL.getValue())); - } - - protected Reader getReader(final S3Object s3Object) throws IOException { - return new InputStreamReader(new GZIPInputStream(s3Object.getObjectContent()), StandardCharsets.UTF_8); - } - -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java deleted file mode 100644 index 4d9fb0e21636..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.s3.S3Format; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.commons.json.Jsons; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class GcsJsonlDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { - - public GcsJsonlDestinationAcceptanceTest() { - super(S3Format.JSONL); - } - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - - @Override - protected JsonNode getFormatConfig() { - return Jsons.jsonNode(Map.of( - "format_type", outputFormat, - "compression", Jsons.jsonNode(Map.of("compression_type", "No Compression")))); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv testEnv, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws IOException { - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - final List jsonRecords = new LinkedList<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - try (final BufferedReader reader = getReader(object)) { - String line; - while ((line = reader.readLine()) != null) { - jsonRecords.add(Jsons.deserialize(line).get(JavaBaseConstants.COLUMN_NAME_DATA)); - } - } - } - - return jsonRecords; - } - - protected BufferedReader getReader(final S3Object s3Object) throws IOException { - return new BufferedReader(new InputStreamReader(s3Object.getObjectContent(), StandardCharsets.UTF_8)); - } - -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java deleted file mode 100644 index 152bf22d1535..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.commons.json.Jsons; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.zip.GZIPInputStream; - -public class GcsJsonlGzipDestinationAcceptanceTest extends GcsJsonlDestinationAcceptanceTest { - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - - @Override - protected JsonNode getFormatConfig() { - // config without compression defaults to GZIP - return Jsons.jsonNode(Map.of("format_type", outputFormat)); - } - - protected BufferedReader getReader(final S3Object s3Object) throws IOException { - return new BufferedReader(new InputStreamReader(new GZIPInputStream(s3Object.getObjectContent()), StandardCharsets.UTF_8)); - } - -} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java deleted file mode 100644 index 65b2e68e8161..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.gcs; - -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectReader; -import io.airbyte.cdk.integrations.destination.s3.S3Format; -import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; -import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.parquet.GcsParquetWriter; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Record; -import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.avro.AvroReadSupport; -import org.apache.parquet.hadoop.ParquetReader; - -public class GcsParquetDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { - - public GcsParquetDestinationAcceptanceTest() { - super(S3Format.PARQUET); - } - - @Override - public ProtocolVersion getProtocolVersion() { - return ProtocolVersion.V1; - } - - @Override - protected JsonNode getFormatConfig() { - return Jsons.jsonNode(Map.of( - "format_type", "Parquet", - "compression_codec", "GZIP")); - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new GcsAvroTestDataComparator(); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv testEnv, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws IOException, URISyntaxException { - final JsonFieldNameUpdater nameUpdater = AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema); - - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - final List jsonRecords = new LinkedList<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - final URI uri = new URI(String.format("s3a://%s/%s", object.getBucketName(), object.getKey())); - final var path = new org.apache.hadoop.fs.Path(uri); - final Configuration hadoopConfig = GcsParquetWriter.getHadoopConfig(config); - - try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) - .withConf(hadoopConfig) - .build()) { - final ObjectReader jsonReader = MAPPER.reader(); - GenericData.Record record; - while ((record = parquetReader.read()) != null) { - final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); - JsonNode jsonRecord = jsonReader.readTree(jsonBytes); - jsonRecord = nameUpdater.getJsonWithOriginalFieldNames(jsonRecord); - jsonRecords.add(AvroRecordHelper.pruneAirbyteJson(jsonRecord)); - } - } - } - - return jsonRecords; - } - - @Override - protected Map> retrieveDataTypesFromPersistedFiles(final String streamName, final String namespace) throws Exception { - - final List objectSummaries = getAllSyncedObjects(streamName, namespace); - final Map> resultDataTypes = new HashMap<>(); - - for (final S3ObjectSummary objectSummary : objectSummaries) { - final S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - final URI uri = new URI(String.format("s3a://%s/%s", object.getBucketName(), object.getKey())); - final var path = new org.apache.hadoop.fs.Path(uri); - final Configuration hadoopConfig = S3ParquetWriter.getHadoopConfig(config); - - try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) - .withConf(hadoopConfig) - .build()) { - GenericData.Record record; - while ((record = parquetReader.read()) != null) { - final Map> actualDataTypes = getTypes(record); - resultDataTypes.putAll(actualDataTypes); - } - } - } - - return resultDataTypes; - } - -} diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index 37f06943d35d..a8672abce6e6 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -52,12 +52,6 @@ dependencies { testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation libs.testcontainers.postgresql - compileOnly 'org.projectlombok:lombok:1.18.24' - annotationProcessor 'org.projectlombok:lombok:1.18.24' - - testCompileOnly 'org.projectlombok:lombok:1.18.24' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' - testImplementation 'org.mockito:mockito-inline:4.7.0' } diff --git a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml index 659f3fcafe61..64e40d6491fd 100644 --- a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: df65a8f3-9908-451b-aa9b-445462803560 - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-iceberg githubIssueLabel: destination-iceberg license: MIT diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json index 503b24a6cf71..245874d890e0 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json @@ -214,31 +214,40 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 3 }, diff --git a/airbyte-integrations/connectors/destination-kvdb/.dockerignore b/airbyte-integrations/connectors/destination-kvdb/.dockerignore deleted file mode 100644 index 1b4b5767b554..000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!Dockerfile -!main.py -!destination_kvdb -!setup.py diff --git a/airbyte-integrations/connectors/destination-kvdb/Dockerfile b/airbyte-integrations/connectors/destination-kvdb/Dockerfile deleted file mode 100644 index fba4f5cb3d41..000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9.11-alpine3.15 - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY destination_kvdb ./destination_kvdb -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-kvdb diff --git a/airbyte-integrations/connectors/destination-kvdb/icon.svg b/airbyte-integrations/connectors/destination-kvdb/icon.svg new file mode 100644 index 000000000000..249b913e9424 --- /dev/null +++ b/airbyte-integrations/connectors/destination-kvdb/icon.svg @@ -0,0 +1,83 @@ + + + + + diff --git a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py deleted file mode 100644 index 5e083acc5351..000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from typing import Any, Dict, List, Mapping - -import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStateMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - Status, - SyncMode, - Type, -) -from destination_kvdb import DestinationKvdb -from destination_kvdb.client import KvDbClient - - -@pytest.fixture(name="config") -def config_fixture() -> Mapping[str, Any]: - with open("secrets/config.json", "r") as f: - return json.loads(f.read()) - - -@pytest.fixture(name="configured_catalog") -def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} - - append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.append, - ) - - overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - - return ConfiguredAirbyteCatalog(streams=[append_stream, overwrite_stream]) - - -@pytest.fixture(autouse=True) -def teardown(config: Mapping): - yield - client = KvDbClient(**config) - client.delete(list(client.list_keys())) - - -@pytest.fixture(name="client") -def client_fixture(config) -> KvDbClient: - return KvDbClient(**config) - - -def test_check_valid_config(config: Mapping): - outcome = DestinationKvdb().check(AirbyteLogger(), config) - assert outcome.status == Status.SUCCEEDED - - -def test_check_invalid_config(): - outcome = DestinationKvdb().check(AirbyteLogger(), {"bucket_id": "not_a_real_id"}) - assert outcome.status == Status.FAILED - - -def _state(data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) - - -def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: - return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) - ) - - -def retrieve_all_records(client: KvDbClient) -> List[AirbyteRecordMessage]: - """retrieves and formats all records in kvdb as Airbyte messages""" - all_records = client.list_keys(list_values=True) - out = [] - for record in all_records: - key = record[0] - stream = key.split("__ab__")[0] - value = record[1] - out.append(_record(stream, value["str_col"], value["int_col"])) - return out - - -def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: KvDbClient): - """ - This test verifies that: - 1. writing a stream in "overwrite" mode overwrites any existing data for that stream - 2. writing a stream in "append" mode appends new records without deleting the old ones - 3. The correct state message is output by the connector at the end of the sync - """ - append_stream, overwrite_stream = configured_catalog.streams[0].stream.name, configured_catalog.streams[1].stream.name - first_state_message = _state({"state": "1"}) - first_record_chunk = [_record(append_stream, str(i), i) for i in range(5)] + [_record(overwrite_stream, str(i), i) for i in range(5)] - - second_state_message = _state({"state": "2"}) - second_record_chunk = [_record(append_stream, str(i), i) for i in range(5, 10)] + [ - _record(overwrite_stream, str(i), i) for i in range(5, 10) - ] - - destination = DestinationKvdb() - - expected_states = [first_state_message, second_state_message] - output_states = list( - destination.write( - config, configured_catalog, [*first_record_chunk, first_state_message, *second_record_chunk, second_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - expected_records = [_record(append_stream, str(i), i) for i in range(10)] + [_record(overwrite_stream, str(i), i) for i in range(10)] - records_in_destination = retrieve_all_records(client) - assert expected_records == records_in_destination, "Records in destination should match records expected" - - # After this sync we expect the append stream to have 15 messages and the overwrite stream to have 5 - third_state_message = _state({"state": "3"}) - third_record_chunk = [_record(append_stream, str(i), i) for i in range(10, 15)] + [ - _record(overwrite_stream, str(i), i) for i in range(10, 15) - ] - - output_states = list(destination.write(config, configured_catalog, [*third_record_chunk, third_state_message])) - assert [third_state_message] == output_states - - records_in_destination = retrieve_all_records(client) - expected_records = [_record(append_stream, str(i), i) for i in range(15)] + [ - _record(overwrite_stream, str(i), i) for i in range(10, 15) - ] - assert expected_records == records_in_destination diff --git a/airbyte-integrations/connectors/destination-kvdb/metadata.yaml b/airbyte-integrations/connectors/destination-kvdb/metadata.yaml new file mode 100644 index 000000000000..64a38cfa1441 --- /dev/null +++ b/airbyte-integrations/connectors/destination-kvdb/metadata.yaml @@ -0,0 +1,27 @@ +data: + registries: + cloud: + enabled: false + oss: + enabled: false + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: destination + definitionId: f2e549cd-8e2a-48f8-822d-cc13630eb42d + dockerImageTag: 0.1.3 + dockerRepository: airbyte/destination-kvdb + githubIssueLabel: destination-kvdb + icon: kvdb.svg + license: MIT + name: KVdb + releaseDate: 2023-07-15 + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/kvdb + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: archived +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-kvdb/poetry.lock b/airbyte-integrations/connectors/destination-kvdb/poetry.lock new file mode 100644 index 000000000000..7835868ea21e --- /dev/null +++ b/airbyte-integrations/connectors/destination-kvdb/poetry.lock @@ -0,0 +1,1108 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.2.tar.gz", hash = "sha256:bf45cb847e2d2ab7063d0e1989f6c9cf022771c6ae4fb1e854438c3b8377da85"}, + {file = "airbyte_cdk-0.62.2-py3-none-any.whl", hash = "sha256:6d04d2e8a9a32aa707ddf27a1916ac76969fb50ac39d60582ad2daa08ef832ef"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.23.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, + {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "sgqlc" +version = "16.3" +description = "Simple GraphQL Client" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "sgqlc-16.3-py3-none-any.whl", hash = "sha256:89d468386a4ba4b5ade991623228b6fb0a25bea1f25643ccac130fb3ef565b72"}, + {file = "sgqlc-16.3.tar.gz", hash = "sha256:be08857775aa3e65ef7b2c1f0cdcc65dd5794907b162b393c189187fee664558"}, +] + +[package.dependencies] +graphql-core = ">=3.1.7,<4.0.0" + +[package.extras] +requests = ["requests"] +websocket = ["websocket-client"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "40cc246c45e6c2d626e016673f3aa60794f3464d82c8ccd0b62a6b66df2b30da" diff --git a/airbyte-integrations/connectors/destination-kvdb/pyproject.toml b/airbyte-integrations/connectors/destination-kvdb/pyproject.toml new file mode 100644 index 000000000000..2f61ed29c449 --- /dev/null +++ b/airbyte-integrations/connectors/destination-kvdb/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.3" +name = "destination-kvdb" +description = "Destination implementation for kvdb." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/destinations/kvdb" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "destination_kvdb" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0.62.1" +sgqlc = "==16.3" + +[tool.poetry.scripts] +destination-kvdb = "destination_kvdb.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +freezegun = "^1.2" +pytest-mock = "^3.6.1" +pytest = "^6.2" +responses = "^0.23.1" diff --git a/airbyte-integrations/connectors/destination-kvdb/setup.py b/airbyte-integrations/connectors/destination-kvdb/setup.py deleted file mode 100644 index dab5520718ab..000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "requests", -] - -TEST_REQUIREMENTS = ["pytest~=6.1"] - -setup( - name="destination_kvdb", - description="Destination implementation for Kvdb.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile index f573460c64a2..00bbfb9c4846 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile @@ -34,5 +34,5 @@ COPY destination_meilisearch ./destination_meilisearch ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/destination-meilisearch diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py index d6a44c1d5f9c..32d08b787bf1 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py @@ -3,14 +3,16 @@ # -from logging import Logger -from typing import Any, Iterable, Mapping +from logging import Logger, getLogger +from typing import Any, Dict, Iterable, Mapping from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from destination_meilisearch.writer import MeiliWriter from meilisearch import Client +logger = getLogger("airbyte") + def get_client(config: Mapping[str, Any]) -> Client: host = config.get("host") @@ -21,36 +23,51 @@ def get_client(config: Mapping[str, Any]) -> Client: class DestinationMeilisearch(Destination): primary_key = "_ab_pk" + def _flush_streams(self, streams: Dict[str, MeiliWriter]) -> Iterable[AirbyteMessage]: + for stream in streams: + streams[stream].flush() + def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: client = get_client(config=config) + # Creating Meilisearch writers + writers = {s.stream.name: MeiliWriter(client, s.stream.name, self.primary_key) for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: - steam_name = configured_stream.stream.name + stream_name = configured_stream.stream.name + # Deleting index in Meilisearch if sync mode is overwite if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: - client.delete_index(steam_name) - client.create_index(steam_name, {"primaryKey": self.primary_key}) - - writer = MeiliWriter(client, steam_name, self.primary_key) - for message in input_messages: - if message.type == Type.STATE: - writer.flush() - yield message - elif message.type == Type.RECORD: - writer.queue_write_operation(message.record.data) - else: + logger.debug(f"Deleting index: {stream_name}.") + client.delete_index(stream_name) + # Creating index in Meilisearch + client.create_index(stream_name, {"primaryKey": self.primary_key}) + logger.debug(f"Creating index: {stream_name}.") + + for message in input_messages: + if message.type == Type.STATE: + yield message + elif message.type == Type.RECORD: + data = message.record.data + stream = message.record.stream + # Skip unselected streams + if stream not in writers: + logger.debug(f"Stream {stream} was not present in configured streams, skipping") continue - writer.flush() + writers[stream].queue_write_operation(data) + else: + logger.info(f"Unhandled message type {message.type}: {message}") + + # Flush any leftover messages + self._flush_streams(writers) def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: try: client = get_client(config=config) - create_index_job = client.create_index("_airbyte", {"primaryKey": "id"}) - client.wait_for_task(create_index_job["taskUid"]) + client.create_index("_airbyte", {"primaryKey": "id"}) - add_documents_job = client.index("_airbyte").add_documents( + client.index("_airbyte").add_documents( [ { "id": 287947, @@ -59,9 +76,7 @@ def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionS } ] ) - client.wait_for_task(add_documents_job.task_uid) - client.index("_airbyte").search("Shazam") client.delete_index("_airbyte") return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py index c2eca6a88ce9..e2450f825106 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py @@ -12,25 +12,28 @@ class MeiliWriter: - write_buffer = [] flush_interval = 50000 - def __init__(self, client: Client, steam_name: str, primary_key: str): + def __init__(self, client: Client, stream_name: str, primary_key: str): self.client = client - self.steam_name = steam_name self.primary_key = primary_key + self.stream_name: str = stream_name + self._write_buffer = [] + + logger.info(f"Creating MeiliWriter for {self.stream_name}") def queue_write_operation(self, data: Mapping): random_key = str(uuid4()) - self.write_buffer.append({**data, self.primary_key: random_key}) - if len(self.write_buffer) == self.flush_interval: + self._write_buffer.append({**data, self.primary_key: random_key}) + if len(self._write_buffer) == self.flush_interval: + logger.debug(f"Reached limit size: flushing records for {self.stream_name}") self.flush() def flush(self): - buffer_size = len(self.write_buffer) + buffer_size = len(self._write_buffer) if buffer_size == 0: return - logger.info(f"flushing {buffer_size} records") - response = self.client.index(self.steam_name).add_documents(self.write_buffer) + logger.info(f"Flushing {buffer_size} records") + response = self.client.index(self.stream_name).add_documents(self._write_buffer) self.client.wait_for_task(response.task_uid, 1800000, 1000) - self.write_buffer.clear() + self._write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py index 9e63e24dc87d..1d9687e97c7d 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py @@ -4,7 +4,6 @@ import json import logging -import time from typing import Any, Dict, Mapping import pytest @@ -56,12 +55,7 @@ def teardown(config: Mapping): def client_fixture(config) -> Client: client = get_client(config=config) resp = client.create_index("_airbyte", {"primaryKey": "_ab_pk"}) - while True: - time.sleep(0.2) - task = client.get_task(resp["taskUid"]) - status = task["status"] - if status == "succeeded" or status == "failed": - break + client.wait_for_task(_handle_breaking_wait_for_task(resp)) return client @@ -87,6 +81,13 @@ def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: ) +def _handle_breaking_wait_for_task(task: Any) -> int: + if type(task) is dict: + return task["taskUid"] + else: + return task.task_uid + + def records_count(client: Client) -> int: documents_results = client.index("_airbyte").get_documents() return documents_results.total diff --git a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml index 7826092b697b..79a5f5851984 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml +++ b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: destination definitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/destination-meilisearch githubIssueLabel: destination-meilisearch icon: meilisearch.svg diff --git a/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py index df1f503df180..c09a3f7d8744 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py @@ -9,15 +9,21 @@ @patch("meilisearch.Client") def test_queue_write_operation(client): - writer = MeiliWriter(client, "steam_name", "primary_key") + writer = MeiliWriter(client, "stream_name", "primary_key") writer.queue_write_operation({"a": "a"}) - assert len(writer.write_buffer) == 1 + assert len(writer._write_buffer) == 1 + writer.queue_write_operation({"b": "b"}) + assert len(writer._write_buffer) == 2 + writer2 = MeiliWriter(client, "stream_name2", "primary_key") + writer2.queue_write_operation({"a": "a"}) + assert len(writer2._write_buffer) == 1 + assert len(writer._write_buffer) == 2 @patch("meilisearch.Client") def test_flush(client): - writer = MeiliWriter(client, "steam_name", "primary_key") + writer = MeiliWriter(client, "stream_name", "primary_key") writer.queue_write_operation({"a": "a"}) writer.flush() - client.index.assert_called_once_with("steam_name") + client.index.assert_called_once_with("stream_name") client.wait_for_task.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-milvus/metadata.yaml b/airbyte-integrations/connectors/destination-milvus/metadata.yaml index 2ad1a4d390dd..09cacd466702 100644 --- a/airbyte-integrations/connectors/destination-milvus/metadata.yaml +++ b/airbyte-integrations/connectors/destination-milvus/metadata.yaml @@ -22,7 +22,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 65de8962-48c9-11ee-be56-0242ac120002 - dockerImageTag: 0.0.11 + dockerImageTag: 0.0.12 dockerRepository: airbyte/destination-milvus githubIssueLabel: destination-milvus icon: milvus.svg diff --git a/airbyte-integrations/connectors/destination-milvus/setup.py b/airbyte-integrations/connectors/destination-milvus/setup.py index fec5c6ee1524..e5c0cf315a83 100644 --- a/airbyte-integrations/connectors/destination-milvus/setup.py +++ b/airbyte-integrations/connectors/destination-milvus/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.55.1", "pymilvus==2.3.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "pymilvus==2.3.0"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle index 106b17a3248c..d52890815c6c 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle @@ -1,32 +1,18 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.19.0' + features = ['db-destinations', 'datastore-mongo'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-integrations:connectors:destination-mongodb') - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - - testImplementation libs.testcontainers.mongodb } diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java index 45a068cbf1f3..03f4306dc8dc 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.mongodb.MongoUtils; import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; @@ -28,10 +28,10 @@ public MongodbDestinationStrictEncrypt() { @Override public AirbyteConnectionStatus check(final JsonNode config) throws Exception { final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE); - final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); + final var instance = MongoUtils.MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); // If the MongoDb destination connector is not set up to use a TLS connection, then check should // fail - if (instance.equals(MongoInstanceType.STANDALONE) && !MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig)) { + if (instance.equals(MongoUtils.MongoInstanceType.STANDALONE) && !MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig)) { throw new ConfigErrorException("TLS connection must be used to read from MongoDB."); } return super.check(config); diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java index ec753b76370d..7ac931167e6d 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java @@ -13,10 +13,11 @@ import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.mongodb.MongoDatabase; +import io.airbyte.cdk.db.mongodb.MongoUtils; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -52,7 +53,7 @@ static void setupConfig() throws IOException { final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", MongoInstanceType.ATLAS.getType()) + .put("instance", MongoUtils.MongoInstanceType.ATLAS.getType()) .put("cluster_url", credentialsJson.get("cluster_url").asText()) .build()); @@ -110,7 +111,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Test void testCheck() throws Exception { final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", MongoInstanceType.STANDALONE.getType()) + .put("instance", MongoUtils.MongoInstanceType.STANDALONE.getType()) .put("tls", false) .build()); diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index cad1ef429466..b48535e992b0 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -1,37 +1,26 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.19.0' + features = ['db-destinations', 'datastore-mongo'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later java { compileJava { - options.compilerArgs.remove("-Werror") + options.compilerArgs += "-Xlint:-try" } } -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation 'org.apache.commons:commons-lang3:3.11' - - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - - // TODO: remove this dependency - implementation libs.google.cloud.storage - - testImplementation libs.testcontainers.mongodb + implementation 'commons-codec:commons-codec:1.16.0' - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') + testImplementation 'org.testcontainers:mongodb:1.19.0' } diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java index dacd5f500265..d3bb397a6bfc 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java @@ -15,6 +15,9 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.mongodb.MongoDatabase; +import io.airbyte.cdk.db.mongodb.MongoDatabaseException; +import io.airbyte.cdk.db.mongodb.MongoUtils; import io.airbyte.cdk.integrations.BaseConnector; import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; @@ -23,8 +26,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.destination.mongodb.exception.MongodbDatabaseException; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; @@ -70,7 +71,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { final var databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); final Set databaseNames = getDatabaseNames(database); if (!databaseNames.contains(databaseName) && !databaseName.equals(database.getName())) { - throw new MongodbDatabaseException(databaseName); + throw new MongoDatabaseException(databaseName); } return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); } catch (final ConnectionErrorException e) { @@ -156,7 +157,7 @@ private String buildConnectionString(final JsonNode config, final String credent final StringBuilder connectionStrBuilder = new StringBuilder(); final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE); - final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); + final var instance = MongoUtils.MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); switch (instance) { case STANDALONE -> { diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java index cd4bb1aeafc9..ab2602c8f2cb 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.db.mongodb.MongoDatabase; import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; @@ -18,6 +19,7 @@ import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.nio.charset.Charset; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,7 +27,6 @@ import java.util.function.Consumer; import org.apache.commons.codec.digest.DigestUtils; import org.bson.Document; -import org.joda.time.LocalDateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,7 +123,7 @@ private void insertRecordToTmpCollection(final MongodbWriteConfig writeConfig, final var newDocument = new Document(); newDocument.put(AIRBYTE_DATA, new Document(result)); newDocument.put(AIRBYTE_DATA_HASH, newDocumentDataHashCode); - newDocument.put(AIRBYTE_EMITTED_AT, new LocalDateTime().toString()); + newDocument.put(AIRBYTE_EMITTED_AT, LocalDateTime.now().toString()); final var collection = writeConfig.getCollection(); diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/exception/MongodbDatabaseException.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/exception/MongodbDatabaseException.java deleted file mode 100644 index 23d11643696e..000000000000 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/exception/MongodbDatabaseException.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.mongodb.exception; - -public class MongodbDatabaseException extends RuntimeException { - - public static final String MONGO_DATA_BASE_NOT_FOUND = "Data Base with given name - %s not found."; - - public MongodbDatabaseException(final String databaseName) { - super(String.format(MONGO_DATA_BASE_NOT_FOUND, databaseName)); - } - -} diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index 9e4d5a0b644a..004ef1535a3c 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.mongodb.MongoDatabase; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java index 25885678d05a..de6a7305ad07 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java @@ -10,6 +10,7 @@ import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.mongodb.MongoDatabase; import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.cdk.integrations.util.HostPortResolver; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java index 8648eb32d48e..b95b9fda8f37 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.mongodb; +import io.airbyte.cdk.db.mongodb.MongoDatabase; import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle index ac8e77da74ea..4f4da7b4aab9 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.8.0' features = ['db-destinations'] useLocalCdk = false } @@ -28,6 +28,16 @@ dependencies { implementation 'mysql:mysql-connector-java:8.0.22' + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql') integrationTestJavaImplementation libs.testcontainers.mysql } + +configurations.all { + resolutionStrategy { + force libs.jooq + } +} diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml index d838a707043b..e03ea32c8f73 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml @@ -1,13 +1,15 @@ data: registries: cloud: + dockerImageTag: 0.2.0 enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. oss: + dockerImageTag: 0.2.0 enabled: false # strict encrypt connectors are not used on OSS. connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/destination-mysql-strict-encrypt githubIssueLabel: destination-mysql icon: mysql.svg diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java index 1e5d3bb33f50..49e7776c7d11 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.cdk.db.Database; @@ -17,6 +18,7 @@ import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -73,23 +75,19 @@ protected boolean supportObjectDataTypeTest() { @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .build()); } @Override protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .build()); + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("password", "wrong password"); + return clone; } @Override @@ -141,7 +139,7 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test } @Override - protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new MySQLContainer<>("mysql:8.0"); db.start(); setLocalInFileToTrue(); diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index 8de8ca4e8497..7c04b7df4447 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.8.0' features = ['db-destinations'] useLocalCdk = false } @@ -24,7 +24,16 @@ application { } dependencies { - implementation 'mysql:mysql-connector-java:8.0.22' integrationTestJavaImplementation libs.testcontainers.mysql + + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) +} + +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-mysql/metadata.yaml b/airbyte-integrations/connectors/destination-mysql/metadata.yaml index ac3c702c6c58..9e6ee1de71e2 100644 --- a/airbyte-integrations/connectors/destination-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/destination-mysql githubIssueLabel: destination-mysql icon: mysql.svg @@ -14,9 +14,11 @@ data: normalizationTag: 0.4.3 registries: cloud: + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mysql-strict-encrypt enabled: true oss: + dockerImageTag: 0.2.0 enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/mysql diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index 9051ee328a78..438086bd8b38 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -17,6 +17,7 @@ import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; @@ -127,6 +128,11 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + throw new UnsupportedOperationException("mysql does not yet support DV2"); + } + public static void main(final String[] args) throws Exception { final Destination destination = MySQLDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", MySQLDestination.class); diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java index 8afed891366c..2fa5ec8dd572 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java @@ -10,7 +10,7 @@ import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -32,7 +32,7 @@ public void executeTransaction(final JdbcDatabase database, final List q @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { @@ -52,8 +52,17 @@ public void insertRecordsInternal(final JdbcDatabase database, } } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + throw new UnsupportedOperationException("mysql does not yet support DV2"); + } + private void loadDataIntoTable(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName, final File tmpFile) diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index 5b91c892972e..b6d83448bf46 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -18,7 +18,9 @@ import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -35,6 +37,7 @@ import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -43,7 +46,7 @@ public class MySQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - private MySQLContainer db; + protected MySQLContainer db; private final StandardNameTransformer namingResolver = new MySQLNameTransformer(); @Override @@ -79,25 +82,31 @@ protected boolean supportObjectDataTypeTest() { @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, false) .build()); } + /** + * {@link #getConfig()} returns a config with host/port set to the in-docker values. This works for + * running the destination-mysql container, but we have some tests which run the destination code + * directly from the JUnit process. These tests need to connect using the "normal" host/port. + */ + private JsonNode getConfigForBareMetalConnection() { + return ((ObjectNode) getConfig()) + .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()); + } + @Override protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.SSL_KEY, false) - .build()); + final ObjectNode config = (ObjectNode) getConfig(); + config.put(JdbcUtils.PASSWORD_KEY, "wrong password"); + return config; } @Override @@ -260,58 +269,76 @@ protected void assertSameValue(final JsonNode expectedValue, final JsonNode actu @Test void testCheckIncorrectPasswordFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.PASSWORD_KEY, "fake"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.PASSWORD_KEY, "fake"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 1045;")); + assertStringContains(status.getMessage(), "State code: 28000; Error code: 1045;"); } @Test public void testCheckIncorrectUsernameFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.USERNAME_KEY, "fake"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.USERNAME_KEY, "fake"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 1045;")); + assertStringContains(status.getMessage(), "State code: 28000; Error code: 1045;"); } @Test public void testCheckIncorrectHostFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.HOST_KEY, "localhost2"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.HOST_KEY, "localhost2"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertStringContains(status.getMessage(), "State code: 08S01;"); } @Test public void testCheckIncorrectPortFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.PORT_KEY, "0000"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.PORT_KEY, "0000"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertStringContains(status.getMessage(), "State code: 08S01;"); } @Test public void testCheckIncorrectDataBaseFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 42000; Error code: 1049;")); + assertStringContains(status.getMessage(), "State code: 42000; Error code: 1049;"); } @Test public void testUserHasNoPermissionToDataBase() { executeQuery("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n"); - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 42000; Error code: 1044;")); + assertStringContains(status.getMessage(), "State code: 42000; Error code: 1044;"); + } + + private static void assertStringContains(final String str, final String target) { + assertTrue(str.contains(target), "Expected message to contain \"" + target + "\" but got " + str); + } + + /** + * Legacy mysql normalization is broken, and uses the FLOAT type for numbers. This rounds off e.g. + * 12345.678 to 12345.7. We can fix this in DV2, but will not fix legacy normalization. As such, + * disabling the test case. + */ + @Override + @Disabled("MySQL normalization uses the wrong datatype for numbers. This will not be fixed, because we intend to replace normalization with DV2.") + public void testDataTypeTestWithNormalization(final String messagesFilename, + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); } } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java index 560f77e46ec2..0750f3393ae3 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java @@ -14,6 +14,7 @@ import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.io.IOs; @@ -25,6 +26,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; /** * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file @@ -159,4 +161,17 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { }); } + /** + * Disabled for the same reason as in {@link MySQLDestinationAcceptanceTest}. But for some reason, + * this class doesn't extend that one so we have to do it again. + */ + @Override + @Disabled("MySQL normalization uses the wrong datatype for numbers. This will not be fixed, because we intend to replace normalization with DV2.") + public void testDataTypeTestWithNormalization(final String messagesFilename, + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); + } + } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java index 59c63a0e2feb..2abd73408c03 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.mysql; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; public class SshPasswordMySQLDestinationAcceptanceTest extends SshMySQLDestinationAcceptanceTest { @@ -13,4 +14,26 @@ public Path getConfigFilePath() { return Path.of("secrets/ssh-pwd-config.json"); } + /** + * Legacy normalization doesn't correctly parse the SSH password (or something). All tests involving + * the normalization container are broken. That's (mostly) fine; DV2 doesn't rely on that container. + */ + @Override + @Disabled("Our dbt interface doesn't correctly parse the SSH password. Won't fix this test, since DV2 will replace normalization.") + public void testSyncWithNormalization(final String messagesFilename, final String catalogFilename) + throws Exception { + super.testSyncWithNormalization(messagesFilename, catalogFilename); + } + + /** + * Similar to {@link #testSyncWithNormalization(String, String)}, disable the custom dbt test. + *

      + * TODO: get custom dbt transformations working https://github.com/airbytehq/airbyte/issues/33547 + */ + @Override + @Disabled("Our dbt interface doesn't correctly parse the SSH password. https://github.com/airbytehq/airbyte/issues/33547 to fix this.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java index bb5eec3faec9..815e661bbbc1 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java @@ -14,6 +14,7 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.sql.SQLException; @@ -23,22 +24,20 @@ import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; public class SslMySQLDestinationAcceptanceTest extends MySQLDestinationAcceptanceTest { - private MySQLContainer db; private DSLContext dslContext; private final StandardNameTransformer namingResolver = new MySQLNameTransformer(); @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, true) .build()); } @@ -46,11 +45,11 @@ protected JsonNode getConfig() { @Override protected JsonNode getFailCheckConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, "wrong password") .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, false) .build()); } @@ -86,8 +85,7 @@ public void testCustomDbtTransformations() { @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { - db = new MySQLContainer<>("mysql:8.0"); - db.start(); + super.setup(testEnv, TEST_SCHEMAS); dslContext = DSLContextFactory.create( db.getUsername(), @@ -98,10 +96,6 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet TES db.getFirstMappedPort(), db.getDatabaseName()), SQLDialect.DEFAULT); - - setLocalInFileToTrue(); - revokeAllPermissions(); - grantCorrectPermissions(); } @Override @@ -151,6 +145,7 @@ private void executeQuery(final String query) { } } + @Override @Test public void testUserHasNoPermissionToDataBase() { executeQuery("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n"); diff --git a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml index 0b66caa7e378..a743a3609f40 100644 --- a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml +++ b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd - dockerImageTag: 0.0.21 + dockerImageTag: 0.0.22 dockerRepository: airbyte/destination-pinecone documentationUrl: https://docs.airbyte.com/integrations/destinations/pinecone githubIssueLabel: destination-pinecone diff --git a/airbyte-integrations/connectors/destination-pinecone/setup.py b/airbyte-integrations/connectors/destination-pinecone/setup.py index 93a1a917a0fe..3e1fbd33d1dc 100644 --- a/airbyte-integrations/connectors/destination-pinecone/setup.py +++ b/airbyte-integrations/connectors/destination-pinecone/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[vector-db-based]==0.55.1", + "airbyte-cdk[vector-db-based]==0.57.0", "pinecone-client[grpc]", ] diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index ca5db84576a7..60e06e23de2d 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] + cdkVersionRequired = '0.23.2' + features = ['db-destinations', 'typing-deduping', 'datastore-postgres'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -26,5 +16,5 @@ application { dependencies { implementation project(':airbyte-integrations:connectors:destination-postgres') - integrationTestJavaImplementation libs.testcontainers.postgresql + integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:destination-postgres')) } diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties index 2b147dcf7175..23da4989675e 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties @@ -1,3 +1,3 @@ -# currently limit the number of parallel threads until further investigation into the issues \ -# where integration tests run into race conditions -testExecutionConcurrency=1 +# our testcontainer has issues with too much concurrency. +# 4 threads seems to be the sweet spot. +testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml index b24719b753e4..43e8aa23e427 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml @@ -1,24 +1,29 @@ data: - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.4.0 + dockerImageTag: 2.0.1 dockerRepository: airbyte/destination-postgres-strict-encrypt + documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres icon: postgresql.svg license: ELv2 name: Postgres - normalizationConfig: - normalizationIntegrationType: postgres - normalizationRepository: airbyte/normalization - normalizationTag: 0.4.1 + registries: + cloud: + enabled: false + oss: + enabled: false + releases: + breakingChanges: + 2.0.0: + message: > + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. + To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). + These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. For more controlled upgrade [see instructions](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#upgrading-connections-one-by-one-with-dual-writing). + upgradeDeadline: "2024-05-31" releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres supportsDbt: true tags: - language:java diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java index 28aa74c8861f..2ac912ae5962 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java @@ -60,6 +60,11 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { return super.check(config); } + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { final Destination destination = new PostgresDestinationStrictEncrypt(); LOGGER.info("starting destination: {}", PostgresDestinationStrictEncrypt.class); diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java index 6bb3602f54e7..7b01a95ab65b 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java @@ -4,43 +4,24 @@ package io.airbyte.integrations.destination.postgres; -import static io.airbyte.cdk.db.PostgresUtils.getCertificate; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.PostgresUtils; -import io.airbyte.cdk.db.factory.DSLContextFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.commons.json.Jsons; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import java.sql.SQLException; -import java.util.ArrayList; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -// todo (cgardens) - DRY this up with PostgresDestinationAcceptanceTest -public class PostgresDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest { +@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") +public class PostgresDestinationStrictEncryptAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private PostgreSQLContainer db; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + private PostgresTestDatabase testDb; protected static final String PASSWORD = "Passw0rd"; - protected static PostgresUtils.Certificate certs; - private static final String NORMALIZATION_VERSION = "dev"; // this is hacky. This test should extend or encapsulate - // PostgresDestinationAcceptanceTest @Override protected String getImageName() { @@ -49,128 +30,45 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() - .put("mode", "verify-full") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs + .put("ca_certificate", testDb.getCertificates().caCertificate()) .build()) - .build()); + .build(); } @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - // Temporarily disabling the behavior of the StandardNameTransformer, see (issue #1785) so we don't - // use quoted names - // if (!tableName.startsWith("\"")) { - // // Currently, Normalization always quote tables identifiers - // //tableName = "\"" + tableName + "\""; - // } - return retrieveRecordsFromTable(tableName, namespace); - } - - @Override - protected List resolveIdentifier(final String identifier) { - final List result = new ArrayList<>(); - final String resolved = namingResolver.getIdentifier(identifier); - result.add(identifier); - result.add(resolved); - if (!resolved.startsWith("\"")) { - result.add(resolved.toLowerCase()); - result.add(resolved.toUpperCase()); - } - return result; - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query( - ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { - db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - db.start(); - certs = getCertificate(db); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - db.stop(); - db.close(); + testDb.close(); } @Test void testStrictSSLUnsecuredNoTunnel() throws Exception { - final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() + final JsonNode config = testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", TunnelMethod.NO_TUNNEL.toString()).build()) + .with("ssl_mode", ImmutableMap.builder() .put("mode", "prefer") .build()) - .put("tunnel_method", ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build()) - .build()); - + .build(); final var actual = runCheck(config); assertEquals(Status.FAILED, actual.getStatus()); assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); @@ -178,21 +76,16 @@ void testStrictSSLUnsecuredNoTunnel() throws Exception { @Test void testStrictSSLSecuredNoTunnel() throws Exception { - final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() + final JsonNode config = testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", TunnelMethod.NO_TUNNEL.toString()).build()) + .with("ssl_mode", ImmutableMap.builder() .put("mode", "require") .build()) - .put("tunnel_method", ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build()) - .build()); - + .build(); final var actual = runCheck(config); assertEquals(Status.SUCCEEDED, actual.getStatus()); } @@ -212,4 +105,14 @@ protected String getDestinationDefinitionKey() { return "airbyte/destination-postgres"; } + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + + @Disabled("Custom DBT does not have root certificate created in the Postgres container.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + } diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.java new file mode 100644 index 000000000000..73d9866dde6a --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.integrations.destination.postgres.PostgresDestination; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; +import javax.sql.DataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +// TODO: This test is added to ensure coverage missed by disabling DATs. Redundant when DATs +// enabled. +public class PostgresStrictEncryptTypingDedupingTest extends AbstractPostgresTypingDedupingTest { + + protected static PostgresTestDatabase testContainer; + + @BeforeAll + public static void setupPostgres() { + // Postgres-13 is alpine image and SSL conf is failing to load, intentionally using 12:bullseye + testContainer = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT); + } + + @AfterAll + public static void teardownPostgres() { + testContainer.close(); + } + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs + .put("ca_certificate", testContainer.getCertificates().caCertificate()) + .build()) + .build(); + } + + @Override + protected DataSource getDataSource(final JsonNode config) { + // Intentionally ignore the config and rebuild it. + // The config param has the resolved (i.e. in-docker) host/port. + // We need the unresolved host/port since the test wrapper code is running from the docker host + // rather than in a container. + return new PostgresDestination().getDataSource(testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withHostAndPort() + .withCredentials() + .withoutSsl() + .build()); + } + + @Override + protected String getImageName() { + return "airbyte/destination-postgres-strict-encrypt:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..9f11b2293a95 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..7f75f0f804e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..61024be7867d --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,5 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..b2bf47df66c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..f3a225756ced --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..4012c086a9e6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b489accda1bb --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..c26d4a49aacd --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..03f28e155af5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..0989dfc17ed0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,9 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..9d1f1499469f --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..33bc3280be27 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..fd2a4b3adbf3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..53c304c89d31 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..2f634c6ad4e9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -0,0 +1,10 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..88b8ee7746c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json index 5410a917e982..c076616ec0b1 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json @@ -209,6 +209,19 @@ "type": "string", "order": 8 }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into", + "title": "Raw table schema (defaults to airbyte_internal)", + "order": 9 + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", + "order": 10 + }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index a685564c1d60..ab746b991351 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -1,35 +1,18 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = [ - 'db-sources', // required for tests - 'db-destinations', - ] + cdkVersionRequired = '0.23.2' + features = ['db-destinations', 'datastore-postgres', 'typing-deduping'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation libs.postgresql - - testImplementation libs.testcontainers.postgresql - - integrationTestJavaImplementation libs.testcontainers.postgresql + testFixturesApi 'org.testcontainers:postgresql:1.19.0' } diff --git a/airbyte-integrations/connectors/destination-postgres/gradle.properties b/airbyte-integrations/connectors/destination-postgres/gradle.properties index 2b147dcf7175..23da4989675e 100644 --- a/airbyte-integrations/connectors/destination-postgres/gradle.properties +++ b/airbyte-integrations/connectors/destination-postgres/gradle.properties @@ -1,3 +1,3 @@ -# currently limit the number of parallel threads until further investigation into the issues \ -# where integration tests run into race conditions -testExecutionConcurrency=1 +# our testcontainer has issues with too much concurrency. +# 4 threads seems to be the sweet spot. +testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index 2b7a003fb13f..af88d008829f 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -1,30 +1,35 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.4.0 + dockerImageTag: 2.0.1 dockerRepository: airbyte/destination-postgres + documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres icon: postgresql.svg license: ELv2 name: Postgres - normalizationConfig: - normalizationIntegrationType: postgres - normalizationRepository: airbyte/normalization - normalizationTag: 0.4.3 registries: cloud: dockerRepository: airbyte/destination-postgres-strict-encrypt enabled: true oss: enabled: true + releases: + breakingChanges: + 2.0.0: + message: > + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. + To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). + These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. For more controlled upgrade [see instructions](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#upgrading-connections-one-by-one-with-dual-writing). + upgradeDeadline: "2024-05-31" releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres + supportLevel: community supportsDbt: true tags: - language:java - ab_internal: - sl: 100 - ql: 200 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 18ba7a7b655e..93c51df74259 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -12,15 +12,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.postgres.typing_deduping.PostgresDestinationHandler; +import io.airbyte.integrations.destination.postgres.typing_deduping.PostgresSqlGenerator; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -41,6 +48,31 @@ public PostgresDestination() { super(DRIVER_CLASS, new PostgresSQLNameTransformer(), new PostgresSqlOperations()); } + @Override + protected DataSourceFactory.DataSourceBuilder modifyDataSourceBuilder(final DataSourceFactory.DataSourceBuilder builder) { + // Anything in the pg_temp schema is only visible to the connection that created it. + // So this creates an airbyte_safe_cast function that only exists for the duration of + // a single connection. + // This avoids issues with creating the same function concurrently (e.g. if multiple syncs run + // at the same time). + // Function definition copied from https://dba.stackexchange.com/a/203986 + + // Adding 60 seconds to connection timeout, for ssl connections, default 10 seconds is not enough + return builder.withConnectionTimeout(Duration.ofSeconds(60)) + .withConnectionInitSql(""" + CREATE FUNCTION pg_temp.airbyte_safe_cast(_in text, INOUT _out ANYELEMENT) + LANGUAGE plpgsql AS + $func$ + BEGIN + EXECUTE format('SELECT %L::%s', $1, pg_typeof(_out)) + INTO _out; + EXCEPTION WHEN others THEN + -- do nothing: _out already carries default + END + $func$; + """); + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { final Map additionalParameters = new HashMap<>(); @@ -67,7 +99,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { if (encodedDatabase != null) { try { encodedDatabase = URLEncoder.encode(encodedDatabase, "UTF-8"); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { // Should never happen e.printStackTrace(); } @@ -93,6 +125,21 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + + @Override + protected JdbcDestinationHandler getDestinationHandler(String databaseName, JdbcDatabase database) { + return new PostgresDestinationHandler(databaseName, database); + } + + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { final Destination destination = PostgresDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", PostgresDestination.class); diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java index dab1eef5560b..210cc6d9bc4d 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java @@ -4,9 +4,12 @@ package io.airbyte.integrations.destination.postgres; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.*; + import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -14,7 +17,9 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.sql.SQLException; +import java.util.Collections; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.postgresql.copy.CopyManager; import org.postgresql.core.BaseConnection; @@ -24,16 +29,56 @@ public PostgresSqlOperations() { super(new PostgresDataAdapter()); } + @Override + protected List postCreateTableQueries(final String schemaName, final String tableName) { + if (TypingAndDedupingFlag.isDestinationV2()) { + return List.of( + // the raw_id index _could_ be unique (since raw_id is a UUID) + // but there's no reason to do that (because it's a UUID :P ) + // and it would just slow down inserts. + // also, intentionally don't specify the type of index (btree, hash, etc). Just use the default. + "CREATE INDEX IF NOT EXISTS " + tableName + "_raw_id" + " ON " + schemaName + "." + tableName + "(_airbyte_raw_id)", + "CREATE INDEX IF NOT EXISTS " + tableName + "_extracted_at" + " ON " + schemaName + "." + tableName + "(_airbyte_extracted_at)", + "CREATE INDEX IF NOT EXISTS " + tableName + "_loaded_at" + " ON " + schemaName + "." + tableName + + "(_airbyte_loaded_at, _airbyte_extracted_at)"); + } else { + return Collections.emptyList(); + } + } + + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + insertRecordsInternal(database, records, schemaName, tableName, + COLUMN_NAME_AB_RAW_ID, + COLUMN_NAME_DATA, + COLUMN_NAME_AB_EXTRACTED_AT, + COLUMN_NAME_AB_LOADED_AT); + } + @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { + insertRecordsInternal(database, records, schemaName, tmpTableName, COLUMN_NAME_AB_ID, COLUMN_NAME_DATA, COLUMN_NAME_EMITTED_AT); + } + + private void insertRecordsInternal(final JdbcDatabase database, + final List records, + final String schemaName, + final String tmpTableName, + final String... columnNames) + throws SQLException { if (records.isEmpty()) { return; } - + // Explicitly passing column order to avoid order mismatches between CREATE TABLE and COPY statement + final String orderedColumnNames = StringUtils.join(columnNames, ", "); database.execute(connection -> { File tmpFile = null; try { @@ -41,7 +86,7 @@ public void insertRecordsInternal(final JdbcDatabase database, writeBatchToFile(tmpFile, records); final var copyManager = new CopyManager(connection.unwrap(BaseConnection.class)); - final var sql = String.format("COPY %s.%s FROM stdin DELIMITER ',' CSV", schemaName, tmpTableName); + final var sql = String.format("COPY %s.%s (%s) FROM stdin DELIMITER ',' CSV", schemaName, tmpTableName, orderedColumnNames); final var bufferedReader = new BufferedReader(new FileReader(tmpFile, StandardCharsets.UTF_8)); copyManager.copyIn(sql, bufferedReader); } catch (final Exception e) { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresDestinationHandler.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresDestinationHandler.java new file mode 100644 index 000000000000..21cc549b3d38 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresDestinationHandler.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; + +public class PostgresDestinationHandler extends JdbcDestinationHandler { + + public PostgresDestinationHandler(String databaseName, JdbcDatabase jdbcDatabase) { + super(databaseName, jdbcDatabase); + } + + @Override + protected String toJdbcTypeName(AirbyteType airbyteType) { + // This is mostly identical to the postgres implementation, but swaps jsonb to super + if (airbyteType instanceof final AirbyteProtocolType airbyteProtocolType) { + return toJdbcTypeName(airbyteProtocolType); + } + return switch (airbyteType.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE, Array.TYPE -> "jsonb"; + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> toJdbcTypeName(((Union) airbyteType).chooseType()); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + airbyteType); + }; + } + + private String toJdbcTypeName(final AirbyteProtocolType airbyteProtocolType) { + return switch (airbyteProtocolType) { + case STRING -> "varchar"; + case NUMBER -> "numeric"; + case INTEGER -> "int8"; + case BOOLEAN -> "bool"; + case TIMESTAMP_WITH_TIMEZONE -> "timestamptz"; + case TIMESTAMP_WITHOUT_TIMEZONE -> "timestamp"; + case TIME_WITH_TIMEZONE -> "timetz"; + case TIME_WITHOUT_TIMEZONE -> "time"; + case DATE -> "date"; + case UNKNOWN -> "jsonb"; + }; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java new file mode 100644 index 000000000000..9d7217e3f826 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static java.util.Collections.emptyList; +import static org.jooq.impl.DSL.array; +import static org.jooq.impl.DSL.case_; +import static org.jooq.impl.DSL.cast; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.function; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.quotedName; +import static org.jooq.impl.DSL.rowNumber; +import static org.jooq.impl.DSL.val; + +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.Condition; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultDataType; +import org.jooq.impl.SQLDataType; + +public class PostgresSqlGenerator extends JdbcSqlGenerator { + + public static final DataType JSONB_TYPE = new DefaultDataType<>(null, Object.class, "jsonb"); + + public PostgresSqlGenerator(final NamingConventionTransformer namingTransformer) { + super(namingTransformer); + } + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + // There is a mismatch between convention used in create table query in SqlOperations vs this. + // For postgres specifically, when a create table is issued without a quoted identifier, it will be + // converted to lowercase. + // To keep it consistent when querying raw table in T+D query, convert it to lowercase. + // TODO: This logic should be unified across Raw and final table operations in a single class + // operating on a StreamId. + return new StreamId( + namingTransformer.getNamespace(namespace), + namingTransformer.convertStreamName(name), + namingTransformer.getNamespace(rawNamespaceOverride).toLowerCase(), + namingTransformer.convertStreamName(StreamId.concatenateRawTableName(namespace, name)).toLowerCase(), + namespace, + name); + } + + @Override + protected DataType getStructType() { + return JSONB_TYPE; + } + + @Override + protected DataType getArrayType() { + return JSONB_TYPE; + } + + @Override + protected DataType getWidestType() { + return JSONB_TYPE; + } + + @Override + protected SQLDialect getDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public DataType toDialectType(AirbyteProtocolType airbyteProtocolType) { + if (airbyteProtocolType.equals(AirbyteProtocolType.STRING)) { + // https://www.postgresql.org/docs/current/datatype-character.html + // If specified, the length n must be greater than zero and cannot exceed 10,485,760 (10 MB). + // If you desire to store long strings with no specific upper limit, + // use text or character varying without a length specifier, + // rather than making up an arbitrary length limit. + return SQLDataType.VARCHAR; + } + return super.toDialectType(airbyteProtocolType); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + final List statements = new ArrayList<>(); + final Name finalTableName = name(stream.id().finalNamespace(), stream.id().finalName() + suffix); + + statements.add(super.createTable(stream, suffix, force)); + + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + // An index for our ROW_NUMBER() PARTITION BY pk ORDER BY cursor, extracted_at function + final List pkNames = stream.primaryKey().stream() + .map(pk -> quotedName(pk.name())) + .toList(); + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + Stream.of( + pkNames.stream(), + // if cursor is present, then a stream containing its name + // but if no cursor, then empty stream + stream.cursor().stream().map(cursor -> quotedName(cursor.name())), + Stream.of(name(COLUMN_NAME_AB_EXTRACTED_AT))).flatMap(Function.identity()).toList()) + .getSQL())); + } + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + name(COLUMN_NAME_AB_EXTRACTED_AT)) + .getSQL())); + + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + name(COLUMN_NAME_AB_RAW_ID)) + .getSQL())); + + return Sql.concat(statements); + } + + @Override + protected List createIndexSql(final StreamConfig stream, final String suffix) { + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && !stream.primaryKey().isEmpty()) { + return List.of( + getDslContext().createIndex().on( + name(stream.id().finalNamespace(), stream.id().finalName() + suffix), + stream.primaryKey().stream() + .map(pk -> quotedName(pk.name())) + .toList()) + .getSQL()); + } else { + return emptyList(); + } + } + + @Override + protected List> extractRawDataFields(final LinkedHashMap columns, final boolean useExpensiveSaferCasting) { + return columns + .entrySet() + .stream() + .map(column -> castedField( + extractColumnAsJson(column.getKey()), + column.getValue(), + column.getKey().name(), + useExpensiveSaferCasting)) + .collect(Collectors.toList()); + } + + @Override + protected Field castedField( + final Field field, + final AirbyteType type, + final String alias, + final boolean useExpensiveSaferCasting) { + return castedField(field, type, useExpensiveSaferCasting).as(quotedName(alias)); + } + + protected Field castedField( + final Field field, + final AirbyteType type, + final boolean useExpensiveSaferCasting) { + if (type instanceof Struct) { + // If this field is a struct, verify that the raw data is an object. + return cast( + case_() + .when(field.isNull().or(jsonTypeof(field).ne("object")), val((Object) null)) + .else_(field), + JSONB_TYPE); + } else if (type instanceof Array) { + // Do the same for arrays. + return cast( + case_() + .when(field.isNull().or(jsonTypeof(field).ne("array")), val((Object) null)) + .else_(field), + JSONB_TYPE); + } else if (type == AirbyteProtocolType.UNKNOWN) { + return cast(field, JSONB_TYPE); + } else if (type == AirbyteProtocolType.STRING) { + // we need to render the jsonb to a normal string. For strings, this is the difference between + // "\"foo\"" and "foo". + // postgres provides the #>> operator, which takes a json path and returns that extraction as a + // string. + // '{}' is an empty json path (it's an empty array literal), so it just stringifies the json value. + return field("{0} #>> '{}'", String.class, field); + } else { + final DataType dialectType = toDialectType(type); + // jsonb can't directly cast to most types, so convert to text first. + // also convert jsonb null to proper sql null. + final Field extractAsText = case_() + .when(field.isNull().or(jsonTypeof(field).eq("null")), val((String) null)) + .else_(cast(field, SQLDataType.VARCHAR)); + if (useExpensiveSaferCasting) { + return function(name("pg_temp", "airbyte_safe_cast"), dialectType, extractAsText, cast(val((Object) null), dialectType)); + } else { + return cast(extractAsText, dialectType); + } + } + } + + // TODO this isn't actually used right now... can we refactor this out? + // (redshift is doing something interesting with this method, so leaving it for now) + @Override + protected Field castedField(final Field field, final AirbyteProtocolType type, final boolean useExpensiveSaferCasting) { + return cast(field, toDialectType(type)); + } + + @Override + protected Field buildAirbyteMetaColumn(final LinkedHashMap columns) { + final Field[] dataFieldErrors = columns + .entrySet() + .stream() + .map(column -> toCastingErrorCaseStmt(column.getKey(), column.getValue())) + .toArray(Field[]::new); + return function( + "JSONB_BUILD_OBJECT", + JSONB_TYPE, + val("errors"), + function("ARRAY_REMOVE", JSONB_TYPE, array(dataFieldErrors), val((String) null))).as(COLUMN_NAME_AB_META); + } + + private Field toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) { + final Field extract = extractColumnAsJson(column); + if (type instanceof Struct) { + // If this field is a struct, verify that the raw data is an object or null. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).notIn("object", "null")), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } else if (type instanceof Array) { + // Do the same for arrays. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).notIn("array", "null")), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } else if (type == AirbyteProtocolType.UNKNOWN || type == AirbyteProtocolType.STRING) { + // Unknown types require no casting, so there's never an error. + // Similarly, everything can cast to string without error. + return val((String) null); + } else { + // For other type: If the raw data is not NULL or 'null', but the casted data is NULL, + // then we have a typing error. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).ne("null")) + .and(castedField(extract, type, true).isNull()), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } + } + + @Override + protected Condition cdcDeletedAtNotNullCondition() { + return field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull() + .and(jsonTypeof(extractColumnAsJson(cdcDeletedAtColumn)).ne("null")); + } + + @Override + protected Field getRowNumber(final List primaryKeys, final Optional cursor) { + // literally identical to redshift's getRowNumber implementation, changes here probably should + // be reflected there + final List> primaryKeyFields = + primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.name()))).collect(Collectors.toList()) + : new ArrayList<>(); + final List> orderedFields = new ArrayList<>(); + // We can still use Jooq's field to get the quoted name with raw sql templating. + // jooq's .desc returns SortField instead of Field and NULLS LAST doesn't work with it + cursor.ifPresent(columnId -> orderedFields.add(field("{0} desc NULLS LAST", field(quotedName(columnId.name()))))); + orderedFields.add(field("{0} desc", quotedName(COLUMN_NAME_AB_EXTRACTED_AT))); + return rowNumber() + .over() + .partitionBy(primaryKeyFields) + .orderBy(orderedFields).as(ROW_NUMBER_COLUMN_NAME); + } + + /** + * Extract a raw field, leaving it as jsonb + */ + private Field extractColumnAsJson(final ColumnId column) { + return field("{0} -> {1}", name(COLUMN_NAME_DATA), val(column.originalName())); + } + + private Field jsonTypeof(final Field field) { + return function("JSONB_TYPEOF", SQLDataType.VARCHAR, field); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json index e310cb5a10f0..4c775be8b887 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json @@ -215,6 +215,19 @@ "title": "JDBC URL Params", "type": "string", "order": 8 + }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into", + "title": "Raw table schema (defaults to airbyte_internal)", + "order": 9 + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", + "order": 10 } } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java index 3566ea78cd45..1bec8925facb 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java @@ -5,138 +5,39 @@ package io.airbyte.integrations.destination.postgres; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.factory.DSLContextFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.cdk.integrations.util.HostPortResolver; -import io.airbyte.commons.json.Jsons; -import java.sql.SQLException; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; +import org.junit.jupiter.api.Disabled; -public class PostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { +@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") +public class PostgresDestinationAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private PostgreSQLContainer db; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - - @Override - protected String getImageName() { - return "airbyte/destination-postgres:dev"; - } + private PostgresTestDatabase testDb; @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withoutSsl() + .build(); } @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected boolean supportIncrementalSchemaChanges() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query(ctx -> { - ctx.execute("set time zone 'UTC';"); - return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(this::getJsonFromRecord) - .collect(Collectors.toList()); - }); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { - db = new PostgreSQLContainer<>("postgres:13-alpine"); - db.start(); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_13); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - db.stop(); - db.close(); + testDb.close(); } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java index 9428137b8369..fa92f3a9f663 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java @@ -4,34 +4,16 @@ package io.airbyte.integrations.destination.postgres; -import static io.airbyte.cdk.db.PostgresUtils.getCertificate; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.PostgresUtils; -import io.airbyte.cdk.db.factory.DSLContextFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.commons.json.Jsons; -import java.sql.SQLException; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -public class PostgresDestinationSSLFullCertificateAcceptanceTest extends JdbcDestinationAcceptanceTest { +import org.junit.jupiter.api.Disabled; - private PostgreSQLContainer db; +@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") +public class PostgresDestinationSSLFullCertificateAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - protected static PostgresUtils.Certificate certs; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + private PostgresTestDatabase testDb; @Override protected String getImageName() { @@ -40,111 +22,36 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("username", "postgres") - .put("password", "postgres") - .put("schema", "public") - .put("port", db.getFirstMappedPort()) - .put("database", db.getDatabaseName()) - .put("ssl", true) - .put("ssl_mode", ImmutableMap.builder() - .put("mode", "verify-full") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", "Passw0rd") + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs + .put("ca_certificate", testDb.getCertificates().caCertificate()) .build()) - .build()); - } - - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("username", db.getUsername()) - .put("password", "wrong password") - .put("schema", "public") - .put("port", db.getFirstMappedPort()) - .put("database", db.getDatabaseName()) - .put("ssl", false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); + .build(); } @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query(ctx -> { - ctx.execute("set time zone 'UTC';"); - return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(this::getJsonFromRecord) - .collect(Collectors.toList()); - }); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { - db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - db.start(); - certs = getCertificate(db); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - db.stop(); - db.close(); + testDb.close(); + } + + @Disabled("Custom DBT does not have root certificate created in the Postgres container.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java index ce0e53ca1fdf..0bab6cb695e5 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.destination.postgres; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") public class SshKeyPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java index 6b6f32a3947d..454058e4a99c 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java @@ -5,7 +5,10 @@ package io.airbyte.integrations.destination.postgres; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.TestCompatibility; +import org.junit.jupiter.api.Disabled; +@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") public class SshPasswordPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest { @Override @@ -13,4 +16,33 @@ public SshTunnel.TunnelMethod getTunnelMethod() { return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; } + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + public void testIncrementalDedupeSync() throws Exception { + super.testIncrementalDedupeSync(); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testDataTypeTestWithNormalization(String messagesFilename, + String catalogFilename, + TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testSyncWithNormalization(String messagesFilename, String catalogFilename) throws Exception { + super.testSyncWithNormalization(messagesFilename, catalogFilename); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + + // TODO: Although testCustomDbtTransformationsFailure is passing, the failure is for wrong reasons. + // See disabled tests. + } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java index 7a699ee519bd..4412909f1539 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.postgres; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.CONNECTION_OPTIONS_KEY; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.cdk.db.Database; @@ -13,95 +15,33 @@ import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.ContainerModifier; import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.RandomStringUtils; import org.jooq.SQLDialect; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; - -// todo (cgardens) - likely some of this could be further de-duplicated with -// PostgresDestinationAcceptanceTest. /** * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file * or with a password. */ -public abstract class SshPostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { +public abstract class SshPostgresDestinationAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - private static final String schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase(); - private static final Network network = Network.newNetwork(); - private static PostgreSQLContainer db; - private final SshBastionContainer bastion = new SshBastionContainer(); + private PostgresTestDatabase testdb; + private SshBastionContainer bastion; public abstract SshTunnel.TunnelMethod getTunnelMethod(); - @Override - protected String getImageName() { - return "airbyte/destination-postgres:dev"; - } - @Override protected JsonNode getConfig() throws Exception { - return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db).put("schema", schemaName), false); - } - - @Override - protected JsonNode getFailCheckConfig() throws Exception { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("password", "wrong password"); - return clone; - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); + // Here we use inner address because the tunnel is created inside the connector's container. + return testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), true)) + .with("schema", "public") + .withoutSsl() + .build(); } private static Database getDatabaseFromConfig(final JsonNode config) { @@ -117,8 +57,16 @@ private static Database getDatabaseFromConfig(final JsonNode config) { SQLDialect.POSTGRES)); } - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { - final JsonNode config = getConfig(); + @Override + protected List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { + // Here we DO NOT use the inner address because the tunnel is created in the integration test's java + // process. + final JsonNode config = testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .with("schema", "public") + .withoutSsl() + .build(); + ((ObjectNode) config).putObject(CONNECTION_OPTIONS_KEY); return SshTunnel.sshWrap( config, JdbcUtils.HOST_LIST_KEY, @@ -135,42 +83,20 @@ private List retrieveRecordsFromTable(final String tableName, final St @Override protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { - - startTestContainers(); - // do everything in a randomly generated schema so that we can wipe it out at the end. - SshTunnel.sshWrap( - getConfig(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY, - mangledConfig -> { - getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName))); - TEST_SCHEMAS.add(schemaName); - }); - } - - private void startTestContainers() { - bastion.initAndStartBastion(network); - initAndStartJdbcContainer(); - } - - private void initAndStartJdbcContainer() { - db = new PostgreSQLContainer<>("postgres:13-alpine") - .withNetwork(network); - db.start(); + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_13, ContainerModifier.NETWORK); + bastion = new SshBastionContainer(); + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - // blow away the test schema at the end. - SshTunnel.sshWrap( - getConfig(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY, - mangledConfig -> { - getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("DROP SCHEMA %s CASCADE;", schemaName))); - }); + testdb.close(); + bastion.stopAndClose(); + } - bastion.stopAndCloseContainers(db); + @Override + protected PostgresTestDatabase getTestDb() { + return testdb; } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..a4841e7b44d1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideDisableTypingDedupingTest.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class PostgresRawOverrideDisableTypingDedupingTest extends PostgresTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return super.getBaseConfig() + .put("raw_data_schema", "overridden_raw_dataset") + .put("disable_type_dedupe", true); + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Disabled + @Test + @Override + public void identicalNameSimultaneousSync() {} + + @Disabled + @Test + @Override + public void testVarcharLimitOver64K() {} + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java new file mode 100644 index 000000000000..f31c3325d226 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class PostgresRawOverrideTypingDedupingTest extends PostgresTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return super.getBaseConfig() + .put("raw_data_schema", "overridden_raw_dataset"); + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..6efac136e4c3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import static io.airbyte.integrations.destination.postgres.typing_deduping.PostgresSqlGenerator.JSONB_TYPE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.destination.postgres.PostgresDestination; +import io.airbyte.integrations.destination.postgres.PostgresSQLNameTransformer; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase; +import java.util.List; +import javax.sql.DataSource; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class PostgresSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { + + private static PostgresTestDatabase testContainer; + private static String databaseName; + private static JdbcDatabase database; + + @BeforeAll + public static void setupPostgres() { + testContainer = PostgresTestDatabase.in(PostgresTestDatabase.BaseImage.POSTGRES_13); + final JsonNode config = testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withHostAndPort() + .withCredentials() + .withoutSsl() + .build(); + + databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + final PostgresDestination postgresDestination = new PostgresDestination(); + final DataSource dataSource = postgresDestination.getDataSource(config); + database = new DefaultJdbcDatabase(dataSource, new PostgresSourceOperations()); + } + + @AfterAll + public static void teardownPostgres() { + testContainer.close(); + } + + @Override + protected JdbcDatabase getDatabase() { + return database; + } + + @Override + protected DataType getStructType() { + return JSONB_TYPE; + } + + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + + @Override + protected DestinationHandler getDestinationHandler() { + return new PostgresDestinationHandler(databaseName, database); + } + + @Override + protected SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + protected Field toJsonValue(final String valueAsString) { + return DSL.cast(DSL.val(valueAsString), JSONB_TYPE); + } + + @Test + @Override + public void testCreateTableIncremental() throws Exception { + final Sql sql = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(sql); + + List initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + assertEquals(1, initialStates.size()); + final DestinationInitialState initialState = initialStates.getFirst(); + assertTrue(initialState.isFinalTablePresent()); + assertFalse(initialState.isSchemaMismatch()); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java new file mode 100644 index 000000000000..4bf10317d207 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.integrations.destination.postgres.PostgresDestination; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase; +import javax.sql.DataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +public class PostgresTypingDedupingTest extends AbstractPostgresTypingDedupingTest { + + protected static PostgresTestDatabase testContainer; + + @BeforeAll + public static void setupPostgres() { + testContainer = PostgresTestDatabase.in(PostgresTestDatabase.BaseImage.POSTGRES_13); + } + + @AfterAll + public static void teardownPostgres() { + testContainer.close(); + } + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withoutSsl() + .build(); + } + + @Override + protected DataSource getDataSource(final JsonNode config) { + // Intentionally ignore the config and rebuild it. + // The config param has the resolved (i.e. in-docker) host/port. + // We need the unresolved host/port since the test wrapper code is running from the docker host + // rather than in a container. + return new PostgresDestination().getDataSource(testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withHostAndPort() + .withCredentials() + .withoutSsl() + .build()); + } + + @Override + protected String getImageName() { + return "airbyte/destination-postgres:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..9f11b2293a95 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..7f75f0f804e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..61024be7867d --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,5 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..b2bf47df66c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..f3a225756ced --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..4012c086a9e6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b489accda1bb --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..c26d4a49aacd --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..03f28e155af5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..0989dfc17ed0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,9 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..9d1f1499469f --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..33bc3280be27 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..fd2a4b3adbf3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..53c304c89d31 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..2f634c6ad4e9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -0,0 +1,10 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..88b8ee7746c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl new file mode 100644 index 000000000000..76d0442ebe79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -0,0 +1,8 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`","Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`","Problem with `time_without_timezone`", "Problem with `date`"]}} +// Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. +// But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "IamACaseSensitiveColumnName": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..6b99169ececf --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl new file mode 100644 index 000000000000..5842f7b37e42 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00.000000Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..63569975abc2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl new file mode 100644 index 000000000000..edcc0cc462d6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -0,0 +1,5 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\", \"am\", \"an\", \"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\": \"am\", \"an\": \"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..5c10203c7837 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl new file mode 100644 index 000000000000..4ecd95d83b63 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..cd7c03aba677 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl new file mode 100644 index 000000000000..b34ad054ab33 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_airbyte_extracted_at":"2023-01-01T00:00:00.000000Z","_airbyte_meta":{"errors":[]}, "current_date": "foo", "join": "bar"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl new file mode 100644 index 000000000000..78ded5f99d0e --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -0,0 +1,16 @@ +// https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timetz +// TIME, TIMETZ, TIMESTAMP, TIMESTAMPTZ values are UTC in user tables. +// Note that redshift stores precision to microseconds. Java deserialization in tests preserves them only for non-zero values +// except for timestamp with time zone where Z is required at end for even zero values +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000Z", "time_with_timezone": "12:34:56.123Z"} + +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} +// Bigquery returns 6 decimal places if there are any decimal places... but not for timestamp_with_timezone +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl new file mode 100644 index 000000000000..adfbd06d6a55 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -0,0 +1,9 @@ +// column renamings: +// * $starts_with_dollar_sign -> _starts_with_dollar_sign +// * includes"doublequote -> includes_doublequote +// * includes'singlequote -> includes_singlequote +// * includes`backtick -> includes_backtick +// * includes$$doubledollar -> includes__doubledollar +// * includes.period -> includes_period +// * endswithbackslash\ -> endswithbackslash_ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo", "endswithbackslash_": "foo"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..2b602082a349 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index 2fbc0a05d184..84eb63509696 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -15,8 +15,9 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -30,6 +31,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.HashMap; import java.util.List; @@ -240,20 +242,24 @@ public void testUserHasNoPermissionToDataBase() throws Exception { @Test void sanityTest() throws Exception { final Destination destination = new PostgresDestination(); - final AirbyteMessageConsumer consumer = destination.getConsumer(config, CATALOG, Destination::defaultOutputRecordCollector); + DestinationConfig.initialize(config); + final SerializedAirbyteMessageConsumer consumer = + destination.getSerializedMessageConsumer(config, CATALOG, Destination::defaultOutputRecordCollector); final List expectedRecords = getNRecords(10); consumer.start(); expectedRecords.forEach(m -> { try { - consumer.accept(m); + String message = Jsons.serialize(m); + consumer.accept(message, message.getBytes(StandardCharsets.UTF_8).length); } catch (final Exception e) { throw new RuntimeException(e); } }); - consumer.accept(new AirbyteMessage() + final String stateMessage = Jsons.serialize(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of(SCHEMA_NAME + "." + STREAM_NAME, 10))))); + consumer.accept(stateMessage, stateMessage.getBytes(StandardCharsets.UTF_8).length); consumer.close(); final JdbcDatabase database = getJdbcDatabaseFromConfig(getDataSourceFromConfig(config)); diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java new file mode 100644 index 000000000000..d5bb6d01fcb0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.stream.Collectors; + +public abstract class AbstractPostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { + + public static final String DEFAULT_DEV_IMAGE = "airbyte/destination-postgres:dev"; + + private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + + @Override + protected String getImageName() { + return DEFAULT_DEV_IMAGE; + } + + @Override + protected JsonNode getFailCheckConfig() throws Exception { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("password", "wrong password"); + return clone; + } + + @Override + protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) + throws Exception { + final String tableName = namingResolver.getIdentifier(streamName); + return retrieveRecordsFromTable(tableName, namespace); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv env, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + .stream() + .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) + .collect(Collectors.toList()); + } + + protected List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { + // TODO: Change emitted_at with DV2 + return getTestDb().query(ctx -> { + ctx.execute("set time zone 'UTC';"); + return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + .stream() + .map(this::getJsonFromRecord) + .collect(Collectors.toList()); + }); + } + + protected abstract PostgresTestDatabase getTestDb(); + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new PostgresTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java new file mode 100644 index 000000000000..60e588214f72 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import io.airbyte.cdk.testutils.ContainerFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +/** + * TODO: This class is a copy from source-postgres:testFixtures. Eventually merge into a common + * fixtures module. + */ +public class PostgresContainerFactory extends ContainerFactory> { + + @Override + protected PostgreSQLContainer createNewContainer(DockerImageName imageName) { + return new PostgreSQLContainer<>(imageName.asCompatibleSubstituteFor("postgres")); + } + + /** + * Apply the postgresql.conf file that we've packaged as a resource. + */ + public void withConf(PostgreSQLContainer container) { + container + .withCopyFileToContainer( + MountableFile.forClasspathResource("postgresql.conf"), + "/etc/postgresql/postgresql.conf") + .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); + } + + /** + * Create a new network and bind it to the container. + */ + public void withNetwork(PostgreSQLContainer container) { + container.withNetwork(Network.newNetwork()); + } + + /** + * Configure postgres with wal_level=logical. + */ + public void withWalLevelLogical(PostgreSQLContainer container) { + container.withCommand("postgres -c wal_level=logical"); + } + + /** + * Generate SSL certificates and tell postgres to enable SSL and use them. + */ + public void withCert(PostgreSQLContainer container) { + container.start(); + String[] commands = { + "psql -U test -c \"CREATE USER postgres WITH PASSWORD 'postgres';\"", + "psql -U test -c \"GRANT CONNECT ON DATABASE \"test\" TO postgres;\"", + "psql -U test -c \"ALTER USER postgres WITH SUPERUSER;\"", + "openssl ecparam -name prime256v1 -genkey -noout -out ca.key", + "openssl req -new -x509 -sha256 -key ca.key -out ca.crt -subj \"/CN=127.0.0.1\"", + "openssl ecparam -name prime256v1 -genkey -noout -out server.key", + "openssl req -new -sha256 -key server.key -out server.csr -subj \"/CN=localhost\"", + "openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256", + "cp server.key /etc/ssl/private/", + "cp server.crt /etc/ssl/private/", + "cp ca.crt /etc/ssl/private/", + "chmod og-rwx /etc/ssl/private/server.* /etc/ssl/private/ca.*", + "chown postgres:postgres /etc/ssl/private/server.crt /etc/ssl/private/server.key /etc/ssl/private/ca.crt", + "echo \"ssl = on\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_cert_file = '/etc/ssl/private/server.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_key_file = '/etc/ssl/private/server.key'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_ca_file = '/etc/ssl/private/ca.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "mkdir root/.postgresql", + "echo \"hostssl all all 127.0.0.1/32 cert clientcert=verify-full\" >> /var/lib/postgresql/data/pg_hba.conf", + "openssl ecparam -name prime256v1 -genkey -noout -out client.key", + "openssl req -new -sha256 -key client.key -out client.csr -subj \"/CN=postgres\"", + "openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365 -sha256", + "cp client.crt ~/.postgresql/postgresql.crt", + "cp client.key ~/.postgresql/postgresql.key", + "chmod 0600 ~/.postgresql/postgresql.crt ~/.postgresql/postgresql.key", + "cp ca.crt root/.postgresql/ca.crt", + "chown postgres:postgres ~/.postgresql/ca.crt", + "psql -U test -c \"SELECT pg_reload_conf();\"", + }; + for (String cmd : commands) { + try { + container.execInContainer("su", "-c", cmd); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Tell postgres to enable SSL. + */ + public void withSSL(PostgreSQLContainer container) { + container.withCommand("postgres " + + "-c ssl=on " + + "-c ssl_cert_file=/var/lib/postgresql/server.crt " + + "-c ssl_key_file=/var/lib/postgresql/server.key"); + } + + /** + * Configure postgres with client_encoding=sql_ascii. + */ + public void withASCII(PostgreSQLContainer container) { + container.withCommand("postgres -c client_encoding=sql_ascii"); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java similarity index 100% rename from airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java rename to airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java new file mode 100644 index 000000000000..31fb23b9fa79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * TODO: This class is a copy from source-postgres:testFixtures. Eventually merge into a common + * fixtures module. + */ +public class PostgresTestDatabase extends + TestDatabase, PostgresTestDatabase, PostgresTestDatabase.PostgresConfigBuilder> { + + public enum BaseImage { + + POSTGRES_16("postgres:16-bullseye"), + POSTGRES_12("postgres:12-bullseye"), + POSTGRES_13("postgres:13-alpine"), + POSTGRES_9("postgres:9-alpine"), + POSTGRES_SSL_DEV("marcosmarxm/postgres-ssl:dev"); + + private final String reference; + + private BaseImage(String reference) { + this.reference = reference; + }; + + } + + public static enum ContainerModifier { + + ASCII("withASCII"), + CONF("withConf"), + NETWORK("withNetwork"), + SSL("withSSL"), + WAL_LEVEL_LOGICAL("withWalLevelLogical"), + CERT("withCert"), + ; + + private String methodName; + + private ContainerModifier(String methodName) { + this.methodName = methodName; + } + + } + + static public PostgresTestDatabase in(BaseImage baseImage, ContainerModifier... modifiers) { + String[] methodNames = Stream.of(modifiers).map(im -> im.methodName).toList().toArray(new String[0]); + final var container = new PostgresContainerFactory().shared(baseImage.reference, methodNames); + return new PostgresTestDatabase(container).initialized(); + } + + public PostgresTestDatabase(PostgreSQLContainer container) { + super(container); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.of(psqlCmd(Stream.of( + String.format("CREATE DATABASE %s", getDatabaseName()), + String.format("CREATE USER %s PASSWORD '%s'", getUserName(), getPassword()), + String.format("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", getDatabaseName(), getUserName()), + String.format("ALTER USER %s WITH SUPERUSER", getUserName())))); + } + + /** + * Close resources held by this instance. This deliberately avoids dropping the database, which is + * really expensive in Postgres. This is because a DROP DATABASE in Postgres triggers a CHECKPOINT. + * Call {@link #dropDatabaseAndUser} to explicitly drop the database and the user. + */ + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + /** + * Drop the database owned by this instance. + */ + public void dropDatabaseAndUser() { + execInContainer(psqlCmd(Stream.of( + String.format("DROP DATABASE %s", getDatabaseName()), + String.format("DROP OWNED BY %s", getUserName()), + String.format("DROP USER %s", getUserName())))); + } + + public Stream psqlCmd(Stream sql) { + return Stream.concat( + Stream.of("psql", + "-d", getContainer().getDatabaseName(), + "-U", getContainer().getUsername(), + "-v", "ON_ERROR_STOP=1", + "-a"), + sql.flatMap(stmt -> Stream.of("-c", stmt))); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + private Certificates cachedCerts; + + public synchronized Certificates getCertificates() { + if (cachedCerts == null) { + final String caCert, clientKey, clientCert; + try { + caCert = getContainer().execInContainer("su", "-c", "cat ca.crt").getStdout().trim(); + clientKey = getContainer().execInContainer("su", "-c", "cat client.key").getStdout().trim(); + clientCert = getContainer().execInContainer("su", "-c", "cat client.crt").getStdout().trim(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + cachedCerts = new Certificates(caCert, clientCert, clientKey); + } + return cachedCerts; + } + + public record Certificates(String caCertificate, String clientCertificate, String clientKey) {} + + @Override + public PostgresConfigBuilder configBuilder() { + return new PostgresConfigBuilder(this); + } + + public String getReplicationSlotName() { + return withNamespace("debezium_slot"); + } + + public String getPublicationName() { + return withNamespace("publication"); + } + + public PostgresTestDatabase withReplicationSlot() { + return this + .with("SELECT pg_create_logical_replication_slot('%s', 'pgoutput');", getReplicationSlotName()) + .onClose("SELECT pg_drop_replication_slot('%s');", getReplicationSlotName()); + } + + public PostgresTestDatabase withPublicationForAllTables() { + return this + .with("CREATE PUBLICATION %s FOR ALL TABLES;", getPublicationName()) + .onClose("DROP PUBLICATION %s CASCADE;", getPublicationName()); + } + + static public class PostgresConfigBuilder extends ConfigBuilder { + + protected PostgresConfigBuilder(PostgresTestDatabase testdb) { + super(testdb); + } + + public PostgresConfigBuilder withSchemas(String... schemas) { + return with(JdbcUtils.SCHEMAS_KEY, List.of(schemas)); + } + + public PostgresConfigBuilder withStandardReplication() { + return with("replication_method", ImmutableMap.builder().put("method", "Standard").build()); + } + + public PostgresConfigBuilder withCdcReplication() { + return withCdcReplication("While reading Data"); + } + + public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) { + return this + .with("is_test", true) + .with("replication_method", Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .put("replication_slot", testDatabase.getReplicationSlotName()) + .put("publication", testDatabase.getPublicationName()) + .put("initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds()) + .put("lsn_commit_behaviour", LsnCommitBehaviour) + .build())); + } + + public PostgresConfigBuilder withXminReplication() { + return this.with("replication_method", Jsons.jsonNode(ImmutableMap.builder().put("method", "Xmin").build())); + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/AbstractPostgresTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/AbstractPostgresTypingDedupingTest.java new file mode 100644 index 000000000000..128d8d2de1cf --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/AbstractPostgresTypingDedupingTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcTypingDedupingTest; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.destination.postgres.PostgresSQLNameTransformer; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.junit.jupiter.api.Test; + +public abstract class AbstractPostgresTypingDedupingTest extends JdbcTypingDedupingTest { + + private static final int DEFAULT_VARCHAR_LIMIT_IN_JDBC_GEN = 65535; + + private static final Random RANDOM = new Random(); + + private String generateBigString() { + // Generate exactly 2 chars over the limit + final int length = DEFAULT_VARCHAR_LIMIT_IN_JDBC_GEN + 2; + return RANDOM + .ints('a', 'z' + 1) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + + @Override + protected SqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + + @Override + protected JdbcCompatibleSourceOperations getSourceOperations() { + return new PostgresSourceOperations(); + } + + @Test + public void testMixedCasedSchema() throws Exception { + streamName = "MixedCaseSchema" + streamName; + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + } + + @Override + protected List dumpRawTableRecords(String streamNamespace, String streamName) throws Exception { + return super.dumpRawTableRecords(streamNamespace, streamName.toLowerCase()); + } + + @Test + public void testVarcharLimitOver64K() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + final AirbyteMessage message = new AirbyteMessage(); + final String largeString = generateBigString(); + final Map data = ImmutableMap.of( + "id1", 1, + "id2", 200, + "updated_at", "2021-01-01T00:00:00Z", + "name", largeString); + message.setType(Type.RECORD); + message.setRecord(new AirbyteRecordMessage() + .withNamespace(streamNamespace) + .withStream(streamName) + .withData(Jsons.jsonNode(data)) + .withEmittedAt(1000L)); + final List messages1 = new ArrayList<>(); + messages1.add(message); + runSync(catalog, messages1); + + // Only assert on the large varchar string landing in final table. + // Rest of the fields' correctness is tested by other means in other tests. + final List actualFinalRecords = dumpFinalTableRecords(streamNamespace, streamName); + assertEquals(1, actualFinalRecords.size()); + assertEquals(largeString, actualFinalRecords.get(0).get("name").asText()); + + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSourceOperations.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSourceOperations.java new file mode 100644 index 000000000000..997aa66bf2a2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSourceOperations.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.commons.json.Jsons; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * See + * {@link io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGeneratorIntegrationTest.RedshiftSourceOperations}. + * Copied here to avoid weird dependencies. + */ +public class PostgresSourceOperations extends JdbcSourceOperations { + + @Override + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + final String columnTypeName = resultSet.getMetaData().getColumnTypeName(colIndex).toLowerCase(); + + switch (columnTypeName) { + // JSONB has no equivalent in JDBCType + case "jsonb" -> json.set(columnName, Jsons.deserializeExact(resultSet.getString(colIndex))); + // For some reason, the driver maps these to their timezoneless equivalents (TIME and TIMESTAMP) + case "timetz" -> putTimeWithTimezone(json, columnName, resultSet, colIndex); + case "timestamptz" -> putTimestampWithTimezone(json, columnName, resultSet, colIndex); + default -> super.copyToJsonField(resultSet, colIndex, json); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-qdrant/Dockerfile b/airbyte-integrations/connectors/destination-qdrant/Dockerfile index 9d87bb68b743..bbbdae6005c1 100644 --- a/airbyte-integrations/connectors/destination-qdrant/Dockerfile +++ b/airbyte-integrations/connectors/destination-qdrant/Dockerfile @@ -41,5 +41,5 @@ COPY destination_qdrant ./destination_qdrant ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.9 +LABEL io.airbyte.version=0.0.10 LABEL io.airbyte.name=airbyte/destination-qdrant diff --git a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml index 65467ebb587f..73c87125aef2 100644 --- a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml +++ b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml @@ -20,7 +20,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 6eb1198a-6d38-43e5-aaaa-dccd8f71db2b - dockerImageTag: 0.0.9 + dockerImageTag: 0.0.10 dockerRepository: airbyte/destination-qdrant githubIssueLabel: destination-qdrant icon: qdrant.svg diff --git a/airbyte-integrations/connectors/destination-qdrant/setup.py b/airbyte-integrations/connectors/destination-qdrant/setup.py index 127d413a5312..f30ca62213c4 100644 --- a/airbyte-integrations/connectors/destination-qdrant/setup.py +++ b/airbyte-integrations/connectors/destination-qdrant/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.55.1", "qdrant-client", "fastembed"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "qdrant-client", "fastembed"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 14b61c945692..298b24ec4012 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,12 +4,11 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.7.1' - features = ['db-destinations', 's3-destinations'] + cdkVersionRequired = '0.23.2' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later java { compileJava { options.compilerArgs.remove("-Werror") @@ -39,14 +38,4 @@ dependencies { testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' testImplementation "org.mockito:mockito-inline:4.1.0" - // TODO: declare typing-deduping as a CDK feature instead of importing from source. - implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) - integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) -} - -configurations.all { - resolutionStrategy { - force libs.jooq - } } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 537814340584..368369cfe5d2 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,22 +5,25 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.7.3 + dockerImageTag: 2.1.8 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift icon: redshift.svg license: MIT name: Redshift - normalizationConfig: - normalizationIntegrationType: redshift - normalizationRepository: airbyte/normalization-redshift - normalizationTag: 0.4.3 registries: cloud: enabled: true oss: enabled: true + releases: + breakingChanges: + 2.0.0: + message: > + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + upgradeDeadline: "2024-03-15" releaseStage: beta resourceRequirements: jobSpecific: diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java index d10cabd45730..5b48a24ca115 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java @@ -69,6 +69,11 @@ public ConnectorSpecification spec() throws Exception { return originalSpec; } + @Override + public boolean isV2Destination() { + return true; + } + public static void main(final String[] args) throws Exception { final Destination destination = new RedshiftDestination(); LOGGER.info("starting destination: {}", RedshiftDestination.class); diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 646b45299f7d..a4ba7a669557 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -22,6 +22,9 @@ import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; +import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import javax.sql.DataSource; @@ -54,7 +57,13 @@ public DataSource getDataSource(final JsonNode config) { jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, RedshiftInsertDestination.DRIVER_CLASS, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - SSL_JDBC_PARAMETERS); + getDefaultConnectionProperties(config), + Duration.ofMinutes(2)); + } + + @Override + protected void destinationSpecificTableOperations(final JdbcDatabase database) throws Exception { + RedshiftUtil.checkSvvTableAccess(database); } @Override @@ -68,7 +77,18 @@ public JdbcDatabase getDatabase(final DataSource dataSource, final JdbcSourceOpe @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return SSL_JDBC_PARAMETERS; + // The following properties can be overriden through jdbcUrlParameters in the config. + final Map connectionOptions = new HashMap<>(); + // Redshift properties + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-configuration-options.html#jdbc20-connecttimeout-option + // connectTimeout is different from Hikari pool's connectionTimout, driver defaults to 10seconds so + // increase it to match hikari's default + connectionOptions.put("connectTimeout", "120"); + // HikariPool properties + // https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#frequently-used + // TODO: Change data source factory to configure these properties + connectionOptions.putAll(SSL_JDBC_PARAMETERS); + return connectionOptions; } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index db8dfefcbc43..16189ce2004b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -42,7 +42,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; -import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; @@ -51,11 +50,14 @@ import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; import javax.sql.DataSource; @@ -101,7 +103,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { try { final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String outputSchema = super.getNamingResolver().getIdentifier(config.get(JdbcUtils.SCHEMA_KEY).asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, redshiftS3StagingSqlOperations); + attemptTableOperations(outputSchema, database, nameTransformer, redshiftS3StagingSqlOperations, false); + RedshiftUtil.checkSvvTableAccess(database); return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); } catch (final ConnectionErrorException e) { final String message = getErrorMessage(e.getStateCode(), e.getErrorCode(), e.getExceptionMessage(), e); @@ -131,7 +134,8 @@ public DataSource getDataSource(final JsonNode config) { jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, RedshiftInsertDestination.DRIVER_CLASS, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - SSL_JDBC_PARAMETERS); + getDefaultConnectionProperties(config), + Duration.ofMinutes(2)); } @Override @@ -141,7 +145,23 @@ protected NamingConventionTransformer getNamingResolver() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return SSL_JDBC_PARAMETERS; + // TODO: Pull common code from RedshiftInsertDestination and RedshiftStagingS3Destination into a + // base class. + // The following properties can be overriden through jdbcUrlParameters in the config. + final Map connectionOptions = new HashMap<>(); + // Redshift properties + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-configuration-options.html#jdbc20-connecttimeout-option + // connectTimeout is different from Hikari pool's connectionTimout, driver defaults to 10seconds so + // increase it to match hikari's default + connectionOptions.put("connectTimeout", "120"); + // HikariPool properties + // https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#frequently-used + // connectionTimeout is set explicitly to 2 minutes when creating data source. + // Do aggressive keepAlive with minimum allowed value, this only applies to connection sitting idle + // in the pool. + connectionOptions.put("keepaliveTime", Long.toString(Duration.ofSeconds(30).toMillis())); + connectionOptions.putAll(SSL_JDBC_PARAMETERS); + return connectionOptions; } // this is a no op since we override getDatabase. @@ -156,7 +176,7 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(String databaseName, JdbcDatabase database) { + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { return new RedshiftDestinationHandler(databaseName, database); } @@ -185,25 +205,6 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN of streams {} this will create more buffers than necessary, leading to nonexistent gains """, FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER, catalog.getStreams().size()); } - // Short circuit old way of running things during transition. - if (!TypingAndDedupingFlag.isDestinationV2()) { - return new StagingConsumerFactory().createAsync( - outputRecordCollector, - getDatabase(getDataSource(config)), - new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), - getNamingResolver(), - config, - catalog, - isPurgeStagingData(s3Options), - new TypeAndDedupeOperationValve(), - new NoopTyperDeduper(), - // The parsedcatalog is only used in v2 mode, so just pass null for now - null, - // Overwriting null namespace with null is perfectly safe - null, - // still using v1 table format - false); - } final String defaultNamespace = config.get("schema").asText(); for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { @@ -226,16 +227,14 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN parsedCatalog = catalogParser.parseCatalog(catalog); final JdbcV1V2Migrator migrator = new JdbcV1V2Migrator(getNamingResolver(), database, databaseName); final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); - boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); - final int defaultThreadCount = 8; + final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, - defaultThreadCount); + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator); } else { typerDeduper = - new DefaultTyperDeduper<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, defaultThreadCount); + new DefaultTyperDeduper(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator); } - return new StagingConsumerFactory().createAsync( + return StagingConsumerFactory.builder( outputRecordCollector, database, new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), @@ -247,7 +246,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN typerDeduper, parsedCatalog, defaultNamespace, - true); + true).build().createAsync(); } /** diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java deleted file mode 100644 index 3a8f801c4689..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift.copiers; - -import com.amazonaws.services.s3.AmazonS3; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3StreamCopier; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.destination.redshift.manifest.Entry; -import io.airbyte.integrations.destination.redshift.manifest.Manifest; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RedshiftStreamCopier extends S3StreamCopier { - - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftStreamCopier.class); - // From https://docs.aws.amazon.com/redshift/latest/dg/t_loading-tables-from-s3.html - // "Split your load data files so that the files are about equal size, between 1 MB and 1 GB after - // compression" - public static final int MAX_PARTS_PER_FILE = 4; - - private final ObjectMapper objectMapper; - private String manifestFilePath = null; - - public RedshiftStreamCopier(final String stagingFolder, - final String schema, - final AmazonS3 client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final ConfiguredAirbyteStream configuredAirbyteStream) { - this( - stagingFolder, - schema, - client, - db, - config, - nameTransformer, - sqlOperations, - Timestamp.from(Instant.now()), - configuredAirbyteStream); - } - - @VisibleForTesting - RedshiftStreamCopier(final String stagingFolder, - final String schema, - final AmazonS3 client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final Timestamp uploadTime, - final ConfiguredAirbyteStream configuredAirbyteStream) { - super(stagingFolder, - schema, - client, - db, - config, - nameTransformer, - sqlOperations, - configuredAirbyteStream, - uploadTime, - MAX_PARTS_PER_FILE); - objectMapper = new ObjectMapper(); - } - - @Override - public void copyStagingFileToTemporaryTable() { - final var possibleManifest = Optional.ofNullable(createManifest()); - LOGGER.info("Starting copy to tmp table: {} in destination for stream: {}, schema: {}, .", tmpTableName, streamName, schemaName); - possibleManifest.stream() - .map(this::putManifest) - .forEach(this::executeCopy); - LOGGER.info("Copy to tmp table {} in destination for stream {} complete.", tmpTableName, streamName); - } - - @Override - public void copyS3CsvFileIntoTable(final JdbcDatabase database, - final String s3FileLocation, - final String schema, - final String tableName, - final S3DestinationConfig s3Config) { - throw new RuntimeException("Redshift Stream Copier should not copy individual files without use of a manifest"); - } - - @Override - public void removeFileAndDropTmpTable() throws Exception { - super.removeFileAndDropTmpTable(); - if (manifestFilePath != null) { - LOGGER.info("Begin cleaning s3 manifest file {}.", manifestFilePath); - if (s3Client.doesObjectExist(s3Config.getBucketName(), manifestFilePath)) { - s3Client.deleteObject(s3Config.getBucketName(), manifestFilePath); - } - LOGGER.info("S3 manifest file {} cleaned.", manifestFilePath); - } - } - - /** - * Creates the contents of a manifest file given the `s3StagingFiles`. There must be at least one - * entry in a manifest file otherwise it is not considered valid for the COPY command. - * - * @return null if no stagingFiles exist otherwise the manifest body String - */ - private String createManifest() { - if (getStagingFiles().isEmpty()) { - return null; - } - - final var s3FileEntries = getStagingFiles().stream() - .map(filePath -> new Entry(getFullS3Path(s3Config.getBucketName(), filePath))) - .collect(Collectors.toList()); - final var manifest = new Manifest(s3FileEntries); - - return Exceptions.toRuntime(() -> objectMapper.writeValueAsString(manifest)); - } - - /** - * Upload the supplied manifest file to S3 - * - * @param manifestContents the manifest contents, never null - * @return the path where the manifest file was placed in S3 - */ - private String putManifest(final String manifestContents) { - manifestFilePath = - String.join("/", s3Config.getBucketPath(), stagingFolder, schemaName, String.format("%s.manifest", UUID.randomUUID())); - - s3Client.putObject(s3Config.getBucketName(), manifestFilePath, manifestContents); - - return manifestFilePath; - } - - /** - * Run Redshift COPY command with the given manifest file - * - * @param manifestPath the path in S3 to the manifest file - */ - private void executeCopy(final String manifestPath) { - final S3AccessKeyCredentialConfig credentialConfig = (S3AccessKeyCredentialConfig) s3Config.getS3CredentialConfig(); - final var copyQuery = String.format( - "COPY %s.%s FROM '%s'\n" - + "CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s'\n" - + "CSV REGION '%s' TIMEFORMAT 'auto'\n" - + "STATUPDATE OFF\n" - + "MANIFEST;", - schemaName, - tmpTableName, - getFullS3Path(s3Config.getBucketName(), manifestPath), - credentialConfig.getAccessKeyId(), - credentialConfig.getSecretAccessKey(), - s3Config.getBucketRegion()); - - Exceptions.toRuntime(() -> db.execute(copyQuery)); - } - -} diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java deleted file mode 100644 index 5527002288bc..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift.copiers; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3StreamCopierFactory; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; - -/** - * Very similar to the {@link S3StreamCopierFactory}, but we need some additional - */ -public class RedshiftStreamCopierFactory extends S3StreamCopierFactory { - - @Override - public StreamCopier create(final String stagingFolder, - final String schema, - final AmazonS3 s3Client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final ConfiguredAirbyteStream configuredStream) { - return new RedshiftStreamCopier(stagingFolder, schema, s3Client, db, config, nameTransformer, sqlOperations, configuredStream); - } - -} diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index b3adb95f8b35..c2b4da5c97ef 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -19,13 +19,15 @@ import io.airbyte.commons.lang.Exceptions; import io.airbyte.integrations.destination.redshift.manifest.Entry; import io.airbyte.integrations.destination.redshift.manifest.Manifest; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Base64; import java.util.Base64.Encoder; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -import org.joda.time.DateTime; public class RedshiftS3StagingSqlOperations extends RedshiftSqlOperations implements StagingOperations { @@ -44,7 +46,7 @@ public RedshiftS3StagingSqlOperations(final NamingConventionTransformer nameTran this.s3StorageOperations = new S3StorageOperations(nameTransformer, s3Client, s3Config); this.s3Config = s3Config; this.objectMapper = new ObjectMapper(); - if (encryptionConfig instanceof AesCbcEnvelopeEncryption e) { + if (encryptionConfig instanceof final AesCbcEnvelopeEncryption e) { this.s3StorageOperations.addBlobDecorator(new AesCbcEnvelopeEncryptionBlobDecorator(e.key())); this.keyEncryptingKey = e.key(); } else { @@ -52,41 +54,32 @@ public RedshiftS3StagingSqlOperations(final NamingConventionTransformer nameTran } } - /** - * I suspect this value is ignored. The stage name is eventually passed into - * {@link io.airbyte.cdk.integrations.destination.s3.S3StorageOperations#uploadRecordsToBucket(SerializableBuffer, String, String, String)} - * as the streamName parameter... which is completely ignored. - * - */ - @Override - public String getStageName(final String namespace, final String streamName) { - return nameTransformer.applyDefaultCase(String.join("_", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); - } - @Override public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final String outputTableName, - final DateTime writeDatetime) { + final Instant writeDatetime) { final String bucketPath = s3Config.getBucketPath(); final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); + final ZonedDateTime zdt = writeDatetime.atZone(ZoneOffset.UTC); return nameTransformer.applyDefaultCase(String.format("%s%s/%s_%02d_%02d_%02d_%s/", prefix, nameTransformer.applyDefaultCase(nameTransformer.convertStreamName(outputTableName)), - writeDatetime.year().get(), - writeDatetime.monthOfYear().get(), - writeDatetime.dayOfMonth().get(), - writeDatetime.hourOfDay().get(), + zdt.getYear(), + zdt.getMonthValue(), + zdt.getDayOfMonth(), + zdt.getHour(), connectionId)); } + @Override + public String getStageName(final String namespace, final String streamName) { + return "garbage-unused"; + } + @Override public void createStageIfNotExists(final JdbcDatabase database, final String stageName) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); s3StorageOperations.createBucketIfNotExists(); } @@ -97,7 +90,7 @@ public String uploadRecordsToStage(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception { - return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stageName, stagingPath); + return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stagingPath); } private String putManifest(final String manifestContents, final String stagingPath) { @@ -177,17 +170,9 @@ private static String getManifestPath(final String s3BucketName, final String s3 } @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.cleanUpBucketObject(prefix + stageName, stagedFiles); - } - - @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.dropBucketObject(prefix + stageName); + public void dropStageIfExists(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception { + // stageName is unused here but used in Snowflake. This interface needs to be fixed. + s3StorageOperations.dropBucketObject(stagingPath); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index 7b85a0d92706..4c8927098ea8 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -31,6 +31,8 @@ import org.jooq.InsertValuesStep4; import org.jooq.Record; import org.jooq.SQLDialect; +import org.jooq.conf.Settings; +import org.jooq.conf.StatementType; import org.jooq.impl.DefaultDataType; import org.jooq.impl.SQLDataType; import org.slf4j.Logger; @@ -111,7 +113,19 @@ protected void insertRecordsInternalV2(final JdbcDatabase database, // > default for (final List batch : Iterables.partition(records, 10_000)) { LOGGER.info("Prepared batch size: {}, {}, {}", batch.size(), schemaName, tableName); - final DSLContext create = using(connection, SQLDialect.POSTGRES); + final DSLContext create = using( + connection, + SQLDialect.POSTGRES, + // Force inlined params. + // jooq normally tries to intelligently use bind params when possible. + // This would cause queries with many params to use inline params, + // but small queries would use bind params. + // In turn, that would force us to intelligently escape string values, + // since we need to escape inlined strings + // but need to not escape bound strings. + // Instead, we force jooq to always inline params, + // and always call escapeStringLiteral() on the string values. + new Settings().withStatementType(StatementType.STATIC_STATEMENT)); // JOOQ adds some overhead here. Building the InsertValuesStep object takes about 139ms for 5K // records. // That's a nontrivial execution speed loss when the actual statement execution takes 500ms. @@ -133,7 +147,7 @@ protected void insertRecordsInternalV2(final JdbcDatabase database, for (final PartialAirbyteMessage record : batch) { insert = insert.values( val(UUID.randomUUID().toString()), - function("JSON_PARSE", String.class, val(record.getSerialized())), + function("JSON_PARSE", String.class, val(escapeStringLiteral(record.getSerialized()))), val(Instant.ofEpochMilli(record.getRecord().getEmittedAt()).atOffset(ZoneOffset.UTC)), val((OffsetDateTime) null)); } @@ -147,4 +161,15 @@ protected void insertRecordsInternalV2(final JdbcDatabase database, } } + public static String escapeStringLiteral(final String str) { + if (str == null) { + return null; + } else { + // jooq handles most things + // but we need to manually escape backslashes because postgres and redshift have + // different backslash handling. + return str.replace("\\", "\\\\"); + } + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java index 05201f082934..5a47c2436d00 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -4,94 +4,87 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; -import com.fasterxml.jackson.databind.JsonNode; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.*; + import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; -import io.airbyte.integrations.base.destination.typing_deduping.StreamId; -import java.sql.ResultSet; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import org.jooq.impl.DSL; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class RedshiftDestinationHandler extends JdbcDestinationHandler { - // Redshift doesn't seem to let you actually specify HH:MM TZ offsets, so we have - // build our own formatter rather than just use Instant.parse - private static final DateTimeFormatter TIMESTAMPTZ_FORMAT = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ISO_LOCAL_DATE) - .appendLiteral(' ') - .append(DateTimeFormatter.ISO_LOCAL_TIME) - .append(DateTimeFormatter.ofPattern("X")) - .toFormatter(); - public RedshiftDestinationHandler(final String databaseName, final JdbcDatabase jdbcDatabase) { super(databaseName, jdbcDatabase); } @Override - public boolean isFinalTableEmpty(final StreamId id) throws Exception { - // Redshift doesn't have an information_schema.tables table, so we have to use SVV_TABLE_INFO. - // From https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html: - // > The SVV_TABLE_INFO view doesn't return any information for empty tables. - // So we just query for our specific table, and if we get no rows back, - // then we assume the table is empty. - // Note that because the column names are reserved words (table, schema, database), - // we need to enquote them. - final List query = jdbcDatabase.queryJsons( - """ - SELECT 1 - FROM SVV_TABLE_INFO - WHERE "database" = ? - AND "schema" = ? - AND "table" = ? - """, - databaseName, - id.finalNamespace(), - id.finalName()); - return query.isEmpty(); + public void execute(final Sql sql) throws Exception { + final List> transactions = sql.transactions(); + final UUID queryId = UUID.randomUUID(); + for (final List transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + log.info("Executing sql {}-{}: {}", queryId, transactionId, String.join("\n", transaction)); + final long startTime = System.currentTimeMillis(); + + try { + // Original list is immutable, so copying it into a different list. + final List modifiedStatements = new ArrayList<>(); + // This is required for Redshift to retrieve Json path query with upper case characters, even after + // specifying quotes. + // see https://github.com/airbytehq/airbyte/issues/33900 + modifiedStatements.add("SET enable_case_sensitive_identifier to TRUE;\n"); + modifiedStatements.addAll(transaction); + jdbcDatabase.executeWithinTransaction(modifiedStatements); + } catch (final SQLException e) { + log.error("Sql {}-{} failed", queryId, transactionId, e); + throw e; + } + + log.info("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); + } } @Override - public Optional getMinTimestampForSync(final StreamId id) throws Exception { - final ResultSet tables = jdbcDatabase.getMetaData().getTables( - databaseName, - id.rawNamespace(), - id.rawName(), - null); - if (!tables.next()) { - return Optional.empty(); - } - // Redshift timestamps have microsecond precision, but it's basically impossible to work with that. - // Decrement by 1 second instead. - // And use two explicit queries because docs don't specify whether COALESCE - // short-circuits evaluation. - // This first query tries to find the oldest raw record with loaded_at = NULL - Optional minUnloadedTimestamp = Optional.ofNullable(jdbcDatabase.queryStrings( - conn -> conn.createStatement().executeQuery( - DSL.select(DSL.field("MIN(_airbyte_extracted_at) - INTERVAL '1 second'").as("min_timestamp")) - .from(DSL.name(id.rawNamespace(), id.rawName())) - .where(DSL.condition("_airbyte_loaded_at IS NULL")) - .getSQL()), - // The query will always return exactly one record, so use .get(0) - record -> record.getString("min_timestamp")).get(0)); - if (minUnloadedTimestamp.isEmpty()) { - // If there are no unloaded raw records, then we can safely skip all existing raw records. - // This second query just finds the newest raw record. - minUnloadedTimestamp = Optional.ofNullable(jdbcDatabase.queryStrings( - conn -> conn.createStatement().executeQuery( - DSL.select(DSL.field("MAX(_airbyte_extracted_at)").as("min_timestamp")) - .from(DSL.name(id.rawNamespace(), id.rawName())) - .getSQL()), - record -> record.getString("min_timestamp")).get(0)); + protected String toJdbcTypeName(AirbyteType airbyteType) { + // This is mostly identical to the postgres implementation, but swaps jsonb to super + if (airbyteType instanceof final AirbyteProtocolType airbyteProtocolType) { + return toJdbcTypeName(airbyteProtocolType); } - return minUnloadedTimestamp.map(RedshiftDestinationHandler::parseInstant); + return switch (airbyteType.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE, Array.TYPE -> "super"; + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> toJdbcTypeName(((Union) airbyteType).chooseType()); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + airbyteType); + }; } - private static Instant parseInstant(final String ts) { - return TIMESTAMPTZ_FORMAT.parse(ts, Instant::from); + private String toJdbcTypeName(final AirbyteProtocolType airbyteProtocolType) { + return switch (airbyteProtocolType) { + case STRING -> "varchar"; + case NUMBER -> "numeric"; + case INTEGER -> "int8"; + case BOOLEAN -> "bool"; + case TIMESTAMP_WITH_TIMEZONE -> "timestamptz"; + case TIMESTAMP_WITHOUT_TIMEZONE -> "timestamp"; + case TIME_WITH_TIMEZONE -> "timetz"; + case TIME_WITHOUT_TIMEZONE -> "time"; + case DATE -> "date"; + case UNKNOWN -> "super"; + }; } + // Do not use SVV_TABLE_INFO to get isFinalTableEmpty. + // See https://github.com/airbytehq/airbyte/issues/34357 + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java index 710a217bdb4e..37f72c21c9f1 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java @@ -5,68 +5,36 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; -import static org.jooq.impl.DSL.alterTable; -import static org.jooq.impl.DSL.asterisk; import static org.jooq.impl.DSL.cast; -import static org.jooq.impl.DSL.createSchemaIfNotExists; -import static org.jooq.impl.DSL.dropTableIfExists; import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.function; -import static org.jooq.impl.DSL.inline; import static org.jooq.impl.DSL.name; -import static org.jooq.impl.DSL.noCondition; import static org.jooq.impl.DSL.quotedName; import static org.jooq.impl.DSL.rowNumber; -import static org.jooq.impl.DSL.select; -import static org.jooq.impl.DSL.table; -import static org.jooq.impl.DSL.update; import static org.jooq.impl.DSL.val; -import static org.jooq.impl.DSL.with; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; -import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; -import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; -import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.base.destination.typing_deduping.Struct; import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.time.Instant; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import org.jooq.CommonTableExpression; import org.jooq.Condition; -import org.jooq.CreateSchemaFinalStep; -import org.jooq.CreateTableColumnStep; -import org.jooq.DSLContext; import org.jooq.DataType; import org.jooq.Field; -import org.jooq.InsertValuesStepN; -import org.jooq.Name; -import org.jooq.Record; import org.jooq.SQLDialect; -import org.jooq.SelectConditionStep; -import org.jooq.conf.ParamType; -import org.jooq.impl.DSL; import org.jooq.impl.DefaultDataType; import org.jooq.impl.SQLDataType; @@ -74,17 +42,9 @@ public class RedshiftSqlGenerator extends JdbcSqlGenerator { public static final String CASE_STATEMENT_SQL_TEMPLATE = "CASE WHEN {0} THEN {1} ELSE {2} END "; public static final String CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE = "CASE WHEN {0} THEN {1} END "; - private static final Map REDSHIFT_TYPE_NAME_TO_JDBC_TYPE = ImmutableMap.of( - "numeric", "decimal", - "int8", "bigint", - "bool", "boolean", - "timestamptz", "timestamp with time zone", - "timetz", "time with time zone"); private static final String COLUMN_ERROR_MESSAGE_FORMAT = "Problem with `%s`"; private static final String AIRBYTE_META_COLUMN_ERRORS_KEY = "errors"; - private final ColumnId CDC_DELETED_AT_COLUMN = buildColumnId("_ab_cdc_deleted_at"); - public RedshiftSqlGenerator(final NamingConventionTransformer namingTransformer) { super(namingTransformer); } @@ -119,10 +79,6 @@ protected SQLDialect getDialect() { return SQLDialect.POSTGRES; } - protected DSLContext getDslContext() { - return DSL.using(getDialect()); - } - /** * Notes about Redshift specific SQL * 16MB Limit on the total size of the SQL sent in a session * * Default mode of casting within SUPER is lax mode, to enable strict use SET @@ -137,57 +93,18 @@ protected DSLContext getDslContext() { * KEYS, DISTKEY in redshift for optimizing the query performance. */ - /** - * build jooq fields for final table with customers columns first and then meta columns. - * - * @param columns - * @param metaColumns - * @return - */ - @VisibleForTesting - List> buildFinalTableFields(final LinkedHashMap columns, final Map> metaColumns) { - final List> fields = - metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(Collectors.toList()); - final List> dataFields = - columns.entrySet().stream().map(column -> field(quotedName(column.getKey().name()), toDialectType(column.getValue()))).collect( - Collectors.toList()); - dataFields.addAll(fields); - return dataFields; - } - - /** - * build jooq fields for raw table with type-casted data columns first and then meta columns without - * _airbyte_meta. - * - * @param columns - * @param metaColumns - * @return - */ - @VisibleForTesting - List> buildRawTableSelectFields(final LinkedHashMap columns, final Map> metaColumns) { - final List> fields = - metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(Collectors.toList()); - // Use originalName with non-sanitized characters when extracting data from _airbyte_data - final List> dataFields = columns - .entrySet() - .stream() - .map(column -> castedField(field(quotedName(COLUMN_NAME_DATA, column.getKey().originalName())), column.getValue(), column.getKey().name())) - .collect(Collectors.toList()); - dataFields.addAll(fields); - return dataFields; - } - - private Field castedField(final Field field, final AirbyteType type, final String alias) { + @Override + protected Field castedField(final Field field, final AirbyteType type, final String alias, final boolean useExpensiveSaferCasting) { if (type instanceof final AirbyteProtocolType airbyteProtocolType) { switch (airbyteProtocolType) { case STRING -> { return field(CASE_STATEMENT_SQL_TEMPLATE, jsonTypeOf(field).ne("string").and(field.isNotNull()), jsonSerialize(field), - castedField(field, airbyteProtocolType)).as(quotedName(alias)); + castedField(field, airbyteProtocolType, useExpensiveSaferCasting)).as(quotedName(alias)); } default -> { - return castedField(field, airbyteProtocolType).as(quotedName(alias)); + return castedField(field, airbyteProtocolType, useExpensiveSaferCasting).as(quotedName(alias)); } } @@ -201,13 +118,22 @@ private Field castedField(final Field field, final AirbyteType type, final jsonTypeOf(field).eq("array"), cast(field, getArrayType())).as(quotedName(alias)); // No nested Unions supported so this will definitely not result in infinite recursion. - case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias); + case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias, useExpensiveSaferCasting); default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type); }; } - private Field castedField(final Field field, final AirbyteProtocolType type) { - return cast(field, toDialectType(type)); + @Override + protected List> extractRawDataFields(final LinkedHashMap columns, final boolean useExpensiveSaferCasting) { + return columns + .entrySet() + .stream() + .map(column -> castedField( + field(quotedName(COLUMN_NAME_DATA, column.getKey().originalName())), + column.getValue(), + column.getKey().name(), + useExpensiveSaferCasting)) + .collect(Collectors.toList()); } private Field jsonTypeOf(final Field field) { @@ -219,7 +145,8 @@ private Field jsonSerialize(final Field field) { } /** - * Redshift ARRAY_CONCAT supports only 2 arrays, recursively build ARRAY_CONCAT for n arrays. + * Redshift ARRAY_CONCAT supports only 2 arrays. Iteratively nest ARRAY_CONCAT to support more than + * 2 * * @param arrays * @return @@ -229,16 +156,13 @@ Field arrayConcatStmt(final List> arrays) { return field("ARRAY()"); // Return an empty string if the list is empty } - // Base case: if there's only one element, return it - if (arrays.size() == 1) { - return arrays.get(0); + Field result = arrays.get(0); + for (int i = 1; i < arrays.size(); i++) { + // We lose some nice indentation but thats ok. Queryparts + // are intentionally rendered here to avoid deep stack for function sql rendering. + result = field(getDslContext().renderNamedOrInlinedParams(function("ARRAY_CONCAT", getSuperType(), result, arrays.get(i)))); } - - // Recursive case: construct ARRAY_CONCAT function call - final Field lastValue = arrays.get(arrays.size() - 1); - final Field recursiveCall = arrayConcatStmt(arrays.subList(0, arrays.size() - 1)); - - return function("ARRAY_CONCAT", getSuperType(), recursiveCall, lastValue); + return result; } Field toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) { @@ -248,11 +172,12 @@ Field toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) { // TODO: Timestamp format issues can result in null values when cast, add regex check if destination // supports regex functions. return field(CASE_STATEMENT_SQL_TEMPLATE, - field.isNotNull().and(castedField(field, type, column.name()).isNull()), + field.isNotNull().and(castedField(field, type, column.name(), true).isNull()), function("ARRAY", getSuperType(), val(COLUMN_ERROR_MESSAGE_FORMAT.formatted(column.name()))), field("ARRAY()")); } - Field buildAirbyteMetaColumn(final LinkedHashMap columns) { + @Override + protected Field buildAirbyteMetaColumn(final LinkedHashMap columns) { final List> dataFields = columns .entrySet() .stream() @@ -262,159 +187,6 @@ Field buildAirbyteMetaColumn(final LinkedHashMap colum } - /** - * Use this method to get the final table meta columns with or without _airbyte_meta column. - * - * @param includeMetaColumn - * @return - */ - LinkedHashMap> getFinalTableMetaColumns(final boolean includeMetaColumn) { - final LinkedHashMap> metaColumns = new LinkedHashMap<>(); - metaColumns.put(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)); - metaColumns.put(COLUMN_NAME_AB_EXTRACTED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); - if (includeMetaColumn) - metaColumns.put(COLUMN_NAME_AB_META, getSuperType().nullable(false)); - return metaColumns; - } - - @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { - final DSLContext dsl = getDslContext(); - final CreateSchemaFinalStep createSchemaSql = createSchemaIfNotExists(quotedName(stream.id().finalNamespace())); - - // TODO: Use Naming transformer to sanitize these strings with redshift restrictions. - final String finalTableIdentifier = stream.id().finalName() + suffix.toLowerCase(); - final CreateTableColumnStep createTableSql = dsl - .createTable(quotedName(stream.id().finalNamespace(), finalTableIdentifier)) - .columns(buildFinalTableFields(stream.columns(), getFinalTableMetaColumns(true))); - if (!force) { - return Strings.join( - List.of( - createSchemaSql.getSQL() + ";", - // Redshift doesn't care about primary key but we can use SORTKEY for performance, its a table - // attribute not supported by jooq. - createTableSql.getSQL() + System.lineSeparator() + " SORTKEY(\"" + COLUMN_NAME_AB_EXTRACTED_AT + "\");"), - System.lineSeparator()); - } - return Strings.join( - List.of( - createSchemaSql.getSQL() + ";", - "BEGIN;", - dropTableIfExists(quotedName(stream.id().finalNamespace(), finalTableIdentifier)) + ";", - createTableSql.getSQL() + System.lineSeparator() + " SORTKEY(\"" + COLUMN_NAME_AB_EXTRACTED_AT + "\");", - "COMMIT;"), - System.lineSeparator()); - } - - @Override - public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final TableDefinition existingTable) { - // Check that the columns match, with special handling for the metadata columns. - final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() - .collect(LinkedHashMap::new, - (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue()).getTypeName()), - LinkedHashMap::putAll); - final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() - .filter(column -> JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream() - .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) - .collect(LinkedHashMap::new, - (map, column) -> map.put(column.getKey(), jdbcTypeNameFromRedshiftTypeName(column.getValue().type())), - LinkedHashMap::putAll); - - final boolean sameColumns = actualColumns.equals(intendedColumns) - && "varchar".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).type()) - && "timestamptz".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).type()) - && "super".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META).type()); - - return sameColumns; - } - - @Override - public String updateTable(final StreamConfig streamConfig, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { - - // TODO: Add flag to use merge vs insert/delete - return insertAndDeleteTransaction(streamConfig, finalSuffix, minRawTimestamp, useExpensiveSaferCasting); - - } - - private String insertAndDeleteTransaction(final StreamConfig streamConfig, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { - final String finalSchema = streamConfig.id().finalNamespace(); - final String finalTable = streamConfig.id().finalName() + (finalSuffix != null ? finalSuffix.toLowerCase() : ""); - final String rawSchema = streamConfig.id().rawNamespace(); - final String rawTable = streamConfig.id().rawName(); - - // Poor person's guarantee of ordering of fields by using same source of ordered list of columns to - // generate fields. - final CommonTableExpression rawTableRowsWithCast = name("intermediate_data").as( - selectFromRawTable(rawSchema, rawTable, streamConfig.columns(), - getFinalTableMetaColumns(false), - rawTableCondition(streamConfig.destinationSyncMode(), - streamConfig.columns().containsKey(CDC_DELETED_AT_COLUMN), - minRawTimestamp))); - final List> finalTableFields = buildFinalTableFields(streamConfig.columns(), getFinalTableMetaColumns(true)); - final Field rowNumber = getRowNumber(streamConfig.primaryKey(), streamConfig.cursor()); - final CommonTableExpression filteredRows = name("numbered_rows").as( - select(asterisk(), rowNumber).from(rawTableRowsWithCast)); - - // Used for append-dedupe mode. - final String insertStmtWithDedupe = - insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) - .select(with(rawTableRowsWithCast) - .with(filteredRows) - .select(finalTableFields) - .from(filteredRows) - .where(field("row_number", Integer.class).eq(1)) // Can refer by CTE.field but no use since we don't strongly type them. - ) - .getSQL(ParamType.INLINED); - - // Used for append and overwrite modes. - final String insertStmt = - insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) - .select(with(rawTableRowsWithCast) - .select(finalTableFields) - .from(rawTableRowsWithCast)) - .getSQL(ParamType.INLINED); - final String deleteStmt = deleteFromFinalTable(finalSchema, finalTable, streamConfig.primaryKey(), streamConfig.cursor()); - final String deleteCdcDeletesStmt = - streamConfig.columns().containsKey(CDC_DELETED_AT_COLUMN) ? deleteFromFinalTableCdcDeletes(finalSchema, finalTable) : ""; - final String checkpointStmt = checkpointRawTable(rawSchema, rawTable, minRawTimestamp); - - if (streamConfig.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { - return Strings.join( - List.of( - "BEGIN", - insertStmt, - checkpointStmt, - "COMMIT;"), - ";" + System.lineSeparator()); - } - - // For append-dedupe - return Strings.join( - List.of( - "BEGIN", - insertStmtWithDedupe, - deleteStmt, - deleteCdcDeletesStmt, - checkpointStmt, - "COMMIT;"), - ";" + System.lineSeparator()); - } - - private String mergeTransaction(final StreamConfig streamConfig, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { - - throw new UnsupportedOperationException("Not implemented yet"); - - } - /** * Return ROW_NUMBER() OVER (PARTITION BY primaryKeys ORDER BY cursor DESC NULLS LAST, * _airbyte_extracted_at DESC) @@ -423,7 +195,10 @@ private String mergeTransaction(final StreamConfig streamConfig, * @param cursor * @return */ - Field getRowNumber(final List primaryKeys, final Optional cursor) { + @Override + protected Field getRowNumber(final List primaryKeys, final Optional cursor) { + // literally identical to postgres's getRowNumber implementation, changes here probably should + // be reflected there final List> primaryKeyFields = primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.name()))).collect(Collectors.toList()) : new ArrayList<>(); @@ -435,120 +210,19 @@ Field getRowNumber(final List primaryKeys, final Optional selectFromRawTable(final String schemaName, - final String tableName, - final LinkedHashMap columns, - final Map> metaColumns, - final Condition condition) { - final DSLContext dsl = getDslContext(); - return dsl - .select(buildRawTableSelectFields(columns, metaColumns)) - .select(buildAirbyteMetaColumn(columns)) - .from(table(quotedName(schemaName, tableName))) - .where(condition); - } - - Condition rawTableCondition(final DestinationSyncMode syncMode, final boolean isCdcDeletedAtPresent, final Optional minRawTimestamp) { - Condition condition = field(name(COLUMN_NAME_AB_LOADED_AT)).isNull(); - if (syncMode == DestinationSyncMode.APPEND_DEDUP) { - if (isCdcDeletedAtPresent) { - condition = condition.or(field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull() - .and(function("JSON_TYPEOF", SQLDataType.VARCHAR, field(quotedName(COLUMN_NAME_DATA, CDC_DELETED_AT_COLUMN.name()))) - .ne("null"))); - } - } - if (minRawTimestamp.isPresent()) { - condition = condition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); - } - return condition; - } - - @VisibleForTesting - InsertValuesStepN insertIntoFinalTable(final String schemaName, - final String tableName, - final LinkedHashMap columns, - final Map> metaFields) { - final DSLContext dsl = getDslContext(); - return dsl - .insertInto(table(quotedName(schemaName, tableName))) - .columns(buildFinalTableFields(columns, metaFields)); - } - - String deleteFromFinalTable(final String schemaName, final String tableName, final List primaryKeys, final Optional cursor) { - final DSLContext dsl = getDslContext(); - // Unknown type doesn't play well with where .. in (select..) - final Field airbyteRawId = field(quotedName(COLUMN_NAME_AB_RAW_ID)); - final Field rowNumber = getRowNumber(primaryKeys, cursor); - return dsl.deleteFrom(table(quotedName(schemaName, tableName))) - .where(airbyteRawId.in( - select(airbyteRawId) - .from(select(airbyteRawId, rowNumber) - .from(table(quotedName(schemaName, tableName))).asTable("airbyte_ids")) - .where(field("row_number").ne(1)))) - .getSQL(ParamType.INLINED); - } - - String deleteFromFinalTableCdcDeletes(final String schema, final String tableName) { - final DSLContext dsl = getDslContext(); - return dsl.deleteFrom(table(quotedName(schema, tableName))) - .where(field(quotedName(CDC_DELETED_AT_COLUMN.name())).isNotNull()) - .getSQL(ParamType.INLINED); - } - - String checkpointRawTable(final String schemaName, final String tableName, final Optional minRawTimestamp) { - final DSLContext dsl = getDslContext(); - Condition extractedAtCondition = noCondition(); - if (minRawTimestamp.isPresent()) { - extractedAtCondition = extractedAtCondition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); - } - return dsl.update(table(quotedName(schemaName, tableName))) - .set(field(quotedName(COLUMN_NAME_AB_LOADED_AT), SQLDataType.TIMESTAMPWITHTIMEZONE), - function("GETDATE", SQLDataType.TIMESTAMPWITHTIMEZONE)) - .where(field(quotedName(COLUMN_NAME_AB_LOADED_AT)).isNull()).and(extractedAtCondition) - .getSQL(ParamType.INLINED); + .orderBy(orderedFields).as(ROW_NUMBER_COLUMN_NAME); } @Override - public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return Strings.join( - List.of( - dropTableIfExists(name(stream.finalNamespace(), stream.finalName())), - alterTable(name(stream.finalNamespace(), stream.finalName() + finalSuffix)) - .renameTo(name(stream.finalName())) - .getSQL()), - ";" + System.lineSeparator()); + protected Condition cdcDeletedAtNotNullCondition() { + return field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull() + .and(function("JSON_TYPEOF", SQLDataType.VARCHAR, field(quotedName(COLUMN_NAME_DATA, cdcDeletedAtColumn.name()))) + .ne("null")); } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - final Name rawTableName = name(streamId.rawNamespace(), streamId.rawName()); - return Strings.join( - List.of( - createSchemaIfNotExists(streamId.rawNamespace()).getSQL(), - dropTableIfExists(rawTableName).getSQL(), - DSL.createTable(rawTableName) - .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) - .column(COLUMN_NAME_AB_EXTRACTED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)) - .column(COLUMN_NAME_AB_LOADED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)) - .column(COLUMN_NAME_DATA, getSuperType().nullable(false)) - .as(select( - field(COLUMN_NAME_AB_ID).as(COLUMN_NAME_AB_RAW_ID), - field(COLUMN_NAME_EMITTED_AT).as(COLUMN_NAME_AB_EXTRACTED_AT), - cast(null, SQLDataType.TIMESTAMPWITHTIMEZONE).as(COLUMN_NAME_AB_LOADED_AT), - field(COLUMN_NAME_DATA).as(COLUMN_NAME_DATA)).from(table(name(namespace, tableName)))) - .getSQL(ParamType.INLINED)), - ";" + System.lineSeparator()); - } - - @Override - public String clearLoadedAt(final StreamId streamId) { - return update(table(name(streamId.rawNamespace(), streamId.rawName()))) - .set(field(COLUMN_NAME_AB_LOADED_AT), inline((String) null)) - .getSQL(); + protected Field currentTimestamp() { + return function("GETDATE", SQLDataType.TIMESTAMP); } @Override @@ -556,8 +230,4 @@ public boolean shouldRetry(final Exception e) { return false; } - private static String jdbcTypeNameFromRedshiftTypeName(final String redshiftType) { - return REDSHIFT_TYPE_NAME_TO_JDBC_TYPE.getOrDefault(redshiftType, redshiftType); - } - } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java index c1433c4aa226..6551820a4831 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java @@ -7,10 +7,13 @@ import static io.airbyte.integrations.destination.redshift.constants.RedshiftDestinationConstants.UPLOADING_METHOD; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import lombok.extern.log4j.Log4j2; /** * Helper class for Destination Redshift connector. */ +@Log4j2 public class RedshiftUtil { private RedshiftUtil() {} @@ -36,4 +39,9 @@ private static boolean isNullOrEmpty(final JsonNode jsonNode) { return null == jsonNode || "".equals(jsonNode.asText()); } + public static void checkSvvTableAccess(final JdbcDatabase database) throws Exception { + log.info("checking SVV_TABLE_INFO permissions"); + database.queryJsons("SELECT 1 FROM SVV_TABLE_INFO LIMIT 1;"); + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index dfd233f4e33c..55cb60c52a62 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -111,29 +111,40 @@ "description": "The region of the S3 staging bucket.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", + "il-central-1", + "me-central-1", + "me-south-1", + "sa-east-1", "sa-east-1", - "me-south-1" + "us-east-1", + "us-east-2", + "us-gov-east-1", + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 2 }, @@ -238,17 +249,28 @@ } ] }, - "use_1s1t_format": { - "type": "boolean", - "description": "(Early Access) Use Destinations V2.", - "title": "Use Destinations V2 (Early Access)", - "order": 9 - }, "raw_data_schema": { "type": "string", - "description": "(Early Access) The schema to write raw tables into", - "title": "Destinations V2 Raw Table Schema (Early Access)", - "order": 10 + "description": "The schema to write raw tables into", + "title": "Destinations V2 Raw Table Schema", + "order": 9, + "group": "connection" + }, + "enable_incremental_final_table_updates": { + "type": "boolean", + "default": false, + "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", + "title": "Enable Loading Data Incrementally to Final Tables", + "order": 10, + "group": "connection" + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", + "order": 11, + "group": "connection" } }, "groups": [ diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java new file mode 100644 index 000000000000..dfefbf0c0f10 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +public class RedshiftConnectionTest { + + private final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + private final RedshiftDestination destination = new RedshiftDestination(); + private AirbyteConnectionStatus status; + + @Test + void testCheckIncorrectPasswordFailure() throws Exception { + ((ObjectNode) config).put("password", "fake"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000;")); + } + + @Test + public void testCheckIncorrectUsernameFailure() throws Exception { + ((ObjectNode) config).put("username", ""); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000;")); + } + + @Test + public void testCheckIncorrectHostFailure() throws Exception { + ((ObjectNode) config).put("host", "localhost2"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001;")); + } + + @Test + public void testCheckIncorrectDataBaseFailure() throws Exception { + ((ObjectNode) config).put("database", "wrongdatabase"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 3D000;")); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java new file mode 100644 index 000000000000..de31216ed222 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import com.amazon.redshift.util.RedshiftTimestamp; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.ConnectionFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.TestingNamespaces; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// these tests are not yet thread-safe, unlike the DV2 tests. +@Execution(ExecutionMode.SAME_THREAD) +public abstract class RedshiftDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftDestinationAcceptanceTest.class); + + // config from which to create / delete schemas. + private JsonNode baseConfig; + // config which refers to the schema that the test is being run in. + protected JsonNode config; + private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); + private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); + + private Database database; + private Connection connection; + protected TestDestinationEnv testDestinationEnv; + + @Override + protected String getImageName() { + return "airbyte/destination-redshift:dev"; + } + + @Override + protected JsonNode getConfig() { + return config; + } + + public abstract JsonNode getStaticConfig() throws IOException; + + @Override + protected JsonNode getFailCheckConfig() { + final JsonNode invalidConfig = Jsons.clone(config); + ((ObjectNode) invalidConfig).put("password", "wrong password"); + return invalidConfig; + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new RedshiftTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + + @Override + protected boolean supportIncrementalSchemaChanges() { + return true; + } + + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + + @Override + protected List retrieveRecords(final TestDestinationEnv env, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + .stream() + .map(j -> j.get(JavaBaseConstants.COLUMN_NAME_DATA)) + .collect(Collectors.toList()); + } + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) + throws Exception { + String tableName = namingResolver.getIdentifier(streamName); + if (!tableName.startsWith("\"")) { + // Currently, Normalization always quote tables identifiers + tableName = "\"" + tableName + "\""; + } + return retrieveRecordsFromTable(tableName, namespace); + } + + private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { + return getDatabase().query( + ctx -> ctx + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + .stream() + .map(record -> getJsonFromRecord( + record, + value -> { + if (value instanceof final RedshiftTimestamp rts) { + // We can't just use rts.toInstant().toString(), because that will mangle historical + // dates (e.g. 1504-02-28...) because toInstant() just converts to epoch millis, + // which works _very badly_ for for very old dates. + // Instead, convert to a string and then parse that string. + // We can't just rts.toString(), because that loses the timezone... + // so instead we use getPostgresqlString and parse that >.> + // Thanks, redshift. + return Optional.of( + ZonedDateTime.parse( + rts.getPostgresqlString(), + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 9, true) + .optionalEnd() + .appendPattern("X") + .toFormatter()) + .withZoneSameInstant(ZoneOffset.UTC) + .toString()); + } else { + return Optional.empty(); + } + })) + .collect(Collectors.toList())); + } + + // for each test we create a new schema in the database. run the test in there and then remove it. + @Override + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { + final String schemaName = TestingNamespaces.generate(); + final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); + baseConfig = getStaticConfig(); + database = createDatabase(); + removeOldNamespaces(); + getDatabase().query(ctx -> ctx.execute(createSchemaQuery)); + final String createUser = String.format("create user %s with password '%s' SESSION TIMEOUT 60;", + USER_WITHOUT_CREDS, baseConfig.get("password").asText()); + getDatabase().query(ctx -> ctx.execute(createUser)); + final JsonNode configForSchema = Jsons.clone(baseConfig); + ((ObjectNode) configForSchema).put("schema", schemaName); + TEST_SCHEMAS.add(schemaName); + config = configForSchema; + testDestinationEnv = testEnv; + } + + private void removeOldNamespaces() { + final List schemas; + try { + schemas = getDatabase().query(ctx -> ctx.fetch("SELECT schema_name FROM information_schema.schemata;")) + .stream() + .map(record -> record.get("schema_name").toString()) + .toList(); + } catch (final SQLException e) { + // if we can't fetch the schemas, just return. + return; + } + + int schemasDeletedCount = 0; + for (final String schema : schemas) { + if (TestingNamespaces.isOlderThan2Days(schema)) { + try { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + schemasDeletedCount++; + } catch (final SQLException e) { + LOGGER.error("Failed to delete old dataset: {}", schema, e); + } + } + } + LOGGER.info("Deleted {} old schemas.", schemasDeletedCount); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) throws Exception { + System.out.println("TEARING_DOWN_SCHEMAS: " + TEST_SCHEMAS); + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", config.get("schema").asText()))); + for (final String schema : TEST_SCHEMAS) { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + } + getDatabase().query(ctx -> ctx.execute(String.format("drop user if exists %s;", USER_WITHOUT_CREDS))); + RedshiftConnectionHandler.close(connection); + } + + protected Database createDatabase() { + connection = ConnectionFactory.create(baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), + baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), + RedshiftInsertDestination.SSL_JDBC_PARAMETERS, + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + baseConfig.get(JdbcUtils.HOST_KEY).asText(), + baseConfig.get(JdbcUtils.PORT_KEY).asInt(), + baseConfig.get(JdbcUtils.DATABASE_KEY).asText())); + + return new Database(DSL.using(connection)); + } + + protected Database getDatabase() { + return database; + } + + @Override + protected int getMaxRecordValueLimit() { + return RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE; + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java new file mode 100644 index 000000000000..bbeab71e6be0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +public class RedshiftFileBufferTest { + + private final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); + private final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); + + @Test + public void testGetFileBufferDefault() { + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + } + + @Test + public void testGetFileBufferMaxLimited() { + ((ObjectNode) config).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); + } + + @Test + public void testGetMinimumFileBufferCount() { + ((ObjectNode) config).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); + // User cannot set number of file counts below the default file buffer count, which is existing + // behavior + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java index 57f61b4f39f2..59e9130af79f 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java @@ -9,11 +9,13 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /** * Integration test testing the {@link RedshiftInsertDestination}. */ -public class RedshiftInsertDestinationAcceptanceTest extends RedshiftStagingS3DestinationAcceptanceTest { +@Disabled +public class RedshiftInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() throws IOException { return Jsons.deserialize(Files.readString(Path.of("secrets/config.json"))); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java index 52cd07ce1748..0732e6041454 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java @@ -8,8 +8,14 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; -public class RedshiftS3StagingInsertDestinationAcceptanceTest extends RedshiftStagingS3DestinationAcceptanceTest { +/** + * Integration test testing {@link RedshiftStagingS3Destination}. The default Redshift integration + * test credentials contain S3 credentials - this automatically causes COPY to be selected. + */ +@Disabled +public class RedshiftS3StagingInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() { return Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java deleted file mode 100644 index 47b6926a0517..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.amazon.redshift.util.RedshiftTimestamp; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.factory.ConnectionFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.destination.TestingNamespaces; -import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.io.IOException; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.jooq.impl.DSL; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Integration test testing {@link RedshiftStagingS3Destination}. The default Redshift integration - * test credentials contain S3 credentials - this automatically causes COPY to be selected. - */ -// these tests are not yet thread-safe, unlike the DV2 tests. -@Execution(ExecutionMode.SAME_THREAD) -public abstract class RedshiftStagingS3DestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftStagingS3DestinationAcceptanceTest.class); - - // config from which to create / delete schemas. - private JsonNode baseConfig; - // config which refers to the schema that the test is being run in. - protected JsonNode config; - private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); - private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); - - private Database database; - private Connection connection; - protected TestDestinationEnv testDestinationEnv; - - private final ObjectMapper mapper = new ObjectMapper(); - - @Override - protected String getImageName() { - return "airbyte/destination-redshift:dev"; - } - - @Override - protected JsonNode getConfig() { - return config; - } - - public JsonNode getStaticConfig() throws IOException { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); - } - - @Override - protected JsonNode getFailCheckConfig() { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("password", "wrong password"); - return invalidConfig; - } - - @Test - void testCheckIncorrectPasswordFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("password", "fake"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000;")); - } - - @Test - public void testCheckIncorrectUsernameFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("username", ""); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000;")); - } - - @Test - public void testCheckIncorrectHostFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("host", "localhost2"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001;")); - } - - @Test - public void testCheckIncorrectDataBaseFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("database", "wrongdatabase"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 3D000;")); - } - - /* - * FileBuffer Default Tests - */ - @Test - public void testGetFileBufferDefault() { - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetFileBufferMaxLimited() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - assertEquals(destination.getNumberOfFileBuffers(defaultConfig), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetMinimumFileBufferCount() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - // User cannot set number of file counts below the default file buffer count, which is existing - // behavior - assertEquals(destination.getNumberOfFileBuffers(defaultConfig), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new RedshiftTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected boolean supportIncrementalSchemaChanges() { - return true; - } - - @Override - protected boolean supportsInDestinationNormalization() { - return true; - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(j -> j.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) - throws Exception { - String tableName = namingResolver.getIdentifier(streamName); - if (!tableName.startsWith("\"")) { - // Currently, Normalization always quote tables identifiers - tableName = "\"" + tableName + "\""; - } - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return getDatabase().query( - ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(record -> getJsonFromRecord( - record, - value -> { - if (value instanceof final RedshiftTimestamp rts) { - // We can't just use rts.toInstant().toString(), because that will mangle historical - // dates (e.g. 1504-02-28...) because toInstant() just converts to epoch millis, - // which works _very badly_ for for very old dates. - // Instead, convert to a string and then parse that string. - // We can't just rts.toString(), because that loses the timezone... - // so instead we use getPostgresqlString and parse that >.> - // Thanks, redshift. - return Optional.of( - ZonedDateTime.parse( - rts.getPostgresqlString(), - new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss") - .optionalStart() - .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 9, true) - .optionalEnd() - .appendPattern("X") - .toFormatter()) - .withZoneSameInstant(ZoneOffset.UTC) - .toString()); - } else { - return Optional.empty(); - } - })) - .collect(Collectors.toList())); - } - - // for each test we create a new schema in the database. run the test in there and then remove it. - @Override - protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { - final String schemaName = TestingNamespaces.generate(); - final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); - baseConfig = getStaticConfig(); - database = createDatabase(); - removeOldNamespaces(); - getDatabase().query(ctx -> ctx.execute(createSchemaQuery)); - final String createUser = String.format("create user %s with password '%s' SESSION TIMEOUT 60;", - USER_WITHOUT_CREDS, baseConfig.get("password").asText()); - getDatabase().query(ctx -> ctx.execute(createUser)); - final JsonNode configForSchema = Jsons.clone(baseConfig); - ((ObjectNode) configForSchema).put("schema", schemaName); - TEST_SCHEMAS.add(schemaName); - config = configForSchema; - testDestinationEnv = testEnv; - } - - private void removeOldNamespaces() { - final List schemas; - try { - schemas = getDatabase().query(ctx -> ctx.fetch("SELECT schema_name FROM information_schema.schemata;")) - .stream() - .map(record -> record.get("schema_name").toString()) - .toList(); - } catch (final SQLException e) { - // if we can't fetch the schemas, just return. - return; - } - - int schemasDeletedCount = 0; - for (final String schema : schemas) { - if (TestingNamespaces.isOlderThan2Days(schema)) { - try { - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); - schemasDeletedCount++; - } catch (final SQLException e) { - LOGGER.error("Failed to delete old dataset: {}", schema, e); - } - } - } - LOGGER.info("Deleted {} old schemas.", schemasDeletedCount); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - System.out.println("TEARING_DOWN_SCHEMAS: " + TEST_SCHEMAS); - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", config.get("schema").asText()))); - for (final String schema : TEST_SCHEMAS) { - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); - } - getDatabase().query(ctx -> ctx.execute(String.format("drop user if exists %s;", USER_WITHOUT_CREDS))); - RedshiftConnectionHandler.close(connection); - } - - protected Database createDatabase() { - connection = ConnectionFactory.create(baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), - baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), - RedshiftInsertDestination.SSL_JDBC_PARAMETERS, - String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - baseConfig.get(JdbcUtils.HOST_KEY).asText(), - baseConfig.get(JdbcUtils.PORT_KEY).asInt(), - baseConfig.get(JdbcUtils.DATABASE_KEY).asText())); - - return new Database(DSL.using(connection)); - } - - protected Database getDatabase() { - return database; - } - - public RedshiftSQLNameTransformer getNamingResolver() { - return namingResolver; - } - - @Override - protected int getMaxRecordValueLimit() { - return RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE; - } - - @Override - protected int getGenerateBigStringAddExtraCharacters() { - return 1; - } - -} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java index 1d80069433da..4c2da0a04ce7 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java @@ -10,11 +10,13 @@ import io.airbyte.commons.json.Jsons; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /* * SshKeyRedshiftInsertDestinationAcceptanceTest runs basic Redshift Destination Tests using the SQL * Insert mechanism for upload of data and "key" authentication for the SSH bastion configuration. */ +@Disabled public class SshKeyRedshiftInsertDestinationAcceptanceTest extends SshRedshiftDestinationBaseAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java index 6f423e5e43d1..72c992fdb183 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java @@ -10,12 +10,14 @@ import io.airbyte.commons.json.Jsons; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; /* * SshPasswordRedshiftStagingDestinationAcceptanceTest runs basic Redshift Destination Tests using * the S3 Staging mechanism for upload of data and "password" authentication for the SSH bastion * configuration. */ +@Disabled public class SshPasswordRedshiftStagingDestinationAcceptanceTest extends SshRedshiftDestinationBaseAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java index 72a3bf1d1906..514fd14363a8 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java @@ -5,38 +5,18 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.db.factory.DataSourceFactory; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcTypingDedupingTest; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; -import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGeneratorIntegrationTest.RedshiftSourceOperations; -import java.nio.file.Path; -import java.util.List; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.conf.Settings; import org.jooq.impl.DSL; -/** - * This class is basically the same as - * {@link io.airbyte.integrations.destination.snowflake.typing_deduping.AbstractSnowflakeTypingDedupingTest}. - * But (a) it uses jooq to construct the sql statements, and (b) it doesn't need to upcase anything. - * At some point we might (?) want to do a refactor to combine them. At the very least, this class - * is probably useful for other JDBC destination implementations. - */ -public abstract class AbstractRedshiftTypingDedupingTest extends BaseTypingDedupingTest { - - private JdbcDatabase database; - private DataSource dataSource; - - protected abstract String getConfigPath(); +public abstract class AbstractRedshiftTypingDedupingTest extends JdbcTypingDedupingTest { @Override protected String getImageName() { @@ -44,49 +24,17 @@ protected String getImageName() { } @Override - protected JsonNode generateConfig() { - final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of(getConfigPath()))); - ((ObjectNode) config).put("schema", "typing_deduping_default_schema" + getUniqueSuffix()); - final RedshiftInsertDestination insertDestination = new RedshiftInsertDestination(); - dataSource = insertDestination.getDataSource(config); - database = insertDestination.getDatabase(dataSource, new RedshiftSourceOperations()); - return config; - } - - @Override - protected List dumpRawTableRecords(String streamNamespace, final String streamName) throws Exception { - if (streamNamespace == null) { - streamNamespace = getDefaultSchema(); - } - final String tableName = StreamId.concatenateRawTableName(streamNamespace, streamName); - final String schema = getRawSchema(); - return database.queryJsons(DSL.selectFrom(DSL.name(schema, tableName)).getSQL()); - } - - @Override - protected List dumpFinalTableRecords(String streamNamespace, final String streamName) throws Exception { - if (streamNamespace == null) { - streamNamespace = getDefaultSchema(); - } - return database.queryJsons(DSL.selectFrom(DSL.name(streamNamespace, streamName)).getSQL()); - } - - @Override - protected void teardownStreamAndNamespace(String streamNamespace, final String streamName) throws Exception { - if (streamNamespace == null) { - streamNamespace = getDefaultSchema(); - } - database.execute(DSL.dropTableIfExists(DSL.name(getRawSchema(), StreamId.concatenateRawTableName(streamNamespace, streamName))).getSQL()); - database.execute(DSL.dropSchemaIfExists(DSL.name(streamNamespace)).cascade().getSQL()); + protected DataSource getDataSource(final JsonNode config) { + return new RedshiftInsertDestination().getDataSource(config); } @Override - protected void globalTeardown() throws Exception { - DataSourceFactory.close(dataSource); + protected JdbcCompatibleSourceOperations getSourceOperations() { + return new RedshiftSourceOperations(); } @Override - protected SqlGenerator getSqlGenerator() { + protected SqlGenerator getSqlGenerator() { return new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { // Override only for tests to print formatted SQL. The actual implementation should use unformatted @@ -99,15 +47,4 @@ protected DSLContext getDslContext() { }; } - /** - * Subclasses using a config with a nonstandard raw table schema should override this method. - */ - protected String getRawSchema() { - return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; - } - - private String getDefaultSchema() { - return getConfig().get("schema").asText(); - } - } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java index 6bec5adb9adc..972cde0a1a58 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java @@ -4,14 +4,18 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { @Override - protected String getConfigPath() { - return "secrets/1s1t_config_staging_raw_schema_override.json"; + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_staging_raw_schema_override.json"))); } @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java index c1c0d0194106..c38182ffa54a 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java @@ -4,11 +4,16 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; + public class RedshiftS3StagingTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { @Override - protected String getConfigPath() { - return "secrets/1s1t_config_staging.json"; + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_staging.json"))); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java index dbb37cd8333e..854fe35cfff6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -5,16 +5,9 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; -import static org.junit.jupiter.api.Assertions.assertAll; +import static io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations.escapeStringLiteral; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; @@ -24,13 +17,12 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; -import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcSqlGeneratorIntegrationTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; -import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; -import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; import java.nio.file.Files; @@ -41,23 +33,20 @@ import java.time.LocalDateTime; import java.time.OffsetTime; import java.time.ZoneOffset; -import java.util.Arrays; import java.util.List; -import java.util.Optional; import javax.sql.DataSource; import org.jooq.DSLContext; -import org.jooq.InsertValuesStepN; -import org.jooq.Name; -import org.jooq.Record; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.SQLDialect; import org.jooq.conf.Settings; import org.jooq.impl.DSL; import org.jooq.impl.DefaultDataType; -import org.jooq.impl.SQLDataType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -public class RedshiftSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { +public class RedshiftSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { /** * Redshift's JDBC driver doesn't map certain data types onto {@link java.sql.JDBCType} usefully. @@ -148,7 +137,7 @@ public static void teardownRedshift() throws Exception { } @Override - protected SqlGenerator getSqlGenerator() { + protected JdbcSqlGenerator getSqlGenerator() { return new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { // Override only for tests to print formatted SQL. The actual implementation should use unformatted @@ -162,160 +151,41 @@ protected DSLContext getDslContext() { } @Override - protected DestinationHandler getDestinationHandler() { + protected DestinationHandler getDestinationHandler() { return new RedshiftDestinationHandler(databaseName, database); } @Override - protected void createNamespace(final String namespace) throws Exception { - database.execute(DSL.createSchemaIfNotExists(namespace).getSQL()); + protected JdbcDatabase getDatabase() { + return database; } @Override - protected void createRawTable(final StreamId streamId) throws Exception { - database.execute(DSL.createTable(DSL.name(streamId.rawNamespace(), streamId.rawName())) - .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) - .column(COLUMN_NAME_AB_EXTRACTED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)) - .column(COLUMN_NAME_AB_LOADED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE) - .column(COLUMN_NAME_DATA, new DefaultDataType<>(null, String.class, "super").nullable(false)) - .getSQL()); + protected DataType getStructType() { + return new DefaultDataType<>(null, String.class, "super"); } @Override - protected void createV1RawTable(final StreamId v1RawTable) throws Exception { - database.execute(DSL.createTable(DSL.name(v1RawTable.rawNamespace(), v1RawTable.rawName())) - .column(COLUMN_NAME_AB_ID, SQLDataType.VARCHAR(36).nullable(false)) - .column(COLUMN_NAME_EMITTED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)) - .column(COLUMN_NAME_DATA, new DefaultDataType<>(null, String.class, "super").nullable(false)) - .getSQL()); + protected SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; } @Override - protected List dumpRawTableRecords(final StreamId streamId) throws Exception { - return database.queryJsons(DSL.selectFrom(DSL.name(streamId.rawNamespace(), streamId.rawName())).getSQL()); - } - - @Override - protected List dumpFinalTableRecords(final StreamId streamId, final String suffix) throws Exception { - return database.queryJsons(DSL.selectFrom(DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix)).getSQL()); - } - - @Override - protected void teardownNamespace(final String namespace) throws Exception { - database.execute(DSL.dropSchema(namespace).cascade().getSQL()); - } - - @Override - protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, - final StreamId streamId, - final String suffix, - final List records) - throws Exception { - final List columnNames = includeCdcDeletedAt ? FINAL_TABLE_COLUMN_NAMES_CDC : FINAL_TABLE_COLUMN_NAMES; - insertRecords( - DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix), - columnNames, - records, - COLUMN_NAME_AB_META, "struct", "array", "unknown"); - } - - @Override - protected void insertV1RawTableRecords(final StreamId streamId, final List records) throws Exception { - insertRecords( - DSL.name(streamId.rawNamespace(), streamId.rawName()), - LEGACY_RAW_TABLE_COLUMNS, - records, - COLUMN_NAME_DATA); - } - - @Override - protected void insertRawTableRecords(final StreamId streamId, final List records) throws Exception { - insertRecords( - DSL.name(streamId.rawNamespace(), streamId.rawName()), - JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES, - records, - COLUMN_NAME_DATA); - } - - /** - * Insert arbitrary records into an arbitrary table. - * - * @param columnsToParseJson Columns that must be wrapped in JSON_PARSE, because we're inserting - * them into a SUPER column. Naively inserting a string results a SUPER value containing a - * json string, rather than a json object. - */ - private void insertRecords(final Name tableName, final List columnNames, final List records, final String... columnsToParseJson) - throws SQLException { - InsertValuesStepN insert = DSL.insertInto( - DSL.table(tableName), - columnNames.stream().map(DSL::field).toList()); - for (final JsonNode record : records) { - insert = insert.values( - columnNames.stream() - .map(fieldName -> { - // Convert this field to a string. Pretty naive implementation. - final JsonNode column = record.get(fieldName); - final String columnAsString; - if (column == null) { - columnAsString = null; - } else if (column.isTextual()) { - columnAsString = column.asText(); - } else { - columnAsString = column.toString(); - } - - if (Arrays.asList(columnsToParseJson).contains(fieldName)) { - // TODO this is redshift-specific. If we try and genericize this class, we need to handle this - // specifically - return DSL.function("JSON_PARSE", String.class, DSL.inline(escapeStringLiteral(columnAsString))); - } else { - return DSL.inline(escapeStringLiteral(columnAsString)); - } - }) - .toList()); - } - database.execute(insert.getSQL()); + protected Field toJsonValue(final String valueAsString) { + return DSL.function("JSON_PARSE", String.class, DSL.val(escapeStringLiteral(valueAsString))); } @Override @Test public void testCreateTableIncremental() throws Exception { - final String sql = generator.createTable(incrementalDedupStream, "", false); + final Sql sql = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(sql); - - final Optional existingTable = destinationHandler.findExistingTable(incrementalDedupStream.id()); - - assertTrue(existingTable.isPresent()); - assertAll( - () -> assertEquals("varchar", existingTable.get().columns().get("_airbyte_raw_id").type()), - () -> assertEquals("timestamptz", existingTable.get().columns().get("_airbyte_extracted_at").type()), - () -> assertEquals("super", existingTable.get().columns().get("_airbyte_meta").type()), - () -> assertEquals("int8", existingTable.get().columns().get("id1").type()), - () -> assertEquals("int8", existingTable.get().columns().get("id2").type()), - () -> assertEquals("timestamptz", existingTable.get().columns().get("updated_at").type()), - () -> assertEquals("super", existingTable.get().columns().get("struct").type()), - () -> assertEquals("super", existingTable.get().columns().get("array").type()), - () -> assertEquals("varchar", existingTable.get().columns().get("string").type()), - () -> assertEquals("numeric", existingTable.get().columns().get("number").type()), - () -> assertEquals("int8", existingTable.get().columns().get("integer").type()), - () -> assertEquals("bool", existingTable.get().columns().get("boolean").type()), - () -> assertEquals("timestamptz", existingTable.get().columns().get("timestamp_with_timezone").type()), - () -> assertEquals("timestamp", existingTable.get().columns().get("timestamp_without_timezone").type()), - () -> assertEquals("timetz", existingTable.get().columns().get("time_with_timezone").type()), - () -> assertEquals("time", existingTable.get().columns().get("time_without_timezone").type()), - () -> assertEquals("date", existingTable.get().columns().get("date").type()), - () -> assertEquals("super", existingTable.get().columns().get("unknown").type())); + List initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + assertEquals(1, initialStates.size()); + final DestinationInitialState initialState = initialStates.getFirst(); + assertTrue(initialState.isFinalTablePresent()); + assertFalse(initialState.isSchemaMismatch()); // TODO assert on table clustering, etc. } - private static String escapeStringLiteral(final String str) { - if (str == null) { - return null; - } else { - // jooq handles most things - // but we need to manually escape backslashes for some reason - return str.replace("\\", "\\\\"); - } - } - } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java index 89a54f8eb1e2..b7c78a4cec8e 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java @@ -4,14 +4,18 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { @Override - protected String getConfigPath() { - return "secrets/1s1t_config_raw_schema_override.json"; + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_raw_schema_override.json"))); } @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java index 17ffb70547a6..d99d597e4510 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java @@ -4,11 +4,16 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; + public class RedshiftStandardInsertsTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { @Override - protected String getConfigPath() { - return "secrets/1s1t_config.json"; + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config.json"))); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl index c805113dc6c2..61024be7867d 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -2,3 +2,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl index 8aa852183061..6f53b9f3c12d 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -3,3 +3,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} // Invalid columns are nulled out (i.e. SQL null, not JSON null) {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl index 80fac124d28d..4012c086a9e6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -3,3 +3,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} // Invalid data is still allowed in the raw table. {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl index 6e9258bab255..0989dfc17ed0 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -2,6 +2,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl index 13c59b2f9912..1187ca159d72 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} // Delete Bob, keep Charlie {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl index 32a7e57b1c14..2f634c6ad4e9 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -3,6 +3,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} // And append the records from the second sync {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index f1b6cd3a5e20..f6441416658b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -5,3 +5,5 @@ // Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. // But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +// Note that redshift downcases IAmACaseSensitiveColumnName to all lowercase +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "iamacasesensitivecolumnname": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index a341d911fbbc..6b99169ececf 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -3,3 +3,4 @@ {"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} {"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java deleted file mode 100644 index 5c029abc5e58..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift.copiers; - -import static java.util.Comparator.comparing; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.base.DestinationConfig; -import io.airbyte.cdk.integrations.destination.StandardNameTransformer; -import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; -import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class RedshiftStreamCopierTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftStreamCopierTest.class); - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - // The full path would be something like - // "fake-namespace/fake_stream/2021_12_09_1639077474000_e549e712-b89c-4272-9496-9690ba7f973e.csv" - // The namespace and stream have their hyphens replaced by underscores. Not super clear that that's - // actually required. - // 2021_12_09_1639077474000 is generated from the timestamp. It's followed by a random UUID, in case - // we need to create multiple files. - private static final String EXPECTED_OBJECT_BEGINNING = "fake-bucketPath/fake_namespace/fake_stream/2021_12_09_1639077474000_"; - private static final String EXPECTED_OBJECT_ENDING = ".csv"; - - // equivalent to Thu, 09 Dec 2021 19:17:54 GMT - private static final Timestamp UPLOAD_TIME = Timestamp.from(Instant.ofEpochMilli(1639077474000L)); - - private AmazonS3Client s3Client; - private JdbcDatabase db; - private SqlOperations sqlOperations; - private RedshiftStreamCopier copier; - - @BeforeEach - public void setup() { - DestinationConfig.initialize(Jsons.emptyObject()); - s3Client = mock(AmazonS3Client.class, RETURNS_DEEP_STUBS); - db = mock(JdbcDatabase.class); - sqlOperations = mock(SqlOperations.class); - - final S3DestinationConfig s3Config = S3DestinationConfig.create( - "fake-bucket", - "fake-bucketPath", - "fake-region") - .withEndpoint("fake-endpoint") - .withAccessKeyCredential("fake-access-key-id", "fake-secret-access-key") - .get(); - - copier = new RedshiftStreamCopier( - // In reality, this is normally a UUID - see CopyConsumerFactory#createWriteConfigs - "fake-staging-folder", - "fake-schema", - s3Client, - db, - new S3CopyConfig(true, s3Config), - new StandardNameTransformer(), - sqlOperations, - UPLOAD_TIME, - new ConfiguredAirbyteStream() - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(new AirbyteStream() - .withName("fake-stream") - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)) - .withNamespace("fake-namespace"))); - } - - @Test - public void copiesCorrectFilesToTable() throws SQLException { - // Generate two files - final String file1 = copier.prepareStagingFile(); - for (int i = 0; i < RedshiftStreamCopier.MAX_PARTS_PER_FILE - 1; i++) { - copier.prepareStagingFile(); - } - final String file2 = copier.prepareStagingFile(); - final List expectedFiles = List.of(file1, file2).stream().sorted().toList(); - - copier.copyStagingFileToTemporaryTable(); - - final AtomicReference manifestUuid = new AtomicReference<>(); - verify(s3Client).putObject( - eq("fake-bucket"), - argThat(path -> { - final boolean startsCorrectly = path.startsWith("fake-bucketPath/fake-staging-folder/fake-schema/"); - final boolean endsCorrectly = path.endsWith(".manifest"); - // Make sure that we have a valid UUID - manifestUuid.set(path.replaceFirst("^fake-bucketPath/fake-staging-folder/fake-schema/", "").replaceFirst(".manifest$", "")); - UUID.fromString(manifestUuid.get()); - - return startsCorrectly && endsCorrectly; - }), - (String) argThat(manifestStr -> { - try { - final JsonNode manifest = OBJECT_MAPPER.readTree((String) manifestStr); - final List entries = Lists.newArrayList(manifest.get("entries").elements()).stream() - .sorted(comparing(entry -> entry.get("url").asText())).toList(); - - boolean entriesAreCorrect = true; - for (int i = 0; i < 2; i++) { - final String expectedFilename = expectedFiles.get(i); - final JsonNode manifestEntry = entries.get(i); - entriesAreCorrect &= isManifestEntryCorrect(manifestEntry, expectedFilename); - if (!entriesAreCorrect) { - LOGGER.error("Invalid entry: {}", manifestEntry); - } - } - - return entriesAreCorrect && entries.size() == 2; - } catch (final JsonProcessingException e) { - throw new RuntimeException(e); - } - })); - - verify(db).execute(String.format( - """ - COPY fake-schema.%s FROM 's3://fake-bucket/fake-bucketPath/fake-staging-folder/fake-schema/%s.manifest' - CREDENTIALS 'aws_access_key_id=fake-access-key-id;aws_secret_access_key=fake-secret-access-key' - CSV REGION 'fake-region' TIMEFORMAT 'auto' - STATUPDATE OFF - MANIFEST;""", - copier.getTmpTableName(), - manifestUuid.get())); - } - - private static boolean isManifestEntryCorrect(final JsonNode entry, final String expectedFilename) { - final String url = entry.get("url").asText(); - final boolean mandatory = entry.get("mandatory").asBoolean(); - - return ("s3://fake-bucket/" + expectedFilename).equals(url) && mandatory; - } - -} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java new file mode 100644 index 000000000000..341c7df14ced --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import org.jooq.DSLContext; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class RedshiftSqlGeneratorTest { + + private static final Random RANDOM = new Random(); + + private static final RedshiftSqlGenerator redshiftSqlGenerator = new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { + + // Override only for tests to print formatted SQL. The actual implementation should use unformatted + // to save bytes. + @Override + protected DSLContext getDslContext() { + return DSL.using(getDialect(), new Settings().withRenderFormatted(true)); + } + + }; + + private StreamId streamId; + + private StreamConfig incrementalDedupStream; + + private StreamConfig incrementalAppendStream; + + @BeforeEach + public void setup() { + streamId = new StreamId("test_schema", "users_final", "test_schema", "users_raw", "test_schema", "users_final"); + final ColumnId id1 = redshiftSqlGenerator.buildColumnId("id1"); + final ColumnId id2 = redshiftSqlGenerator.buildColumnId("id2"); + final List primaryKey = List.of(id1, id2); + final ColumnId cursor = redshiftSqlGenerator.buildColumnId("updated_at"); + + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put(id1, AirbyteProtocolType.INTEGER); + columns.put(id2, AirbyteProtocolType.INTEGER); + columns.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("struct"), new Struct(new LinkedHashMap<>())); + columns.put(redshiftSqlGenerator.buildColumnId("array"), new Array(AirbyteProtocolType.UNKNOWN)); + columns.put(redshiftSqlGenerator.buildColumnId("string"), AirbyteProtocolType.STRING); + columns.put(redshiftSqlGenerator.buildColumnId("number"), AirbyteProtocolType.NUMBER); + columns.put(redshiftSqlGenerator.buildColumnId("integer"), AirbyteProtocolType.INTEGER); + columns.put(redshiftSqlGenerator.buildColumnId("boolean"), AirbyteProtocolType.BOOLEAN); + columns.put(redshiftSqlGenerator.buildColumnId("timestamp_with_timezone"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("timestamp_without_timezone"), AirbyteProtocolType.TIMESTAMP_WITHOUT_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("time_with_timezone"), AirbyteProtocolType.TIME_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("time_without_timezone"), AirbyteProtocolType.TIME_WITHOUT_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("date"), AirbyteProtocolType.DATE); + columns.put(redshiftSqlGenerator.buildColumnId("unknown"), AirbyteProtocolType.UNKNOWN); + columns.put(redshiftSqlGenerator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + incrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + columns); + incrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + columns); + } + + @Test + public void testTypingAndDeduping() throws IOException { + final String expectedSql = MoreResources.readResource("typing_deduping_with_cdc.sql"); + final Sql generatedSql = + redshiftSqlGenerator.updateTable(incrementalDedupStream, "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); + final List expectedSqlLines = Arrays.stream(expectedSql.split("\n")).map(String::trim).toList(); + final List generatedSqlLines = generatedSql.asSqlStrings("BEGIN", "COMMIT").stream() + .flatMap(statement -> Arrays.stream(statement.split("\n"))) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .toList(); + System.out.println(generatedSql); + assertEquals(expectedSqlLines, generatedSqlLines); + } + + @Test + public void test2000ColumnSql() { + final ColumnId id1 = redshiftSqlGenerator.buildColumnId("id1"); + final ColumnId id2 = redshiftSqlGenerator.buildColumnId("id2"); + final List primaryKey = List.of(id1, id2); + final ColumnId cursor = redshiftSqlGenerator.buildColumnId("updated_at"); + + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put(id1, AirbyteProtocolType.INTEGER); + columns.put(id2, AirbyteProtocolType.INTEGER); + columns.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + + for (int i = 0; i < 2000; i++) { + final String columnName = RANDOM + .ints('a', 'z' + 1) + .limit(15) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + columns.put(redshiftSqlGenerator.buildColumnId(columnName), AirbyteProtocolType.STRING); + } + final Sql generatedSql = redshiftSqlGenerator.updateTable(new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + columns), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); + // This should not throw an exception. + assertFalse(generatedSql.transactions().isEmpty()); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql new file mode 100644 index 000000000000..371b189e4856 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql @@ -0,0 +1,214 @@ +BEGIN; +insert into "test_schema"."users_finalunittest" ( + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown", + "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta" +) +with + "intermediate_data" as ( + select + cast("_airbyte_data"."id1" as bigint) as "id1", + cast("_airbyte_data"."id2" as bigint) as "id2", + cast("_airbyte_data"."updated_at" as timestamp with time zone) as "updated_at", + CASE WHEN JSON_TYPEOF("_airbyte_data"."struct") = 'object' THEN cast("_airbyte_data"."struct" as super) END as "struct", + CASE WHEN JSON_TYPEOF("_airbyte_data"."array") = 'array' THEN cast("_airbyte_data"."array" as super) END as "array", + CASE WHEN ( + JSON_TYPEOF("_airbyte_data"."string") <> 'string' + and "_airbyte_data"."string" is not null + ) THEN JSON_SERIALIZE("_airbyte_data"."string") ELSE cast("_airbyte_data"."string" as varchar(65535)) END as "string", + cast("_airbyte_data"."number" as decimal(38, 9)) as "number", + cast("_airbyte_data"."integer" as bigint) as "integer", + cast("_airbyte_data"."boolean" as boolean) as "boolean", + cast("_airbyte_data"."timestamp_with_timezone" as timestamp with time zone) as "timestamp_with_timezone", + cast("_airbyte_data"."timestamp_without_timezone" as timestamp) as "timestamp_without_timezone", + cast("_airbyte_data"."time_with_timezone" as time with time zone) as "time_with_timezone", + cast("_airbyte_data"."time_without_timezone" as time) as "time_without_timezone", + cast("_airbyte_data"."date" as date) as "date", + cast("_airbyte_data"."unknown" as super) as "unknown", + cast("_airbyte_data"."_ab_cdc_deleted_at" as timestamp with time zone) as "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + OBJECT( + 'errors', + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + CASE WHEN ( + "_airbyte_data"."id1" is not null + and "id1" is null + ) THEN ARRAY('Problem with `id1`') ELSE ARRAY() END , + CASE WHEN ( + "_airbyte_data"."id2" is not null + and "id2" is null + ) THEN ARRAY('Problem with `id2`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."updated_at" is not null + and "updated_at" is null + ) THEN ARRAY('Problem with `updated_at`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."struct" is not null + and "struct" is null + ) THEN ARRAY('Problem with `struct`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."array" is not null + and "array" is null + ) THEN ARRAY('Problem with `array`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."string" is not null + and "string" is null + ) THEN ARRAY('Problem with `string`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."number" is not null + and "number" is null + ) THEN ARRAY('Problem with `number`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."integer" is not null + and "integer" is null + ) THEN ARRAY('Problem with `integer`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."boolean" is not null + and "boolean" is null + ) THEN ARRAY('Problem with `boolean`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."timestamp_with_timezone" is not null + and "timestamp_with_timezone" is null + ) THEN ARRAY('Problem with `timestamp_with_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."timestamp_without_timezone" is not null + and "timestamp_without_timezone" is null + ) THEN ARRAY('Problem with `timestamp_without_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."time_with_timezone" is not null + and "time_with_timezone" is null + ) THEN ARRAY('Problem with `time_with_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."time_without_timezone" is not null + and "time_without_timezone" is null + ) THEN ARRAY('Problem with `time_without_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."date" is not null + and "date" is null + ) THEN ARRAY('Problem with `date`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."unknown" is not null + and "unknown" is null + ) THEN ARRAY('Problem with `unknown`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."_ab_cdc_deleted_at" is not null + and "_ab_cdc_deleted_at" is null + ) THEN ARRAY('Problem with `_ab_cdc_deleted_at`') ELSE ARRAY() END + ) + ) as "_airbyte_meta" + from "test_schema"."users_raw" + where ( + ( + "_airbyte_loaded_at" is null + or ( + "_airbyte_loaded_at" is not null + and JSON_TYPEOF("_airbyte_data"."_ab_cdc_deleted_at") <> 'null' + ) + ) + and "_airbyte_extracted_at" > '2023-02-15T18:35:24Z' + ) + ), + "numbered_rows" as ( + select + *, + row_number() over ( + partition by "id1", "id2" + order by + "updated_at" desc NULLS LAST, + "_airbyte_extracted_at" desc + ) as "row_number" + from "intermediate_data" + ) +select + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown", + "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta" +from "numbered_rows" +where "row_number" = 1; +delete from "test_schema"."users_finalunittest" +where "_airbyte_raw_id" in ( + select "_airbyte_raw_id" + from ( + select + "_airbyte_raw_id", + row_number() over ( + partition by "id1", "id2" + order by + "updated_at" desc NULLS LAST, + "_airbyte_extracted_at" desc + ) as "row_number" + from "test_schema"."users_finalunittest" + ) as "airbyte_ids" + where "row_number" <> 1 +); +delete from "test_schema"."users_finalunittest" +where "_ab_cdc_deleted_at" is not null; +update "test_schema"."users_raw" +set +"_airbyte_loaded_at" = GETDATE() +where ( + "_airbyte_loaded_at" is null + and "_airbyte_extracted_at" > '2023-02-15T18:35:24Z' + ); +COMMIT; diff --git a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml index dc0a004c20bd..0ca7298c3f5b 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/destination-s3-glue githubIssueLabel: destination-s3-glue icon: s3-glue.svg diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json index d5571f31ceb1..896817109431 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json @@ -54,31 +54,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index 7547dd1c77ac..1e53b23dced0 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.7.0' + cdkVersionRequired = '0.10.2' features = ['db-destinations', 's3-destinations'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 773835ea9bc3..617cf229c6bb 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 0.5.6 + dockerImageTag: 0.5.9 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg @@ -11,10 +11,8 @@ data: registries: cloud: enabled: true - dockerImageTag: 0.5.1 oss: enabled: true - dockerImageTag: 0.5.1 releaseStage: generally_available resourceRequirements: jobSpecific: diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index a13b2d49bb84..5e779c15eb6b 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -54,31 +54,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 9d625fcd6dbf..b84e054c0609 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -1,15 +1,19 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.7.0' - features = ['db-destinations', 's3-destinations'] + cdkVersionRequired = '0.23.2' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } -airbyteJavaConnector.addCdkDependencies() +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-this-escape" + } +} application { mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestinationRunner' @@ -38,24 +42,6 @@ integrationTestJava { } dependencies { - implementation 'com.google.cloud:google-cloud-storage:1.113.16' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.25.5' implementation 'net.snowflake:snowflake-jdbc:3.14.1' - implementation 'org.apache.commons:commons-csv:1.4' implementation 'org.apache.commons:commons-text:1.10.0' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - implementation "io.aesy:datasize:1.0.0" - implementation 'com.zaxxer:HikariCP:5.0.1' - - implementation project(':airbyte-integrations:connectors:destination-gcs') - -// this is a configuration to make mockito work with final classes - testImplementation 'org.mockito:mockito-inline:2.13.0' - - integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' - - // TODO: declare typing-deduping as a CDK feature instead of importing from source. - implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) - integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) } diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 165c9515924b..d39c5a8c9669 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.4.14 + dockerImageTag: 3.5.14 dockerRepository: airbyte/destination-snowflake documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake githubIssueLabel: destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index 217ed51abb78..012f0846e41c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -40,4 +40,9 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN return new SnowflakeInternalStagingDestination(airbyteEnvironment).getSerializedMessageConsumer(config, catalog, outputRecordCollector); } + @Override + public boolean isV2Destination() { + return true; + } + } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 50e8ad2c4685..253212ecf628 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -13,6 +13,7 @@ import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.cdk.integrations.destination.staging.StagingConsumerFactory; import io.airbyte.commons.json.Jsons; @@ -99,17 +100,17 @@ private static void attemptStageOperations(final String outputSchema, sqlOperations.attemptWriteToStage(outputSchema, stageName, database); } finally { // drop created tmp stage - sqlOperations.dropStageIfExists(database, stageName); + sqlOperations.dropStageIfExists(database, stageName, null); } } @Override - protected DataSource getDataSource(final JsonNode config) { + public DataSource getDataSource(final JsonNode config) { return SnowflakeDatabase.createDataSource(config, airbyteEnvironment); } @Override - protected JdbcDatabase getDatabase(final DataSource dataSource) { + public JdbcDatabase getDatabase(final DataSource dataSource) { return SnowflakeDatabase.getDatabase(dataSource); } @@ -129,6 +130,11 @@ protected JdbcSqlGenerator getSqlGenerator() { throw new UnsupportedOperationException("Snowflake does not yet use the native JDBC DV2 interface"); } + @Override + protected JdbcDestinationHandler getDestinationHandler(String databaseName, JdbcDatabase database) { + throw new UnsupportedOperationException("Snowflake does not yet use the native JDBC DV2 interface"); + } + @Override public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, @@ -156,16 +162,14 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final SnowflakeV1V2Migrator migrator = new SnowflakeV1V2Migrator(getNamingResolver(), database, databaseName); final SnowflakeV2TableMigrator v2TableMigrator = new SnowflakeV2TableMigrator(database, databaseName, sqlGenerator, snowflakeDestinationHandler); final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); - final int defaultThreadCount = 8; if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator, - defaultThreadCount); + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator); } else { typerDeduper = - new DefaultTyperDeduper<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator, defaultThreadCount); + new DefaultTyperDeduper(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator); } - return new StagingConsumerFactory().createAsync( + return StagingConsumerFactory.builder( outputRecordCollector, database, new SnowflakeInternalStagingSqlOperations(getNamingResolver()), @@ -177,8 +181,16 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN typerDeduper, parsedCatalog, defaultNamespace, - true, - Optional.of(getSnowflakeBufferMemoryLimit())); + true) + .setBufferMemoryLimit(Optional.of(getSnowflakeBufferMemoryLimit())) + .setOptimalBatchSizeBytes( + // The per stream size limit is following recommendations from: + // https://docs.snowflake.com/en/user-guide/data-load-considerations-prepare.html#general-file-sizing-recommendations + // "To optimize the number of parallel operations for a load, + // we recommend aiming to produce data files roughly 100-250 MB (or larger) in size compressed." + 200 * 1024 * 1024) + .build() + .createAsync(); } private static long getSnowflakeBufferMemoryLimit() { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java index d789fe7e5c26..6e8f888ef394 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java @@ -11,11 +11,13 @@ import io.airbyte.commons.string.Strings; import java.io.IOException; import java.sql.SQLException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.stream.Stream; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,13 +64,14 @@ public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final String outputTableName, - final DateTime writeDatetime) { + final Instant writeDatetime) { // see https://docs.snowflake.com/en/user-guide/data-load-considerations-stage.html + final var zonedDateTime = ZonedDateTime.ofInstant(writeDatetime, ZoneOffset.UTC); return nameTransformer.applyDefaultCase(String.format("%s/%02d/%02d/%02d/%s/", - writeDatetime.year().get(), - writeDatetime.monthOfYear().get(), - writeDatetime.dayOfMonth().get(), - writeDatetime.hourOfDay().get(), + zonedDateTime.getYear(), + zonedDateTime.getMonthValue(), + zonedDateTime.getDayOfMonth(), + zonedDateTime.getHour(), connectionId)); } @@ -202,7 +205,7 @@ protected String getCopyQuery(final String stageName, } @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { + public void dropStageIfExists(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception { try { final String query = getDropQuery(stageName); LOGGER.debug("Executing query: {}", query); @@ -222,17 +225,6 @@ protected String getDropQuery(final String stageName) { return String.format(DROP_STAGE_QUERY, stageName); } - @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) throws Exception { - try { - final String query = getRemoveQuery(stageName); - LOGGER.debug("Executing query: {}", query); - database.execute(query); - } catch (final SQLException e) { - throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); - } - } - /** * Creates a SQL query used to remove staging files that were just staged See * https://docs.snowflake.com/en/sql-reference/sql/remove.html for more context diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java index 6be94ea5032f..be9ff16282f7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java @@ -109,8 +109,9 @@ public void insertRecordsInternal(final JdbcDatabase database, @Override protected void insertRecordsInternalV2(final JdbcDatabase jdbcDatabase, final List list, final String s, final String s1) throws Exception { - // Snowflake doesn't have standard inserts... so we probably never want to do this - throw new UnsupportedOperationException("Snowflake does not use the native JDBC DV2 interface"); + // Snowflake doesn't have standard inserts... so we don't do this at real runtime. + // Intentionally do nothing. This method is called from the `check` method. + // It probably shouldn't be, but this is the easiest path to getting this working. } protected String generateFilesList(final List files) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java deleted file mode 100644 index 8415fedf587c..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake.typing_deduping; - -/** - * type is notably _not_ a {@link net.snowflake.client.jdbc.SnowflakeType}. That enum doesn't - * contain all the types that snowflake supports (specifically NUMBER). - */ -public record SnowflakeColumn(String name, String type) {} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java deleted file mode 100644 index 06be84ffe67f..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake.typing_deduping; - -/** - * isNullable is only used to execute a migration away from an older version of - * destination-snowflake, where we created PK columns as NOT NULL. This caused a lot of problems - * because many sources emit null PKs. We may want to remove this field eventually. - */ -public record SnowflakeColumnDefinition(String type, boolean isNullable) { - - @Deprecated - public boolean isNullable() { - return isNullable; - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java index 32cec3dac914..5bfeb5d6b25e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java @@ -4,22 +4,46 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS; + +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialStateImpl; +import io.airbyte.integrations.base.destination.typing_deduping.InitialRawTableState; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import net.snowflake.client.jdbc.SnowflakeSQLException; import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SnowflakeDestinationHandler implements DestinationHandler { +public class SnowflakeDestinationHandler extends JdbcDestinationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDestinationHandler.class); public static final String EXCEPTION_COMMON_PREFIX = "JavaScript execution error: Uncaught Execution of multiple statements failed on statement"; @@ -28,68 +52,87 @@ public class SnowflakeDestinationHandler implements DestinationHandler findExistingTable(final StreamId id) throws SQLException { - // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates - // VARIANT as VARCHAR - final LinkedHashMap columns = database.queryJsons( - """ - SELECT column_name, data_type, is_nullable - FROM information_schema.columns - WHERE table_catalog = ? - AND table_schema = ? - AND table_name = ? - ORDER BY ordinal_position; - """, - databaseName.toUpperCase(), - id.finalNamespace().toUpperCase(), - id.finalName().toUpperCase()).stream() - .collect(LinkedHashMap::new, - (map, row) -> map.put( - row.get("COLUMN_NAME").asText(), - new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), - LinkedHashMap::putAll); - if (columns.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(new SnowflakeTableDefinition(columns)); + public static LinkedHashMap> findExistingTables(final JdbcDatabase database, + final String databaseName, + final List streamIds) + throws SQLException { + final LinkedHashMap> existingTables = new LinkedHashMap<>(); + final String paramHolder = String.join(",", Collections.nCopies(streamIds.size(), "?")); + // convert list stream to array + final String[] namespaces = streamIds.stream().map(StreamId::finalNamespace).toArray(String[]::new); + final String[] names = streamIds.stream().map(StreamId::finalName).toArray(String[]::new); + final String query = """ + SELECT table_schema, table_name, column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema IN (%s) + AND table_name IN (%s) + ORDER BY table_schema, table_name, ordinal_position; + """.formatted(paramHolder, paramHolder); + final String[] bindValues = new String[streamIds.size() * 2 + 1]; + bindValues[0] = databaseName.toUpperCase(); + System.arraycopy(namespaces, 0, bindValues, 1, namespaces.length); + System.arraycopy(names, 0, bindValues, namespaces.length + 1, names.length); + final List results = database.queryJsons(query, bindValues); + for (final JsonNode result : results) { + final String tableSchema = result.get("TABLE_SCHEMA").asText(); + final String tableName = result.get("TABLE_NAME").asText(); + final String columnName = result.get("COLUMN_NAME").asText(); + final String dataType = result.get("DATA_TYPE").asText(); + final String isNullable = result.get("IS_NULLABLE").asText(); + final TableDefinition tableDefinition = existingTables + .computeIfAbsent(tableSchema, k -> new LinkedHashMap<>()) + .computeIfAbsent(tableName, k -> new TableDefinition(new LinkedHashMap<>())); + tableDefinition.columns().put(columnName, new ColumnDefinition(columnName, dataType, 0, fromIsNullableIsoString(isNullable))); } + return existingTables; } - @Override - public boolean isFinalTableEmpty(final StreamId id) throws SQLException { - final int rowCount = database.queryInt( - """ - SELECT row_count - FROM information_schema.tables - WHERE table_catalog = ? - AND table_schema = ? - AND table_name = ? - """, - databaseName.toUpperCase(), - id.finalNamespace().toUpperCase(), - id.finalName().toUpperCase()); - return rowCount == 0; + private LinkedHashMap> getFinalTableRowCount(final List streamIds) throws SQLException { + final LinkedHashMap> tableRowCounts = new LinkedHashMap<>(); + final String paramHolder = String.join(",", Collections.nCopies(streamIds.size(), "?")); + // convert list stream to array + final String[] namespaces = streamIds.stream().map(StreamId::finalNamespace).toArray(String[]::new); + final String[] names = streamIds.stream().map(StreamId::finalName).toArray(String[]::new); + final String query = """ + SELECT table_schema, table_name, row_count + FROM information_schema.tables + WHERE table_catalog = ? + AND table_schema IN (%s) + AND table_name IN (%s) + """.formatted(paramHolder, paramHolder); + final String[] bindValues = new String[streamIds.size() * 2 + 1]; + bindValues[0] = databaseName.toUpperCase(); + System.arraycopy(namespaces, 0, bindValues, 1, namespaces.length); + System.arraycopy(names, 0, bindValues, namespaces.length + 1, names.length); + final List results = database.queryJsons(query, bindValues); + for (final JsonNode result : results) { + final String tableSchema = result.get("TABLE_SCHEMA").asText(); + final String tableName = result.get("TABLE_NAME").asText(); + final int rowCount = result.get("ROW_COUNT").asInt(); + tableRowCounts.computeIfAbsent(tableSchema, k -> new LinkedHashMap<>()).put(tableName, rowCount); + } + return tableRowCounts; } - @Override - public Optional getMinTimestampForSync(final StreamId id) throws Exception { + public InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { final ResultSet tables = database.getMetaData().getTables( databaseName, id.rawNamespace(), id.rawName(), null); if (!tables.next()) { - return Optional.empty(); + return new InitialRawTableState(false, Optional.empty()); } // Snowflake timestamps have nanosecond precision, so decrement by 1ns // And use two explicit queries because COALESCE doesn't short-circuit. // This first query tries to find the oldest raw record with loaded_at = NULL - Optional minUnloadedTimestamp = Optional.ofNullable(database.queryStrings( + final Optional minUnloadedTimestamp = Optional.ofNullable(database.queryStrings( conn -> conn.createStatement().executeQuery(new StringSubstitutor(Map.of( "raw_table", id.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( """ @@ -102,58 +145,165 @@ SELECT to_varchar( """)), // The query will always return exactly one record, so use .get(0) record -> record.getString("MIN_TIMESTAMP")).get(0)); - if (minUnloadedTimestamp.isEmpty()) { - // If there are no unloaded raw records, then we can safely skip all existing raw records. - // This second query just finds the newest raw record. - minUnloadedTimestamp = Optional.ofNullable(database.queryStrings( - conn -> conn.createStatement().executeQuery(new StringSubstitutor(Map.of( - "raw_table", id.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( - """ - SELECT to_varchar( - MAX("_airbyte_extracted_at"), - 'YYYY-MM-DDTHH24:MI:SS.FF9TZH:TZM' - ) AS MIN_TIMESTAMP - FROM ${raw_table} - """)), - record -> record.getString("MIN_TIMESTAMP")).get(0)); + if (minUnloadedTimestamp.isPresent()) { + return new InitialRawTableState(true, minUnloadedTimestamp.map(Instant::parse)); } - return minUnloadedTimestamp.map(Instant::parse); + + // If there are no unloaded raw records, then we can safely skip all existing raw records. + // This second query just finds the newest raw record. + final Optional maxTimestamp = Optional.ofNullable(database.queryStrings( + conn -> conn.createStatement().executeQuery(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( + """ + SELECT to_varchar( + MAX("_airbyte_extracted_at"), + 'YYYY-MM-DDTHH24:MI:SS.FF9TZH:TZM' + ) AS MIN_TIMESTAMP + FROM ${raw_table} + """)), + record -> record.getString("MIN_TIMESTAMP")).get(0)); + return new InitialRawTableState(false, maxTimestamp.map(Instant::parse)); } @Override - public void execute(final String sql) throws Exception { - if ("".equals(sql)) { - return; - } + public void execute(final Sql sql) throws Exception { + final List transactions = sql.asSqlStrings("BEGIN TRANSACTION", "COMMIT"); final UUID queryId = UUID.randomUUID(); - LOGGER.debug("Executing sql {}: {}", queryId, sql); - final long startTime = System.currentTimeMillis(); - - try { - database.execute(sql); - } catch (final SnowflakeSQLException e) { - LOGGER.error("Sql {} failed", queryId, e); - // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant - // part of the message. - final String trimmedMessage; - if (e.getMessage().startsWith(EXCEPTION_COMMON_PREFIX)) { - // The first line is a pretty generic message, so just remove it - trimmedMessage = e.getMessage().substring(e.getMessage().indexOf("\n") + 1); - } else { - trimmedMessage = e.getMessage(); + for (final String transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + LOGGER.debug("Executing sql {}-{}: {}", queryId, transactionId, transaction); + final long startTime = System.currentTimeMillis(); + + try { + database.execute(transaction); + } catch (final SnowflakeSQLException e) { + LOGGER.error("Sql {} failed", queryId, e); + // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant + // part of the message. + final String trimmedMessage; + if (e.getMessage().startsWith(EXCEPTION_COMMON_PREFIX)) { + // The first line is a pretty generic message, so just remove it + trimmedMessage = e.getMessage().substring(e.getMessage().indexOf("\n") + 1); + } else { + trimmedMessage = e.getMessage(); + } + throw new RuntimeException(trimmedMessage, e); + } + + LOGGER.debug("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); + } + } + + private Set getPks(final StreamConfig stream) { + return stream.primaryKey() != null ? stream.primaryKey().stream().map(ColumnId::name).collect(Collectors.toSet()) : Collections.emptySet(); + } + + private boolean isAirbyteRawIdColumnMatch(final TableDefinition existingTable) { + final String abRawIdColumnName = COLUMN_NAME_AB_RAW_ID.toUpperCase(); + return existingTable.columns().containsKey(abRawIdColumnName) && + toJdbcTypeName(AirbyteProtocolType.STRING).equals(existingTable.columns().get(abRawIdColumnName).type()); + } + + private boolean isAirbyteExtractedAtColumnMatch(final TableDefinition existingTable) { + final String abExtractedAtColumnName = COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase(); + return existingTable.columns().containsKey(abExtractedAtColumnName) && + toJdbcTypeName(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE).equals(existingTable.columns().get(abExtractedAtColumnName).type()); + } + + private boolean isAirbyteMetaColumnMatch(TableDefinition existingTable) { + final String abMetaColumnName = COLUMN_NAME_AB_META.toUpperCase(); + return existingTable.columns().containsKey(abMetaColumnName) && + "VARIANT".equals(existingTable.columns().get(abMetaColumnName).type()); + } + + protected boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final TableDefinition existingTable) { + final Set pks = getPks(stream); + // This is same as JdbcDestinationHandler#existingSchemaMatchesStreamConfig with upper case + // conversion. + // TODO: Unify this using name transformer or something. + if (!isAirbyteRawIdColumnMatch(existingTable) || + !isAirbyteExtractedAtColumnMatch(existingTable) || + !isAirbyteMetaColumnMatch(existingTable)) { + // Missing AB meta columns from final table, we need them to do proper T+D so trigger soft-reset + return false; + } + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey().name(), toJdbcTypeName(column.getValue())), + LinkedHashMap::putAll); + + // Filter out Meta columns since they don't exist in stream config. + final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() + .filter(column -> V2_FINAL_TABLE_METADATA_COLUMNS.stream().map(String::toUpperCase) + .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey(), column.getValue().type()), + LinkedHashMap::putAll); + // soft-resetting https://github.com/airbytehq/airbyte/pull/31082 + @SuppressWarnings("deprecation") + final boolean hasPksWithNonNullConstraint = existingTable.columns().entrySet().stream() + .anyMatch(c -> pks.contains(c.getKey()) && !c.getValue().isNullable()); + + return !hasPksWithNonNullConstraint + && actualColumns.equals(intendedColumns); + + } + + @Override + public List gatherInitialState(List streamConfigs) throws Exception { + List streamIds = streamConfigs.stream().map(StreamConfig::id).toList(); + final LinkedHashMap> existingTables = findExistingTables(database, databaseName, streamIds); + final LinkedHashMap> tableRowCounts = getFinalTableRowCount(streamIds); + return streamConfigs.stream().map(streamConfig -> { + try { + final String namespace = streamConfig.id().finalNamespace().toUpperCase(); + final String name = streamConfig.id().finalName().toUpperCase(); + boolean isSchemaMismatch = false; + boolean isFinalTableEmpty = true; + boolean isFinalTablePresent = existingTables.containsKey(namespace) && existingTables.get(namespace).containsKey(name); + boolean hasRowCount = tableRowCounts.containsKey(namespace) && tableRowCounts.get(namespace).containsKey(name); + if (isFinalTablePresent) { + final TableDefinition existingTable = existingTables.get(namespace).get(name); + isSchemaMismatch = !existingSchemaMatchesStreamConfig(streamConfig, existingTable); + isFinalTableEmpty = hasRowCount && tableRowCounts.get(namespace).get(name) == 0; + } + final InitialRawTableState initialRawTableState = getInitialRawTableState(streamConfig.id()); + return new DestinationInitialStateImpl(streamConfig, isFinalTablePresent, initialRawTableState, isSchemaMismatch, isFinalTableEmpty); + } catch (Exception e) { + throw new RuntimeException(e); } - throw new RuntimeException(trimmedMessage, e); + }).collect(Collectors.toList()); + } + + @Override + protected String toJdbcTypeName(AirbyteType airbyteType) { + if (airbyteType instanceof final AirbyteProtocolType p) { + return toJdbcTypeName(p); } - LOGGER.debug("Sql {} completed in {} ms", queryId, System.currentTimeMillis() - startTime); + return switch (airbyteType.getTypeName()) { + case Struct.TYPE -> "OBJECT"; + case Array.TYPE -> "ARRAY"; + case UnsupportedOneOf.TYPE -> "VARIANT"; + case Union.TYPE -> toJdbcTypeName(((Union) airbyteType).chooseType()); + default -> throw new IllegalArgumentException("Unrecognized type: " + airbyteType.getTypeName()); + }; } - /** - * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't - * know how to use - */ - private boolean fromSnowflakeBoolean(final String input) { - return input.equalsIgnoreCase("yes"); + private String toJdbcTypeName(final AirbyteProtocolType airbyteProtocolType) { + return switch (airbyteProtocolType) { + case STRING -> "TEXT"; + case NUMBER -> "FLOAT"; + case INTEGER -> "NUMBER"; + case BOOLEAN -> "BOOLEAN"; + case TIMESTAMP_WITH_TIMEZONE -> "TIMESTAMP_TZ"; + case TIMESTAMP_WITHOUT_TIMEZONE -> "TIMESTAMP_NTZ"; + // If you change this - also change the logic in extractAndCast + case TIME_WITH_TIMEZONE -> "TEXT"; + case TIME_WITHOUT_TIMEZONE -> "TIME"; + case DATE -> "DATE"; + case UNKNOWN -> "VARIANT"; + }; } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java index f62ef06f6ad0..37b0bdaefff8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.concat; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; import static io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction.SOFT_RESET_SUFFIX; import static java.util.stream.Collectors.joining; @@ -14,26 +16,23 @@ import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.base.destination.typing_deduping.Struct; -import io.airbyte.integrations.base.destination.typing_deduping.TableNotMigratedException; import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.time.Instant; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; -public class SnowflakeSqlGenerator implements SqlGenerator { +public class SnowflakeSqlGenerator implements SqlGenerator { public static final String QUOTE = "\""; @@ -105,64 +104,37 @@ public String toDialectType(final AirbyteProtocolType airbyteProtocolType) { } @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + public Sql createSchema(final String schema) { + return Sql.of(new StringSubstitutor(Map.of("schema", StringUtils.wrap(schema, QUOTE))) + .replace("CREATE SCHEMA IF NOT EXISTS ${schema};")); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { final String columnDeclarations = stream.columns().entrySet().stream() .map(column -> "," + column.getKey().name(QUOTE) + " " + toDialectType(column.getValue())) .collect(joining("\n")); final String forceCreateTable = force ? "OR REPLACE" : ""; - return new StringSubstitutor(Map.of( - "final_namespace", stream.id().finalNamespace(QUOTE), + return Sql.of(new StringSubstitutor(Map.of( "final_table_id", stream.id().finalTableId(QUOTE, suffix.toUpperCase()), "force_create_table", forceCreateTable, "column_declarations", columnDeclarations)).replace( """ - CREATE SCHEMA IF NOT EXISTS ${final_namespace}; - CREATE ${force_create_table} TABLE ${final_table_id} ( "_AIRBYTE_RAW_ID" TEXT NOT NULL, "_AIRBYTE_EXTRACTED_AT" TIMESTAMP_TZ NOT NULL, "_AIRBYTE_META" VARIANT NOT NULL ${column_declarations} ); - """); + """)); } @Override - public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final SnowflakeTableDefinition existingTable) - throws TableNotMigratedException { - final Set pks = getPks(stream); - - // Check that the columns match, with special handling for the metadata columns. - final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() - .collect(LinkedHashMap::new, - (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue())), - LinkedHashMap::putAll); - final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() - .filter(column -> JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream().map(String::toUpperCase) - .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) - .collect(LinkedHashMap::new, - (map, column) -> map.put(column.getKey(), column.getValue().type()), - LinkedHashMap::putAll); - // soft-resetting https://github.com/airbytehq/airbyte/pull/31082 - @SuppressWarnings("deprecation") - final boolean hasPksWithNonNullConstraint = existingTable.columns().entrySet().stream() - .anyMatch(c -> pks.contains(c.getKey()) && !c.getValue().isNullable()); - - final boolean sameColumns = actualColumns.equals(intendedColumns) - && !hasPksWithNonNullConstraint - && "TEXT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID.toUpperCase()).type()) - && "TIMESTAMP_TZ".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()).type()) - && "VARIANT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META.toUpperCase()).type()); - - return sameColumns; - } - - @Override - public String updateTable(final StreamConfig stream, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns(), minRawTimestamp, useExpensiveSaferCasting); String dedupFinalTable = ""; String cdcDeletes = ""; @@ -172,19 +144,7 @@ public String updateTable(final StreamConfig stream, } final String commitRawTable = commitRawTable(stream.id(), minRawTimestamp); - return new StringSubstitutor(Map.of( - "insert_new_records", insertNewRecords, - "dedup_final_table", dedupFinalTable, - "cdc_deletes", cdcDeletes, - "commit_raw_table", commitRawTable)).replace( - """ - BEGIN TRANSACTION; - ${insert_new_records} - ${dedup_final_table} - ${cdc_deletes} - ${commit_raw_table} - COMMIT; - """); + return transactionally(insertNewRecords, dedupFinalTable, cdcDeletes, commitRawTable); } private String extractAndCast(final ColumnId column, final AirbyteType airbyteType, final boolean useTryCast) { @@ -459,39 +419,35 @@ String commitRawTable(final StreamId id, final Optional minRawTimestamp } @Override - public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return new StringSubstitutor(Map.of( + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + final StringSubstitutor substitutor = new StringSubstitutor(Map.of( "final_table", stream.finalTableId(QUOTE), - "tmp_final_table", stream.finalTableId(QUOTE, finalSuffix.toUpperCase()))).replace( - """ - BEGIN TRANSACTION; - DROP TABLE IF EXISTS ${final_table}; - ALTER TABLE ${tmp_final_table} RENAME TO ${final_table}; - COMMIT; - """); + "tmp_final_table", stream.finalTableId(QUOTE, finalSuffix.toUpperCase()))); + return transactionally( + substitutor.replace("DROP TABLE IF EXISTS ${final_table};"), + substitutor.replace("ALTER TABLE ${tmp_final_table} RENAME TO ${final_table};")); } @Override - public String prepareTablesForSoftReset(final StreamConfig stream) { - return String.join("\n", List.of( + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + return concat( createTable(stream, SOFT_RESET_SUFFIX.toUpperCase(), true), - clearLoadedAt(stream.id()))); + clearLoadedAt(stream.id())); } @Override - public String clearLoadedAt(final StreamId streamId) { - return new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) .replace(""" UPDATE ${raw_table_id} SET "_airbyte_loaded_at" = NULL; - """); + """)); } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { // In the SQL below, the v2 values are quoted to preserve their case while the v1 values are // intentionally _not_ quoted. This is to preserve the implicit upper-casing behavior in v1. - return new StringSubstitutor(Map.of( - "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), + return Sql.of(new StringSubstitutor(Map.of( "raw_table_name", streamId.rawTableId(QUOTE), "raw_id", JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, "extracted_at", JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, @@ -502,8 +458,6 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, "v1_raw_table", String.join(".", namespace, tableName))) .replace( """ - CREATE SCHEMA IF NOT EXISTS ${raw_namespace}; - CREATE OR REPLACE TABLE ${raw_table_name} ( "${raw_id}" VARCHAR PRIMARY KEY, "${extracted_at}" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), @@ -520,7 +474,7 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, FROM ${v1_raw_table} ) ; - """); + """)); } /** @@ -564,8 +518,4 @@ public static String escapeSingleQuotedString(final String str) { .replace("'", "\\'"); } - private static Set getPks(final StreamConfig stream) { - return stream.primaryKey() != null ? stream.primaryKey().stream().map(ColumnId::name).collect(Collectors.toSet()) : Collections.emptySet(); - } - } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java deleted file mode 100644 index 2535d9004b13..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake.typing_deduping; - -import java.util.LinkedHashMap; - -/** - * @param columns Map from column name to type. Type is a plain string because - * {@link net.snowflake.client.jdbc.SnowflakeType} doesn't actually have all the types that - * Snowflake supports. - */ -public record SnowflakeTableDefinition(LinkedHashMap columns) {} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java index aa6eba7f7f96..3226afa58337 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java @@ -4,8 +4,12 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; +import static io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler.*; + import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; import io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils; import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; @@ -15,7 +19,7 @@ import java.util.Optional; import lombok.SneakyThrows; -public class SnowflakeV1V2Migrator extends BaseDestinationV1V2Migrator { +public class SnowflakeV1V2Migrator extends BaseDestinationV1V2Migrator { private final NamingConventionTransformer namingConventionTransformer; @@ -48,18 +52,18 @@ protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamCon } @Override - protected boolean schemaMatchesExpectation(final SnowflakeTableDefinition existingTable, final Collection columns) { + protected boolean schemaMatchesExpectation(final TableDefinition existingTable, final Collection columns) { return CollectionUtils.containsAllIgnoreCase(existingTable.columns().keySet(), columns); } @SneakyThrows @Override - protected Optional getTableIfExists(final String namespace, final String tableName) throws Exception { - // TODO this is mostly copied from SnowflakeDestinationHandler#findExistingTable, we should probably - // reuse this logic + protected Optional getTableIfExists(final String namespace, final String tableName) throws Exception { + // TODO this looks similar to SnowflakeDestinationHandler#findExistingTables, with a twist; + // databaseName not upper-cased and rawNamespace and rawTableName as-is (no uppercase). // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates // VARIANT as VARCHAR - final LinkedHashMap columns = + final LinkedHashMap columns = database.queryJsons( """ SELECT column_name, data_type, is_nullable @@ -75,12 +79,13 @@ protected Optional getTableIfExists(final String names .stream() .collect(LinkedHashMap::new, (map, row) -> map.put(row.get("COLUMN_NAME").asText(), - new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), + new ColumnDefinition(row.get("COLUMN_NAME").asText(), row.get("DATA_TYPE").asText(), 0, + fromIsNullableIsoString(row.get("IS_NULLABLE").asText()))), LinkedHashMap::putAll); if (columns.isEmpty()) { return Optional.empty(); } else { - return Optional.of(new SnowflakeTableDefinition(columns)); + return Optional.of(new TableDefinition(columns)); } } @@ -101,12 +106,4 @@ protected boolean doesValidV1RawTableExist(final String namespace, final String return super.doesValidV1RawTableExist(namespace.toUpperCase(), tableName.toUpperCase()); } - /** - * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't - * know how to use - */ - private boolean fromSnowflakeBoolean(final String input) { - return input.equalsIgnoreCase("yes"); - } - } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java index 9e04ec3b6f22..eef75f86c7bf 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java @@ -9,6 +9,7 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction; @@ -16,6 +17,7 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.sql.SQLException; import java.util.LinkedHashMap; +import java.util.List; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,8 +50,8 @@ public void migrateIfNecessary(final StreamConfig streamConfig) throws Exception streamConfig.id().originalName(), rawNamespace); final boolean syncModeRequiresMigration = streamConfig.destinationSyncMode() != DestinationSyncMode.OVERWRITE; - final boolean existingTableCaseSensitiveExists = findExistingTable_caseSensitive(caseSensitiveStreamId).isPresent(); - final boolean existingTableUppercaseDoesNotExist = !handler.findExistingTable(streamConfig.id()).isPresent(); + final boolean existingTableCaseSensitiveExists = findExistingTable(caseSensitiveStreamId).isPresent(); + final boolean existingTableUppercaseDoesNotExist = findExistingTable(streamConfig.id()).isEmpty(); LOGGER.info( "Checking whether upcasing migration is necessary for {}.{}. Sync mode requires migration: {}; existing case-sensitive table exists: {}; existing uppercased table does not exist: {}", streamConfig.id().originalNamespace(), @@ -87,41 +89,15 @@ private static String escapeIdentifier_caseSensitive(final String identifier) { return identifier.replace("\"", "\"\""); } - // And this was taken from - // https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java - public Optional findExistingTable_caseSensitive(final StreamId id) throws SQLException { + private Optional findExistingTable(final StreamId id) throws SQLException { // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates // VARIANT as VARCHAR - final LinkedHashMap columns = database.queryJsons( - """ - SELECT column_name, data_type, is_nullable - FROM information_schema.columns - WHERE table_catalog = ? - AND table_schema = ? - AND table_name = ? - ORDER BY ordinal_position; - """, - databaseName.toUpperCase(), - id.finalNamespace(), - id.finalName()).stream() - .collect(LinkedHashMap::new, - (map, row) -> map.put( - row.get("COLUMN_NAME").asText(), - new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), - LinkedHashMap::putAll); - if (columns.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(new SnowflakeTableDefinition(columns)); + LinkedHashMap> existingTableMap = + SnowflakeDestinationHandler.findExistingTables(database, databaseName, List.of(id)); + if (existingTableMap.containsKey(id.finalNamespace()) && existingTableMap.get(id.finalNamespace()).containsKey(id.finalName())) { + return Optional.of(existingTableMap.get(id.finalNamespace()).get(id.finalName())); } - } - - /** - * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't - * know how to use - */ - private boolean fromSnowflakeBoolean(String input) { - return input.equalsIgnoreCase("yes"); + return Optional.empty(); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index ccdda93dec75..53abb997a8b8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -63,49 +63,10 @@ "description": "", "type": "object", "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "order": 0, - "required": ["access_token", "refresh_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "OAuth2.0", - "enum": ["OAuth2.0"], - "default": "OAuth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "Enter your application's Client ID", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "Enter your application's Client secret", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Enter you application's Access Token", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Enter your application's Refresh Token", - "airbyte_secret": true - } - } - }, { "title": "Key Pair Authentication", "type": "object", - "order": 1, + "order": 0, "required": ["private_key"], "properties": { "auth_type": { @@ -134,7 +95,7 @@ "title": "Username and Password", "type": "object", "required": ["password"], - "order": 2, + "order": 1, "properties": { "auth_type": { "type": "string", @@ -151,6 +112,45 @@ "order": 1 } } + }, + { + "title": "OAuth2.0", + "type": "object", + "order": 2, + "required": ["access_token", "refresh_token"], + "properties": { + "auth_type": { + "type": "string", + "const": "OAuth2.0", + "enum": ["OAuth2.0"], + "default": "OAuth2.0", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "Enter your application's Client ID", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "Enter your application's Client secret", + "airbyte_secret": true + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "Enter you application's Access Token", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Enter your application's Refresh Token", + "airbyte_secret": true + } + } } ], "order": 6 @@ -173,6 +173,13 @@ "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 11 + }, + "enable_incremental_final_table_updates": { + "type": "boolean", + "default": false, + "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", + "title": "Enable Loading Data Incrementally to Final Tables", + "order": 12 } } }, diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java index a7aac9cef7cc..2c502d1c1ac9 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java @@ -107,7 +107,7 @@ protected void globalTeardown() throws Exception { } @Override - protected SqlGenerator getSqlGenerator() { + protected SqlGenerator getSqlGenerator() { return new SnowflakeSqlGenerator(); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java index 60709358a8be..bf204e1909d7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java @@ -22,6 +22,8 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts; import io.airbyte.integrations.destination.snowflake.SnowflakeDatabase; @@ -43,7 +45,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -public class SnowflakeSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { +public class SnowflakeSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { private static String databaseName; private static JdbcDatabase database; @@ -243,7 +245,7 @@ protected Map getFinalMetadataColumnNames() { @Override @Test public void testCreateTableIncremental() throws Exception { - final String sql = generator.createTable(incrementalDedupStream, "", false); + final Sql sql = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(sql); // Note that USERS_FINAL is uppercased here. This is intentional, because snowflake upcases unquoted @@ -366,8 +368,8 @@ protected void migrationAssertions(final List v1RawRecords, final List record -> record.get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText(), Function.identity())); assertAll( - () -> assertEquals(5, v1RawRecords.size()), - () -> assertEquals(5, v2RawRecords.size())); + () -> assertEquals(6, v1RawRecords.size()), + () -> assertEquals(6, v2RawRecords.size())); v1RawRecords.forEach(v1Record -> { final var v1id = v1Record.get(JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase()).asText(); assertAll( @@ -406,23 +408,27 @@ public void ensurePKsAreIndexedUnique() throws Exception { } """))); - final String createTable = generator.createTable(incrementalDedupStream, "", false); + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); // should be OK with new tables destinationHandler.execute(createTable); - final Optional existingTableA = destinationHandler.findExistingTable(streamId); - assertTrue(generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTableA.get())); - destinationHandler.execute("DROP TABLE " + streamId.finalTableId("")); + List initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + assertEquals(1, initialStates.size()); + assertFalse(initialStates.get(0).isSchemaMismatch()); + destinationHandler.execute(Sql.of("DROP TABLE " + streamId.finalTableId(""))); // Hack the create query to add NOT NULLs to emulate the old behavior - final String createTableModified = Arrays.stream(createTable.split(System.lineSeparator())) - .map(line -> !line.contains("CLUSTER") && (line.contains("id1") || line.contains("id2") || line.contains("ID1") || line.contains("ID2")) - ? line.replace(",", " NOT NULL,") - : line) - .collect(Collectors.joining("\r\n")); - destinationHandler.execute(createTableModified); - final Optional existingTableB = destinationHandler.findExistingTable(streamId); - assertFalse(generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTableB.get())); + List> createTableModified = createTable.transactions().stream().map(transaction -> transaction.stream() + .map(statement -> Arrays.stream(statement.split(System.lineSeparator())).map( + line -> !line.contains("CLUSTER") && (line.contains("id1") || line.contains("id2") || line.contains("ID1") || line.contains("ID2")) + ? line.replace(",", " NOT NULL,") + : line) + .collect(joining("\r\n"))) + .toList()).toList(); + destinationHandler.execute(new Sql(createTableModified)); + initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + assertEquals(1, initialStates.size()); + assertTrue(initialStates.get(0).isSchemaMismatch()); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl index 813561b043bf..136fa8a99003 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -2,3 +2,4 @@ {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl index d0c20a410997..575aa338976c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -3,3 +3,4 @@ {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} // Invalid columns are nulled out (i.e. SQL null, not JSON null) {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl index 75901736b545..d1c3045997b3 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -4,3 +4,4 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} // Invalid data is still allowed in the raw table. {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl index 540aaf560412..67171fa4c01b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -2,6 +2,7 @@ {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "New York", "state": "NY"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl index 9bd9f65927d8..2f7a58c51499 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,4 @@ {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} // Delete Bob, keep Charlie {"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl index b9a53cffb59a..d4bd6c49d4e7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -3,6 +3,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} // And append the records from the second sync {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index b38d23d4e823..f7bffd258123 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -4,3 +4,4 @@ {"ID1": 4, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "UNKNOWN": null, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}} // Note: no loss of precision on these numbers. A naive float64 conversion would yield 67.17411800000001. {"ID1": 5, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "NUMBER": 67.174118, "STRUCT": {"nested_number": 67.174118}, "ARRAY": [67.174118], "UNKNOWN": 67.174118, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} +{"ID1": 6, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "IAMACASESENSITIVECOLUMNNAME": "Case senstive value", "_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META":{"errors":[]}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index 75553fdd9997..e5909080bd83 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -3,3 +3,4 @@ {"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} {"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java index 06374e1fe613..5ab8d85e6c57 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java @@ -49,7 +49,6 @@ class SnowflakeSqlOperationsThrowConfigExceptionTest { private static Executable createStageIfNotExists; private static Executable dropStageIfExists; - private static Executable cleanUpStage; private static Executable copyIntoTableFromStage; private static Executable createSchemaIfNotExists; @@ -65,8 +64,7 @@ public static void setup() { snowflakeSqlOperations = new SnowflakeSqlOperations(); createStageIfNotExists = () -> snowflakeStagingSqlOperations.createStageIfNotExists(dbForExecuteQuery, STAGE_NAME); - dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME); - cleanUpStage = () -> snowflakeStagingSqlOperations.cleanUpStage(dbForExecuteQuery, STAGE_NAME, FILE_PATH); + dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME, null); copyIntoTableFromStage = () -> snowflakeStagingSqlOperations.copyIntoTableFromStage(dbForExecuteQuery, STAGE_NAME, STAGE_PATH, FILE_PATH, TABLE_NAME, SCHEMA_NAME); @@ -84,9 +82,6 @@ private static Stream testArgumentsForDbExecute() { Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, dropStageIfExists), Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, dropStageIfExists), Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, dropStageIfExists), - Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, cleanUpStage), - Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, cleanUpStage), - Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, cleanUpStage), Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, copyIntoTableFromStage), Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, copyIntoTableFromStage), Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, copyIntoTableFromStage), diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 91738294ff4e..de975748835f 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg @@ -10,12 +10,11 @@ data: name: Teradata Vantage registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata - supportsDbt: true tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 37fd84a973a6..55aa93c237b4 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -49,6 +49,10 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest protected static final String CA_CERT_KEY = "ssl_ca_certificate"; + protected static final String ENCRYPTDATA = "ENCRYPTDATA"; + + protected static final String ENCRYPTDATA_ON = "ON"; + public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); } @@ -57,6 +61,12 @@ public TeradataDestination() { super(DRIVER_CLASS, new StandardNameTransformer(), new TeradataSqlOperations()); } + private static void createCertificateFile(String fileName, String fileValue) throws IOException { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { final Map additionalParameters = new HashMap<>(); @@ -69,15 +79,10 @@ protected Map getDefaultConnectionProperties(final JsonNode conf additionalParameters.put(PARAM_SSLMODE, REQUIRE); } } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); return additionalParameters; } - private static void createCertificateFile(String fileName, String fileValue) throws IOException { - try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { - out.print(fileValue); - } - } - private Map obtainConnectionOptions(final JsonNode encryption) { final Map additionalParameters = new HashMap<>(); if (!encryption.isNull()) { diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 85cf7dc27d53..55522de02b66 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -107,10 +107,10 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE SET TABLE %s.%s, FALLBACK ( \n" + "%s VARCHAR(256), \n" + "%s JSON, \n" + "%s TIMESTAMP(6) \n" - + ");\n", + "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " + + " UNIQUE PRIMARY INDEX (%s) ", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID); } @Override diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index c3fa9274ad98..ea6969b8c440 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -179,6 +179,12 @@ public void testSecondSync() { // overrides test in coming releases } + @Override + @Test + public void testCustomDbtTransformations() throws Exception { + // overrides test in coming releases + } + protected DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = destination.toJdbcConfig(config); return DataSourceFactory.create(jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), diff --git a/airbyte-integrations/connectors/destination-typesense/Dockerfile b/airbyte-integrations/connectors/destination-typesense/Dockerfile index f5036e89ab0c..0a9d49772a9f 100644 --- a/airbyte-integrations/connectors/destination-typesense/Dockerfile +++ b/airbyte-integrations/connectors/destination-typesense/Dockerfile @@ -34,5 +34,5 @@ COPY destination_typesense ./destination_typesense ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-typesense diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py index 5e4de404d2af..0f03edf9fe7a 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -3,9 +3,9 @@ # -from logging import Logger from typing import Any, Iterable, Mapping +from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from destination_typesense.writer import TypesenseWriter @@ -38,18 +38,20 @@ def write( pass client.collections.create({"name": steam_name, "fields": [{"name": ".*", "type": "auto"}]}) - writer = TypesenseWriter(client, steam_name, config.get("batch_size")) - for message in input_messages: - if message.type == Type.STATE: - writer.flush() - yield message - elif message.type == Type.RECORD: - writer.queue_write_operation(message.record.data) - else: - continue - writer.flush() - - def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + writer = TypesenseWriter(client, config.get("batch_size")) + for message in input_messages: + if message.type == Type.STATE: + writer.flush() + yield message + elif message.type == Type.RECORD: + record = message.record + writer.queue_write_operation(record.stream, record.data) + else: + continue + writer.flush() + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + logger.debug("TypeSense Destination Config Check") try: client = get_client(config=config) client.collections.create({"name": "_airbyte", "fields": [{"name": "title", "type": "string"}]}) diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index fd9c0e3b5868..54e85d5512b7 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from collections import defaultdict from collections.abc import Mapping from logging import getLogger from uuid import uuid4 @@ -12,17 +13,15 @@ class TypesenseWriter: - write_buffer = [] + write_buffer: list[tuple[str, Mapping]] = [] - def __init__(self, client: Client, steam_name: str, batch_size: int = None): + def __init__(self, client: Client, batch_size: int = None): self.client = client - self.steam_name = steam_name self.batch_size = batch_size or 10000 - def queue_write_operation(self, data: Mapping): + def queue_write_operation(self, stream_name: str, data: Mapping): random_key = str(uuid4()) - data_with_id = data if "id" in data else {**data, "id": random_key} - self.write_buffer.append(data_with_id) + self.write_buffer.append((stream_name, {**data, "id": random_key})) if len(self.write_buffer) == self.batch_size: self.flush() @@ -31,5 +30,11 @@ def flush(self): if buffer_size == 0: return logger.info(f"flushing {buffer_size} records") - self.client.collections[self.steam_name].documents.import_(self.write_buffer) + + grouped_by_stream: defaultdict[str, list[Mapping]] = defaultdict(list) + for stream, data in self.write_buffer: + grouped_by_stream[stream].append(data) + + for (stream, data) in grouped_by_stream.items(): + self.client.collections[stream].documents.import_(data) self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py index cb0d5aae3145..d23bf25f78a1 100644 --- a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py @@ -3,10 +3,10 @@ # import json -from logging import getLogger from typing import Any, Dict, Mapping import pytest +from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -33,13 +33,13 @@ def config_fixture() -> Mapping[str, Any]: def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"col1": {"type": "str"}, "col2": {"type": "integer"}}} - overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="_airbyte", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + overwrite_stream = lambda n: ConfiguredAirbyteStream( + stream=AirbyteStream(name=f"_airbyte_{n}", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) - return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + return ConfiguredAirbyteCatalog(streams=[overwrite_stream(i) for i in range(2)]) @pytest.fixture(autouse=True) @@ -60,12 +60,12 @@ def client_fixture(config) -> Client: def test_check_valid_config(config: Mapping): - outcome = DestinationTypesense().check(getLogger("airbyte"), config) + outcome = DestinationTypesense().check(AirbyteLogger(), config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(): - outcome = DestinationTypesense().check(getLogger("airbyte"), {"api_key": "not_a_real_key", "host": "https://www.fake.com"}) + outcome = DestinationTypesense().check(AirbyteLogger(), {"api_key": "not_a_real_key", "host": "https://www.fake.com"}) assert outcome.status == Status.FAILED @@ -79,17 +79,18 @@ def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: ) -def records_count(client: Client) -> int: - documents_results = client.index("_airbyte").get_documents() - return documents_results.total +def collection_size(client: Client, stream: str) -> int: + collection = client.collections[stream].retrieve() + return collection["num_documents"] def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: Client): - overwrite_stream = configured_catalog.streams[0].stream.name + configured_streams = list(map(lambda s: s.stream.name, configured_catalog.streams)) first_state_message = _state({"state": "1"}) - first_record_chunk = [_record(overwrite_stream, str(i), i) for i in range(2)] + first_record_chunk = [_record(stream, str(i), i) for i, stream in enumerate(configured_streams)] destination = DestinationTypesense() list(destination.write(config, configured_catalog, [*first_record_chunk, first_state_message])) - collection = client.collections["_airbyte"].retrieve() - assert collection["num_documents"] == 2 + + for stream in configured_streams: + assert collection_size(client, stream) == 1 diff --git a/airbyte-integrations/connectors/destination-typesense/metadata.yaml b/airbyte-integrations/connectors/destination-typesense/metadata.yaml index b3ee53fe4fee..4b9a9942aa1b 100644 --- a/airbyte-integrations/connectors/destination-typesense/metadata.yaml +++ b/airbyte-integrations/connectors/destination-typesense/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 36be8dc6-9851-49af-b776-9d4c30e4ab6a - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.3 dockerRepository: airbyte/destination-typesense githubIssueLabel: destination-typesense icon: typesense.svg diff --git a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py index ba065cb9fc02..a14d7f5b2abf 100644 --- a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py @@ -9,32 +9,32 @@ @patch("typesense.Client") def test_default_batch_size(client): - writer = TypesenseWriter(client, "steam_name") + writer = TypesenseWriter(client) assert writer.batch_size == 10000 @patch("typesense.Client") def test_empty_batch_size(client): - writer = TypesenseWriter(client, "steam_name", "") + writer = TypesenseWriter(client, "") assert writer.batch_size == 10000 @patch("typesense.Client") def test_custom_batch_size(client): - writer = TypesenseWriter(client, "steam_name", 9000) + writer = TypesenseWriter(client, 9000) assert writer.batch_size == 9000 @patch("typesense.Client") def test_queue_write_operation(client): - writer = TypesenseWriter(client, "steam_name") - writer.queue_write_operation({"a": "a"}) + writer = TypesenseWriter(client) + writer.queue_write_operation("stream_name", {"a": "a"}) assert len(writer.write_buffer) == 1 @patch("typesense.Client") def test_flush(client): - writer = TypesenseWriter(client, "steam_name") - writer.queue_write_operation({"a": "a"}) + writer = TypesenseWriter(client) + writer.queue_write_operation("stream_name", {"a": "a"}) writer.flush() - client.collections.__getitem__.assert_called_once_with("steam_name") + client.collections.__getitem__.assert_called_once_with("stream_name") diff --git a/airbyte-integrations/connectors/destination-vectara/.dockerignore b/airbyte-integrations/connectors/destination-vectara/.dockerignore new file mode 100644 index 000000000000..f784000e19e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_vectara +!setup.py diff --git a/airbyte-integrations/connectors/destination-vectara/Dockerfile b/airbyte-integrations/connectors/destination-vectara/Dockerfile new file mode 100644 index 000000000000..9afa4fa81a36 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY destination_vectara ./destination_vectara + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.name=airbyte/destination-vectara diff --git a/airbyte-integrations/connectors/destination-vectara/README.md b/airbyte-integrations/connectors/destination-vectara/README.md new file mode 100644 index 000000000000..2c68229551bc --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/README.md @@ -0,0 +1,123 @@ +# Vectara Destination + +This is the repository for the Vectara destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/vectara). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/vectara) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_vectara/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination vectara test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-vectara:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-vectara:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-vectara:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-vectara:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py new file mode 100644 index 000000000000..1bc53911e4ef --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationVectara + +__all__ = ["DestinationVectara"] diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py new file mode 100644 index 000000000000..755d30014780 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py @@ -0,0 +1,199 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import json +import traceback +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Mapping + +import backoff +import requests +from destination_vectara.config import VectaraConfig + +METADATA_STREAM_FIELD = "_ab_stream" + + +def user_error(e: Exception) -> bool: + """ + Return True if this exception is caused by user error, False otherwise. + """ + if not isinstance(e, requests.exceptions.RequestException): + return False + return bool(e.response and 400 <= e.response.status_code < 500) + + +class VectaraClient: + + BASE_URL = "https://api.vectara.io/v1" + + def __init__(self, config: VectaraConfig): + if isinstance(config, dict): + config = VectaraConfig.parse_obj(config) + self.customer_id = config.customer_id + self.corpus_name = config.corpus_name + self.client_id = config.oauth2.client_id + self.client_secret = config.oauth2.client_secret + self.parallelize = config.parallelize + self.check() + + def check(self): + """ + Check for an existing corpus in Vectara. + If more than one exists - then return a message + If exactly one exists with this name - ensure that the corpus has the correct metadata fields, and use it. + If not, create it. + """ + try: + jwt_token = self._get_jwt_token() + if not jwt_token: + return "Unable to get JWT Token. Confirm your Client ID and Client Secret." + + list_corpora_response = self._request(endpoint="list-corpora", data={"numResults": 100, "filter": self.corpus_name}) + possible_corpora_ids_names_map = { + corpus.get("id"): corpus.get("name") + for corpus in list_corpora_response.get("corpus") + if corpus.get("name") == self.corpus_name + } + if len(possible_corpora_ids_names_map) > 1: + return f"Multiple Corpora exist with name {self.corpus_name}" + if len(possible_corpora_ids_names_map) == 1: + self.corpus_id = list(possible_corpora_ids_names_map.keys())[0] + else: + data = { + "corpus": { + "name": self.corpus_name, + "filterAttributes": [ + { + "name": METADATA_STREAM_FIELD, + "indexed": True, + "type": "FILTER_ATTRIBUTE_TYPE__TEXT", + "level": "FILTER_ATTRIBUTE_LEVEL__DOCUMENT", + }, + ], + } + } + + create_corpus_response = self._request(endpoint="create-corpus", data=data) + self.corpus_id = create_corpus_response.get("corpusId") + + except Exception as e: + return str(e) + "\n" + "".join(traceback.TracebackException.from_exception(e).format()) + + def _get_jwt_token(self): + """Connect to the server and get a JWT token.""" + token_endpoint = f"https://vectara-prod-{self.customer_id}.auth.us-west-2.amazoncognito.com/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"grant_type": "client_credentials", "client_id": self.client_id, "client_secret": self.client_secret} + + request_time = datetime.datetime.now().timestamp() + response = requests.request(method="POST", url=token_endpoint, headers=headers, data=data) + response_json = response.json() + + self.jwt_token = response_json.get("access_token") + self.jwt_token_expires_ts = request_time + response_json.get("expires_in") + return self.jwt_token + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) + def _request(self, endpoint: str, http_method: str = "POST", params: Mapping[str, Any] = None, data: Mapping[str, Any] = None): + + url = f"{self.BASE_URL}/{endpoint}" + + current_ts = datetime.datetime.now().timestamp() + if self.jwt_token_expires_ts - current_ts <= 60: + self._get_jwt_token() + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + "customer-id": self.customer_id, + "X-source": "airbyte", + } + + response = requests.request(method=http_method, url=url, headers=headers, params=params, data=json.dumps(data)) + response.raise_for_status() + return response.json() + + def delete_doc_by_metadata(self, metadata_field_name, metadata_field_values): + document_ids = [] + for value in metadata_field_values: + data = { + "query": [ + { + "query": "", + "numResults": 100, + "corpusKey": [ + { + "customerId": self.customer_id, + "corpusId": self.corpus_id, + "metadataFilter": f"doc.{metadata_field_name} = '{value}'", + } + ], + } + ] + } + query_documents_response = self._request(endpoint="query", data=data) + document_ids.extend([document.get("id") for document in query_documents_response.get("responseSet")[0].get("document")]) + self.delete_docs_by_id(document_ids=document_ids) + + def delete_docs_by_id(self, document_ids): + for document_id in document_ids: + self._request( + endpoint="delete-doc", data={"customerId": self.customer_id, "corpusId": self.corpus_id, "documentId": document_id} + ) + + def index_document(self, document): + document_section, document_metadata, document_title, document_id = document + if len(document_section) == 0: + return None # Document is empty, so skip it + document_metadata = self._normalize(document_metadata) + data = { + "customerId": self.customer_id, + "corpusId": self.corpus_id, + "document": { + "documentId": document_id, + "metadataJson": json.dumps(document_metadata), + "title": document_title, + "section": [ + {"text": f"{section_key}: {section_value}"} + for section_key, section_value in document_section.items() + if section_key != METADATA_STREAM_FIELD + ], + }, + } + index_document_response = self._request(endpoint="index", data=data) + return index_document_response + + def index_documents(self, documents): + if self.parallelize: + with ThreadPoolExecutor() as executor: + futures = [executor.submit(self.index_document, doc) for doc in documents] + for future in futures: + try: + response = future.result() + if response is None: + continue + assert ( + response.get("status").get("code") == "OK" + or response.get("status").get("statusDetail") == "Document should have at least one part." + ) + except AssertionError as e: + # Handle the assertion error + pass + else: + for doc in documents: + self.index_document(doc) + + def _normalize(self, metadata: dict) -> dict: + result = {} + for key, value in metadata.items(): + if isinstance(value, (str, int, float, bool)): + result[key] = value + else: + # JSON encode all other types + result[key] = json.dumps(value) + return result diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py new file mode 100644 index 000000000000..86ca2dba16f5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs +from pydantic import BaseModel, Field + + +class OAuth2(BaseModel): + client_id: str = Field(..., title="OAuth Client ID", description="OAuth2.0 client id", order=0) + client_secret: str = Field(..., title="OAuth Client Secret", description="OAuth2.0 client secret", airbyte_secret=True, order=1) + + class Config: + title = "OAuth2.0 Credentials" + schema_extra = { + "description": "OAuth2.0 credentials used to authenticate admin actions (creating/deleting corpora)", + "group": "auth", + } + + +class VectaraConfig(BaseModel): + oauth2: OAuth2 + customer_id: str = Field( + ..., title="Customer ID", description="Your customer id as it is in the authenticaion url", order=2, group="account" + ) + corpus_name: str = Field(..., title="Corpus Name", description="The Name of Corpus to load data into", order=3, group="account") + + parallelize: Optional[bool] = Field( + default=False, + title="Parallelize", + description="Parallelize indexing into Vectara with multiple threads", + always_show=True, + group="account", + ) + + text_fields: Optional[List[str]] = Field( + default=[], + title="Text fields to index with Vectara", + description="List of fields in the record that should be in the section of the document. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + always_show=True, + examples=["text", "user.name", "users.*.name"], + ) + title_field: Optional[str] = Field( + default="", + title="Text field to use as document title with Vectara", + description="A field that will be used to populate the `title` of each document. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + always_show=True, + examples=["document_key"], + ) + metadata_fields: Optional[List[str]] = Field( + default=[], + title="Fields to store as metadata", + description="List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + always_show=True, + examples=["age", "user"], + ) + + class Config: + title = "Vectara Config" + schema_extra = { + "description": "Configuration to connect to the Vectara instance", + "groups": [ + {"id": "account", "title": "Account"}, + {"id": "auth", "title": "Authentication"}, + ], + } + + @classmethod + def schema(cls): + """we're overriding the schema classmethod to enable some post-processing""" + schema = super().schema() + schema = resolve_refs(schema) + return schema diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py new file mode 100644 index 000000000000..6a580655ff91 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + DestinationSyncMode, + Status, + Type, +) +from destination_vectara.client import VectaraClient +from destination_vectara.config import VectaraConfig +from destination_vectara.writer import VectaraWriter + + +class DestinationVectara(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + + """ + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + + config_model = VectaraConfig.parse_obj(config) + writer = VectaraWriter( + client=VectaraClient(config_model), + text_fields=config_model.text_fields, + title_field=config_model.title_field, + metadata_fields=config_model.metadata_fields, + catalog=configured_catalog, + ) + + writer.delete_streams_to_overwrite(catalog=configured_catalog) + + for message in input_messages: + if message.type == Type.STATE: + # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush + # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state + writer.flush() + yield message + elif message.type == Type.RECORD: + record = message.record + writer.queue_write_operation(record) + else: + # ignore other message types for now + continue + + # Make sure to flush any records still in the queue + writer.flush() + + def check(self, logger: AirbyteLogger, config: VectaraConfig) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + client = VectaraClient(config=config) + client_error = client.check() + if client_error: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join([client_error])) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/vectara", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append], + connectionSpecification=VectaraConfig.schema(), + ) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py new file mode 100644 index 000000000000..0794b0dc9410 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Any, Dict, List, Mapping, Optional + +import dpath.util +from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from destination_vectara.client import VectaraClient + +METADATA_STREAM_FIELD = "_ab_stream" + + +class VectaraWriter: + + write_buffer: List[Mapping[str, Any]] = [] + flush_interval = 1000 + + def __init__( + self, + client: VectaraClient, + text_fields: Optional[List[str]], + title_field: Optional[str], + metadata_fields: Optional[List[str]], + catalog: ConfiguredAirbyteCatalog, + ): + self.client = client + self.text_fields = text_fields + self.title_field = title_field + self.metadata_fields = metadata_fields + self.streams = {f"{stream.stream.namespace}_{stream.stream.name}": stream for stream in catalog.streams} + self.ids_to_delete: List[str] = [] + + def delete_streams_to_overwrite(self, catalog: ConfiguredAirbyteCatalog) -> None: + streams_to_overwrite = [ + f"{stream.stream.namespace}_{stream.stream.name}" + for stream in catalog.streams + if stream.destination_sync_mode == DestinationSyncMode.overwrite + ] + if len(streams_to_overwrite): + self.client.delete_doc_by_metadata(metadata_field_name=METADATA_STREAM_FIELD, metadata_field_values=streams_to_overwrite) + + def _delete_documents_to_dedupe(self): + if len(self.ids_to_delete) > 0: + self.client.delete_docs_by_id(document_ids=self.ids_to_delete) + + def queue_write_operation(self, record: AirbyteRecordMessage) -> None: + """Adds messages to the write queue and flushes if the buffer is full""" + + stream_identifier = self._get_stream_id(record=record) + document_section = self._get_document_section(record=record) + document_metadata = self._get_document_metadata(record=record) + document_title = self._get_document_title(record=record) + primary_key = self._get_record_primary_key(record=record) + + if primary_key: + document_id = f"Stream_{stream_identifier}_Key_{primary_key}" + if self.streams[stream_identifier].destination_sync_mode == DestinationSyncMode.append_dedup: + self.ids_to_delete.append(document_id) + else: + document_id = str(uuid.uuid4().int) + + self.write_buffer.append((document_section, document_metadata, document_title, document_id)) + if len(self.write_buffer) == self.flush_interval: + self.flush() + + def flush(self) -> None: + """Flush all documents in Queue to Vectara""" + self._delete_documents_to_dedupe() + self.client.index_documents(self.write_buffer) + self.write_buffer.clear() + self.ids_to_delete.clear() + + def _get_document_section(self, record: AirbyteRecordMessage): + relevant_fields = self._extract_relevant_fields(record, self.text_fields) + if len(relevant_fields) == 0: + text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields" + raise AirbyteTracedException( + internal_message="No text fields found in record", + message=f"Record {str(record.data)[:250]}... does not contain any of the configured text fields: {text_fields}. Please check your processing configuration, there has to be at least one text field set in each record.", + failure_type=FailureType.config_error, + ) + document_section = relevant_fields + return document_section + + def _extract_relevant_fields(self, record: AirbyteRecordMessage, fields: Optional[List[str]]) -> Dict[str, Any]: + relevant_fields = {} + if fields and len(fields) > 0: + for field in fields: + values = dpath.util.values(record.data, field, separator=".") + if values and len(values) > 0: + relevant_fields[field] = values if len(values) > 1 else values[0] + else: + relevant_fields = record.data + return relevant_fields + + def _get_document_metadata(self, record: AirbyteRecordMessage) -> Dict[str, Any]: + document_metadata = self._extract_relevant_fields(record, self.metadata_fields) + document_metadata[METADATA_STREAM_FIELD] = self._get_stream_id(record) + return document_metadata + + def _get_document_title(self, record: AirbyteRecordMessage) -> str: + title = "Untitled" + if self.title_field: + title = dpath.util.get(record.data, self.title_field) + return title + + def _get_stream_id(self, record: AirbyteRecordMessage) -> str: + return f"{record.namespace}_{record.stream}" + + def _get_record_primary_key(self, record: AirbyteRecordMessage) -> Optional[str]: + stream_identifier = self._get_stream_id(record) + current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier] + + if not current_stream.primary_key: + return None + + primary_key = [] + for key in current_stream.primary_key: + try: + primary_key.append(str(dpath.util.get(record.data, key))) + except KeyError: + primary_key.append("__not_found__") + stringified_primary_key = "_".join(primary_key) + return f"{stream_identifier}_{stringified_primary_key}" diff --git a/airbyte-integrations/connectors/destination-vectara/icon.svg b/airbyte-integrations/connectors/destination-vectara/icon.svg new file mode 100644 index 000000000000..70798dc5f55b --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/icon.svg @@ -0,0 +1,40 @@ + + + + + + diff --git a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py new file mode 100644 index 000000000000..052006303d85 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +import unittest +from typing import Any, Dict + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_vectara.client import VectaraClient +from destination_vectara.destination import DestinationVectara + + +class VectaraIntegrationTest(unittest.TestCase): + def _get_configured_catalog(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="mystream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + ), + primary_key=[["int_col"]], + sync_mode=SyncMode.incremental, + destination_sync_mode=destination_mode, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + def _state(self, data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + def _clean(self): + self._client.delete_doc_by_metadata(metadata_field_name="_ab_stream", metadata_field_values=["None_mystream"]) + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self._client = VectaraClient(self.config) + self._clean() + + def tearDown(self): + self._clean() + + def test_check_valid_config(self): + outcome = DestinationVectara().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + outcome = DestinationVectara().check( + logging.getLogger("airbyte"), + { + "oauth2": {"client_id": "myclientid", "client_secret": "myclientsecret"}, + "corpus_name": "teststore", + "customer_id": "123456", + "text_fields": [], + "metadata_fields": [], + "title_field": "", + }, + ) + assert outcome.status == Status.FAILED + + def _query_index(self, query="Everything", num_results=100): + return self._client._request( + "query", + data={ + "query": [ + { + "query": query, + "numResults": num_results, + "corpusKey": [ + { + "customerId": self._client.customer_id, + "corpusId": self._client.corpus_id, + } + ], + } + ] + }, + )["responseSet"][0] + + def test_write(self): + # validate corpus starts empty + initial_result = self._query_index()["document"] + assert len(initial_result) == 0 + + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationVectara() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert len(self._query_index()["document"]) == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + assert len(self._query_index()["document"]) == 5 + + # use semantic search + result = self._query_index("Feline animals", 1) + assert result["document"] == [ + { + "id": "Stream_None_mystream_Key_None_mystream_2", + "metadata": [ + {"name": "int_col", "value": "2"}, + {"name": "_ab_stream", "value": "None_mystream"}, + {"name": "title", "value": "Cats are nice"}, + ], + } + ] diff --git a/airbyte-integrations/connectors/destination-vectara/main.py b/airbyte-integrations/connectors/destination-vectara/main.py new file mode 100644 index 000000000000..289b411fb318 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_vectara import DestinationVectara + +if __name__ == "__main__": + DestinationVectara().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-vectara/metadata.yaml b/airbyte-integrations/connectors/destination-vectara/metadata.yaml new file mode 100644 index 000000000000..eed0bec69693 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/metadata.yaml @@ -0,0 +1,26 @@ +data: + allowedHosts: + hosts: + - api.vectara.io + - "vectara-prod-${self.customer_id}.auth.us-west-2.amazoncognito.com" + registries: + oss: + enabled: true + cloud: + enabled: true + connectorSubtype: database + connectorType: destination + definitionId: 102900e7-a236-4c94-83e4-a4189b99adc2 + dockerImageTag: 0.2.0 + dockerRepository: airbyte/destination-vectara + githubIssueLabel: destination-vectara + icon: vectara.svg + license: MIT + name: Vectara + releaseDate: 2023-12-16 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/vectara + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amplitude/requirements.txt b/airbyte-integrations/connectors/destination-vectara/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-amplitude/requirements.txt rename to airbyte-integrations/connectors/destination-vectara/requirements.txt diff --git a/airbyte-integrations/connectors/destination-vectara/setup.py b/airbyte-integrations/connectors/destination-vectara/setup.py new file mode 100644 index 000000000000..ab10a8c60fb9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/setup.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk==0.57.8", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_vectara", + description="Destination implementation for Vectara.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-vectara/unit_tests/__init__.py b/airbyte-integrations/connectors/destination-vectara/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py index 6c580102e7c3..c4708d59ffc9 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py @@ -65,6 +65,7 @@ class WeaviateIndexingConfigModel(BaseModel): ) batch_size: int = Field(title="Batch Size", description="The number of records to send to Weaviate in each batch", default=128) text_field: str = Field(title="Text Field", description="The field in the object that contains the embedded text", default="text") + tenant_id: str = Field(title="Tenant ID", description="The tenant ID to use for multi tenancy", airbyte_secret=True, default="") default_vectorizer: str = Field( title="Default Vectorizer", description="The vectorizer to use if new classes need to be created", diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py index 45c54d54bfed..93adb9d825a4 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py @@ -52,6 +52,14 @@ def _create_client(self): batch_size=None, dynamic=False, weaviate_error_retries=weaviate.WeaviateErrorRetryConf(number_retries=5) ) + def _add_tenant_to_class_if_missing(self, class_name: str): + class_tenants = self.client.schema.get_class_tenants(class_name=class_name) + if class_tenants is not None and self.config.tenant_id not in [tenant.name for tenant in class_tenants]: + self.client.schema.add_class_tenants(class_name=class_name, tenants=[weaviate.Tenant(name=self.config.tenant_id)]) + logging.info(f"Added tenant {self.config.tenant_id} to class {class_name}") + else: + logging.info(f"Tenant {self.config.tenant_id} already exists in class {class_name}") + def check(self) -> Optional[str]: deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE and not self._uses_safe_config(): @@ -69,6 +77,11 @@ def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: self._create_client() classes = {c["class"]: c for c in self.client.schema.get().get("classes", [])} self.has_record_id_metadata = defaultdict(lambda: False) + + if self.config.tenant_id.strip(): + for class_name in classes.keys(): + self._add_tenant_to_class_if_missing(class_name) + for stream in catalog.streams: class_name = self._stream_to_class_name(stream.stream.name) schema = classes[class_name] if class_name in classes else None @@ -78,24 +91,29 @@ def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: self.client.schema.create_class(schema) logging.info(f"Recreated class {class_name}") elif class_name not in classes: - self.client.schema.create_class( - { - "class": class_name, - "vectorizer": self.config.default_vectorizer, - "properties": [ - { - # Record ID is used for bookkeeping, not for searching - "name": METADATA_RECORD_ID_FIELD, - "dataType": ["text"], - "description": "Record ID, used for bookkeeping.", - "indexFilterable": True, - "indexSearchable": False, - "tokenization": "field", - } - ], - } - ) + config = { + "class": class_name, + "vectorizer": self.config.default_vectorizer, + "properties": [ + { + # Record ID is used for bookkeeping, not for searching + "name": METADATA_RECORD_ID_FIELD, + "dataType": ["text"], + "description": "Record ID, used for bookkeeping.", + "indexFilterable": True, + "indexSearchable": False, + "tokenization": "field", + } + ], + } + if self.config.tenant_id.strip(): + config["multiTenancyConfig"] = {"enabled": True} + + self.client.schema.create_class(config) logging.info(f"Created class {class_name}") + + if self.config.tenant_id.strip(): + self._add_tenant_to_class_if_missing(class_name) else: self.has_record_id_metadata[class_name] = schema is not None and any( prop.get("name") == METADATA_RECORD_ID_FIELD for prop in schema.get("properties", {}) @@ -105,10 +123,18 @@ def delete(self, delete_ids, namespace, stream): if len(delete_ids) > 0: class_name = self._stream_to_class_name(stream) if self.has_record_id_metadata[class_name]: - self.client.batch.delete_objects( - class_name=class_name, - where={"path": [METADATA_RECORD_ID_FIELD], "operator": "ContainsAny", "valueStringArray": delete_ids}, - ) + where_filter = {"path": [METADATA_RECORD_ID_FIELD], "operator": "ContainsAny", "valueStringArray": delete_ids} + if self.config.tenant_id.strip(): + self.client.batch.delete_objects( + class_name=class_name, + tenant=self.config.tenant_id, + where=where_filter, + ) + else: + self.client.batch.delete_objects( + class_name=class_name, + where=where_filter, + ) def index(self, document_chunks, namespace, stream): if len(document_chunks) == 0: @@ -124,7 +150,12 @@ def index(self, document_chunks, namespace, stream): weaviate_object[self.config.text_field] = chunk.page_content object_id = str(uuid.uuid4()) class_name = self._stream_to_class_name(chunk.record.stream) - self.client.batch.add_data_object(weaviate_object, class_name, object_id, vector=chunk.embedding) + if self.config.tenant_id.strip(): + self.client.batch.add_data_object( + weaviate_object, class_name, object_id, vector=chunk.embedding, tenant=self.config.tenant_id + ) + else: + self.client.batch.add_data_object(weaviate_object, class_name, object_id, vector=chunk.embedding) self._flush() def _stream_to_class_name(self, stream_name: str) -> str: diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json index 3923a8851c4d..a5db30c7213d 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json @@ -5,164 +5,6 @@ "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { - "processing": { - "title": "ProcessingConfigModel", - "type": "object", - "properties": { - "chunk_size": { - "title": "Chunk size", - "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", - "minimum": 1, - "maximum": 8191, - "type": "integer" - }, - "chunk_overlap": { - "title": "Chunk overlap", - "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", - "default": 0, - "type": "integer" - }, - "text_fields": { - "title": "Text fields to embed", - "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", - "default": [], - "always_show": true, - "examples": ["text", "user.name", "users.*.name"], - "type": "array", - "items": { "type": "string" } - }, - "metadata_fields": { - "title": "Fields to store as metadata", - "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", - "default": [], - "always_show": true, - "examples": ["age", "user", "user.name"], - "type": "array", - "items": { "type": "string" } - }, - "field_name_mappings": { - "title": "Field name mappings", - "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", - "default": [], - "type": "array", - "items": { - "title": "FieldNameMappingConfigModel", - "type": "object", - "properties": { - "from_field": { - "title": "From field name", - "description": "The field name in the source", - "type": "string" - }, - "to_field": { - "title": "To field name", - "description": "The field name to use in the destination", - "type": "string" - } - }, - "required": ["from_field", "to_field"] - } - }, - "text_splitter": { - "title": "Text splitter", - "description": "Split text fields into chunks based on the specified method.", - "type": "object", - "oneOf": [ - { - "title": "By Separator", - "type": "object", - "properties": { - "mode": { - "title": "Mode", - "default": "separator", - "const": "separator", - "enum": ["separator"], - "type": "string" - }, - "separators": { - "title": "Separators", - "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", - "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], - "type": "array", - "items": { "type": "string" } - }, - "keep_separator": { - "title": "Keep separator", - "description": "Whether to keep the separator in the resulting chunks", - "default": false, - "type": "boolean" - } - }, - "required": ["mode"], - "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." - }, - { - "title": "By Markdown header", - "type": "object", - "properties": { - "mode": { - "title": "Mode", - "default": "markdown", - "const": "markdown", - "enum": ["markdown"], - "type": "string" - }, - "split_level": { - "title": "Split level", - "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", - "default": 1, - "minimum": 1, - "maximum": 6, - "type": "integer" - } - }, - "required": ["mode"], - "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." - }, - { - "title": "By Programming Language", - "type": "object", - "properties": { - "mode": { - "title": "Mode", - "default": "code", - "const": "code", - "enum": ["code"], - "type": "string" - }, - "language": { - "title": "Language", - "description": "Split code in suitable places based on the programming language", - "enum": [ - "cpp", - "go", - "java", - "js", - "php", - "proto", - "python", - "rst", - "ruby", - "rust", - "scala", - "swift", - "markdown", - "latex", - "html", - "sol" - ], - "type": "string" - } - }, - "required": ["language", "mode"], - "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." - } - ] - } - }, - "required": ["chunk_size"], - "group": "processing" - }, "embedding": { "title": "Embedding", "description": "Embedding configuration", @@ -181,8 +23,8 @@ "type": "string" } }, - "required": ["mode"], - "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search." + "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search.", + "required": ["mode"] }, { "title": "Azure OpenAI", @@ -296,8 +138,8 @@ "type": "string" } }, - "required": ["mode"], - "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"] }, { "title": "OpenAI-compatible", @@ -341,6 +183,177 @@ } ] }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { + "type": "string" + } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"] + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"] + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + }, "indexing": { "title": "Indexing", "type": "object", @@ -419,8 +432,8 @@ "type": "string" } }, - "required": ["mode"], - "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", + "required": ["mode"] } ] }, @@ -436,6 +449,13 @@ "default": "text", "type": "string" }, + "tenant_id": { + "title": "Tenant ID", + "description": "The tenant ID to use for multi tenancy", + "airbyte_secret": true, + "default": "", + "type": "string" + }, "default_vectorizer": { "title": "Default Vectorizer", "description": "The vectorizer to use if new classes need to be created", @@ -457,14 +477,20 @@ "description": "Additional HTTP headers to send with every request.", "default": [], "examples": [ - { "header_key": "X-OpenAI-Api-Key", "value": "my-openai-api-key" } + { + "header_key": "X-OpenAI-Api-Key", + "value": "my-openai-api-key" + } ], "type": "array", "items": { "title": "Header", "type": "object", "properties": { - "header_key": { "title": "Header Key", "type": "string" }, + "header_key": { + "title": "Header Key", + "type": "string" + }, "value": { "title": "Header Value", "airbyte_secret": true, @@ -478,21 +504,26 @@ "required": ["host", "auth"], "group": "indexing", "description": "Indexing configuration" - }, - "omit_raw_text": { - "title": "Do not store raw text", - "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", - "default": false, - "group": "advanced", - "type": "boolean" } }, "required": ["embedding", "processing", "indexing"], "groups": [ - { "id": "processing", "title": "Processing" }, - { "id": "embedding", "title": "Embedding" }, - { "id": "indexing", "title": "Indexing" }, - { "id": "advanced", "title": "Advanced" } + { + "id": "processing", + "title": "Processing" + }, + { + "id": "embedding", + "title": "Embedding" + }, + { + "id": "indexing", + "title": "Indexing" + }, + { + "id": "advanced", + "title": "Advanced" + } ] }, "supportsIncremental": true, diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index ac4d45630236..ebd5ba581c6e 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736 - dockerImageTag: 0.2.12 + dockerImageTag: 0.2.15 dockerRepository: airbyte/destination-weaviate documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate githubIssueLabel: destination-weaviate diff --git a/airbyte-integrations/connectors/destination-weaviate/setup.py b/airbyte-integrations/connectors/destination-weaviate/setup.py index 17a91d88d5cf..0a49aa856b74 100644 --- a/airbyte-integrations/connectors/destination-weaviate/setup.py +++ b/airbyte-integrations/connectors/destination-weaviate/setup.py @@ -5,9 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.55.1", "weaviate-client==3.25.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "weaviate-client==3.25.2"] -TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"] +TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker==2.0.1"] setup( name="destination_weaviate", diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py index 043a4b6ea68c..a5b2526e392c 100644 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py @@ -71,6 +71,32 @@ def test_pre_sync_that_creates_class(self, MockClient): } ) + @patch("destination_weaviate.indexer.weaviate.Client") + def test_pre_sync_that_creates_class_with_multi_tenancy_enabled(self, MockClient): + mock_client = Mock() + self.config.tenant_id = "test_tenant" + mock_client.schema.get_class_tenants.return_value = [] + mock_client.schema.get.return_value = {"classes": []} + MockClient.return_value = mock_client + self.indexer.pre_sync(self.mock_catalog) + mock_client.schema.create_class.assert_called_with( + { + "class": "Test", + "multiTenancyConfig": {"enabled": True}, + "vectorizer": "none", + "properties": [ + { + "name": "_ab_record_id", + "dataType": ["text"], + "description": "Record ID, used for bookkeeping.", + "indexFilterable": True, + "indexSearchable": False, + "tokenization": "field", + } + ], + } + ) + @patch("destination_weaviate.indexer.weaviate.Client") def test_pre_sync_that_deletes(self, MockClient): mock_client = Mock() @@ -104,6 +130,19 @@ def test_index_deletes_by_record_id(self): where={"path": ["_ab_record_id"], "operator": "ContainsAny", "valueStringArray": ["some_id", "some_other_id"]}, ) + def test_index_deletes_by_record_id_with_tenant_id(self): + mock_client = Mock() + self.config.tenant_id = "test_tenant" + self.indexer.client = mock_client + self.indexer.has_record_id_metadata = defaultdict(None) + self.indexer.has_record_id_metadata["Test"] = True + self.indexer.delete(["some_id", "some_other_id"], None, "test") + mock_client.batch.delete_objects.assert_called_with( + class_name="Test", + tenant="test_tenant", + where={"path": ["_ab_record_id"], "operator": "ContainsAny", "valueStringArray": ["some_id", "some_other_id"]}, + ) + @patch("destination_weaviate.indexer.weaviate.Client") def test_index_not_delete_no_metadata_field(self, MockClient): mock_client = Mock() @@ -200,31 +239,39 @@ def test_index_flushes_batch_and_normalizes(self): page_content="some_content", embedding=[1, 2, 3], metadata={ - "someField": "some_value", "complex": {"a": [1, 2, 3]}, "UPPERCASE_NAME": "abc", "id": 12, "empty_list": [], - "referral Agency Name": "test1", - "123StartsWithNumber": "test2", - "special&*chars": "test3", - "with spaces": "test4", - "": "test5", - "_startsWithUnderscore": "test6", - "multiple spaces": "test7", - "SpecialCharacters!@#": "test8" - }, + "someField": "some_value", + "complex": {"a": [1, 2, 3]}, + "UPPERCASE_NAME": "abc", + "id": 12, + "empty_list": [], + "referral Agency Name": "test1", + "123StartsWithNumber": "test2", + "special&*chars": "test3", + "with spaces": "test4", + "": "test5", + "_startsWithUnderscore": "test6", + "multiple spaces": "test7", + "SpecialCharacters!@#": "test8", + }, record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), ) self.indexer.index([mock_chunk], None, "test") mock_client.batch.add_data_object.assert_called_with( - {"someField": "some_value", "complex": '{"a": [1, 2, 3]}', "uPPERCASE_NAME": "abc", "text": "some_content", "raw_id": 12, - "referral_Agency_Name": "test1", + { + "someField": "some_value", + "complex": '{"a": [1, 2, 3]}', + "uPPERCASE_NAME": "abc", + "text": "some_content", + "raw_id": 12, + "referral_Agency_Name": "test1", "_123StartsWithNumber": "test2", "specialchars": "test3", "with_spaces": "test4", "_": "test5", "_startsWithUnderscore": "test6", "multiple__spaces": "test7", - "specialCharacters": "test8" - - }, + "specialCharacters": "test8", + }, "Test", ANY, vector=[1, 2, 3], diff --git a/airbyte-integrations/connectors/source-activecampaign/main.py b/airbyte-integrations/connectors/source-activecampaign/main.py index 289325e3c27e..52ba47917495 100644 --- a/airbyte-integrations/connectors/source-activecampaign/main.py +++ b/airbyte-integrations/connectors/source-activecampaign/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_activecampaign import SourceActivecampaign +from source_activecampaign.run import run if __name__ == "__main__": - source = SourceActivecampaign() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml index 7b302f920aa0..e7b926edc0a0 100644 --- a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml +++ b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml @@ -12,6 +12,10 @@ data: icon: activecampaign.svg license: MIT name: ActiveCampaign + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-activecampaign registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-activecampaign/setup.py b/airbyte-integrations/connectors/source-activecampaign/setup.py index d539a3c2757c..3c64964fcef1 100644 --- a/airbyte-integrations/connectors/source-activecampaign/setup.py +++ b/airbyte-integrations/connectors/source-activecampaign/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-activecampaign=source_activecampaign.run:run", + ], + }, name="source_activecampaign", description="Source implementation for Activecampaign.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py new file mode 100644 index 000000000000..adc9dbcc7e05 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_activecampaign import SourceActivecampaign + + +def run(): + source = SourceActivecampaign() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-adjust/main.py b/airbyte-integrations/connectors/source-adjust/main.py index 1639515b23ae..06eebde5d55c 100644 --- a/airbyte-integrations/connectors/source-adjust/main.py +++ b/airbyte-integrations/connectors/source-adjust/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_adjust import SourceAdjust +from source_adjust.run import run if __name__ == "__main__": - source = SourceAdjust() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-adjust/metadata.yaml b/airbyte-integrations/connectors/source-adjust/metadata.yaml index f3a7d7bb290a..f45ead0f39c6 100644 --- a/airbyte-integrations/connectors/source-adjust/metadata.yaml +++ b/airbyte-integrations/connectors/source-adjust/metadata.yaml @@ -12,6 +12,10 @@ data: icon: adjust.svg license: MIT name: Adjust + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-adjust registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-adjust/setup.py b/airbyte-integrations/connectors/source-adjust/setup.py index 99691631d22c..7195646cdfce 100644 --- a/airbyte-integrations/connectors/source-adjust/setup.py +++ b/airbyte-integrations/connectors/source-adjust/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-adjust=source_adjust.run:run", + ], + }, name="source_adjust", description="Source implementation for Adjust.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/run.py b/airbyte-integrations/connectors/source-adjust/source_adjust/run.py new file mode 100644 index 000000000000..db3689a7552d --- /dev/null +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_adjust import SourceAdjust + + +def run(): + source = SourceAdjust() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aha/main.py b/airbyte-integrations/connectors/source-aha/main.py index 79ed7a087f63..b07aafa3db46 100644 --- a/airbyte-integrations/connectors/source-aha/main.py +++ b/airbyte-integrations/connectors/source-aha/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aha import SourceAha +from source_aha.run import run if __name__ == "__main__": - source = SourceAha() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aha/metadata.yaml b/airbyte-integrations/connectors/source-aha/metadata.yaml index cbbecab8c080..88d029de3b5d 100644 --- a/airbyte-integrations/connectors/source-aha/metadata.yaml +++ b/airbyte-integrations/connectors/source-aha/metadata.yaml @@ -12,6 +12,10 @@ data: icon: aha.svg license: MIT name: Aha + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aha registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-aha/setup.py b/airbyte-integrations/connectors/source-aha/setup.py index 30789cbe1a0a..bbcedc901f75 100644 --- a/airbyte-integrations/connectors/source-aha/setup.py +++ b/airbyte-integrations/connectors/source-aha/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aha=source_aha.run:run", + ], + }, name="source_aha", description="Source implementation for Aha.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aha/source_aha/run.py b/airbyte-integrations/connectors/source-aha/source_aha/run.py new file mode 100644 index 000000000000..7a67cc7e710c --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aha import SourceAha + + +def run(): + source = SourceAha() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aircall/main.py b/airbyte-integrations/connectors/source-aircall/main.py index ff8cceaf862c..3a1a1acff9b2 100644 --- a/airbyte-integrations/connectors/source-aircall/main.py +++ b/airbyte-integrations/connectors/source-aircall/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aircall import SourceAircall +from source_aircall.run import run if __name__ == "__main__": - source = SourceAircall() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aircall/metadata.yaml b/airbyte-integrations/connectors/source-aircall/metadata.yaml index 1883e76b2e69..c76243302b10 100644 --- a/airbyte-integrations/connectors/source-aircall/metadata.yaml +++ b/airbyte-integrations/connectors/source-aircall/metadata.yaml @@ -8,6 +8,10 @@ data: icon: aircall.svg license: MIT name: Aircall + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aircall registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-aircall/setup.py b/airbyte-integrations/connectors/source-aircall/setup.py index 25b830a1e3cc..8453ded69fca 100644 --- a/airbyte-integrations/connectors/source-aircall/setup.py +++ b/airbyte-integrations/connectors/source-aircall/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aircall=source_aircall.run:run", + ], + }, name="source_aircall", description="Source implementation for Aircall.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/run.py b/airbyte-integrations/connectors/source-aircall/source_aircall/run.py new file mode 100644 index 000000000000..b6d0e6bb463c --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aircall import SourceAircall + + +def run(): + source = SourceAircall() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-airtable/README.md b/airbyte-integrations/connectors/source-airtable/README.md index d1118541f030..3424957010fc 100644 --- a/airbyte-integrations/connectors/source-airtable/README.md +++ b/airbyte-integrations/connectors/source-airtable/README.md @@ -1,120 +1,55 @@ -# Airtable Source +# Airtable source connector + This is the repository for the Airtable source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/airtable). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/airtable). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** -- Create a base named `users` in your AirTable account. -- Create two tables named `Table 1` and `Table 2` in the `users` base. -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/airtable) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_airtable/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/airtable) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_airtable/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source airtable test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-airtable spec +poetry run source-airtable check --config secrets/config.json +poetry run source-airtable discover --config secrets/config.json +poetry run source-airtable read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-airtable build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-airtable:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-airtable:latest -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-airtable:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-airtable:dev . -# Running the spec command against your patched connector -docker run airbyte/source-airtable:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-airtable:dev spec @@ -123,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-airtable:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-airtable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-airtable test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-airtable test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/airtable.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/airtable.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-airtable/main.py b/airbyte-integrations/connectors/source-airtable/main.py index 61aedaa8b88d..170d6caf75b1 100644 --- a/airbyte-integrations/connectors/source-airtable/main.py +++ b/airbyte-integrations/connectors/source-airtable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_airtable import SourceAirtable +from source_airtable.run import run if __name__ == "__main__": - source = SourceAirtable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-airtable/metadata.yaml b/airbyte-integrations/connectors/source-airtable/metadata.yaml index c722087101dd..fc3d79c11d0e 100644 --- a/airbyte-integrations/connectors/source-airtable/metadata.yaml +++ b/airbyte-integrations/connectors/source-airtable/metadata.yaml @@ -11,13 +11,17 @@ data: connectorSubtype: api connectorType: source definitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212 - dockerImageTag: 4.1.5 + dockerImageTag: 4.1.6 dockerRepository: airbyte/source-airtable documentationUrl: https://docs.airbyte.com/integrations/sources/airtable githubIssueLabel: source-airtable icon: airtable.svg license: MIT name: Airtable + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-airtable registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-airtable/poetry.lock b/airbyte-integrations/connectors/source-airtable/poetry.lock new file mode 100644 index 000000000000..226b83b4d7cf --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/poetry.lock @@ -0,0 +1,1083 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.41" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.41.tar.gz", hash = "sha256:cce614d67872cf66a151e5b72d70f4bf26e2a1ce672c7abfc15a5cb4e45d8429"}, + {file = "airbyte_cdk-0.51.41-py3-none-any.whl", hash = "sha256:bbf82a45d9ec97c4a92b85e3312b327f8060fffec1f7c7ea7dfa720f9adcc13b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "77f1e32173eb6c3117ba3b83b24e6d36db3109cc6ff5c9f292c590998617b18e" diff --git a/airbyte-integrations/connectors/source-airtable/pyproject.toml b/airbyte-integrations/connectors/source-airtable/pyproject.toml new file mode 100644 index 000000000000..abfae85d96bd --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "4.1.6" +name = "source-airtable" +description = "Source implementation for Airtable." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/airtable" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_airtable" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.51.41" + +[tool.poetry.scripts] +source-airtable = "source_airtable.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-airtable/setup.py b/airbyte-integrations/connectors/source-airtable/setup.py deleted file mode 100644 index 2c294e0b0dfb..000000000000 --- a/airbyte-integrations/connectors/source-airtable/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] - -setup( - name="source_airtable", - description="Source implementation for Airtable.", - author="Airbyte", - author_email="anhtuan.nguyen@me.com", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/run.py b/airbyte-integrations/connectors/source-airtable/source_airtable/run.py new file mode 100644 index 000000000000..e993fd6eaa7d --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_airtable import SourceAirtable + + +def run(): + source = SourceAirtable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/main.py b/airbyte-integrations/connectors/source-alpha-vantage/main.py index 422447d19ea6..dcccfe7a535c 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/main.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_alpha_vantage import SourceAlphaVantage +from source_alpha_vantage.run import run if __name__ == "__main__": - source = SourceAlphaVantage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml index d4f4a1a63e0f..2eb0807b3957 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml +++ b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: alpha-vantage.svg license: MIT name: Alpha Vantage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-alpha-vantage registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-alpha-vantage/setup.py b/airbyte-integrations/connectors/source-alpha-vantage/setup.py index bf4ebe78c025..22bb9790ab51 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/setup.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-alpha-vantage=source_alpha_vantage.run:run", + ], + }, name="source_alpha_vantage", description="Source implementation for Alpha Vantage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py new file mode 100644 index 000000000000..fe5a71ac01fb --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_alpha_vantage import SourceAlphaVantage + + +def run(): + source = SourceAlphaVantage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md index db784a8d9651..d94cf866336a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/README.md +++ b/airbyte-integrations/connectors/source-amazon-ads/README.md @@ -1,118 +1,55 @@ -# Amazon Ads Source +# Amazon-Ads source connector -This is the repository for the Amazon Ads source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/amazon-ads). + +This is the repository for the Amazon-Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/amazon-ads). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-ads) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_ads/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/amazon-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_ads/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source amazon-ads test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-amazon-ads spec +poetry run source-amazon-ads check --config secrets/config.json +poetry run source-amazon-ads discover --config secrets/config.json +poetry run source-amazon-ads read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-amazon-ads build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-amazon-ads:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-amazon-ads:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-amazon-ads:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-amazon-ads:dev . -# Running the spec command against your patched connector -docker run airbyte/source-amazon-ads:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-amazon-ads:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-ads:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-amazon-ads test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amazon-ads test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amazon-ads.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/amazon-ads.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 6c89363b6ee0..e3eb72c4b986 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -52,7 +52,7 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: 2.3.1 + disable_for_version: 3.4.3 full_refresh: tests: - config_path: secrets/config.json @@ -64,6 +64,6 @@ acceptance_tests: tests: - spec_path: integration_tests/spec.json backward_compatibility_tests_config: - disable_for_version: "3.2.0" + disable_for_version: 3.4.3 connector_image: airbyte/source-amazon-ads:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 585149544f87..96579cbe995e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -114,6 +114,16 @@ "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "region": { + "type": "string", + "path_in_connector_config": ["region"] + } + } + }, "complete_oauth_output_specification": { "type": "object", "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index 1c292d29e4ca..30a0b6957860 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -2,14 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_ads import SourceAmazonAds -from source_amazon_ads.config_migrations import MigrateStartDate +from source_amazon_ads.run import run if __name__ == "__main__": - source = SourceAmazonAds() - MigrateStartDate.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index cc0993359fb7..c12b2c7341c3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -9,17 +9,21 @@ data: - advertising-api-eu.amazon.com - advertising-api-fe.amazon.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 3.4.2 + dockerImageTag: 4.0.3 dockerRepository: airbyte/source-amazon-ads documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg license: MIT name: Amazon Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-ads registries: cloud: enabled: true @@ -28,6 +32,13 @@ data: releaseStage: generally_available releases: breakingChanges: + 4.0.0: + message: "Streams `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` now have updated schemas." + upgradeDeadline: "2024-01-17" + scopedImpact: + - scopeType: stream + impactedScopes: + ["sponsored_brands_ad_groups", "sponsored_brands_keywords"] 3.0.0: message: Attribution report stream schemas fix. upgradeDeadline: "2023-07-24" diff --git a/airbyte-integrations/connectors/source-amazon-ads/poetry.lock b/airbyte-integrations/connectors/source-amazon-ads/poetry.lock new file mode 100644 index 000000000000..2b1ea1530913 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/poetry.lock @@ -0,0 +1,1113 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.0.tar.gz", hash = "sha256:622f56bd7101493a74f11c33a45a31c251032333989996f137cac8370873c614"}, + {file = "airbyte_cdk-0.62.0-py3-none-any.whl", hash = "sha256:b21330a566b33dbdddde33243eb9855f086ad4272e3585ca626be1225451a3b8"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "responses" +version = "0.23.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, + {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "f7a78e82f7679c556ca51fda798b61e4de3b8b78ac2125427c85dcf24542adf0" diff --git a/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml b/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml new file mode 100644 index 000000000000..0d00d8b9f314 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "4.0.3" +name = "source-amazon-ads" +description = "Source implementation for Amazon Ads." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/amazon-ads" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_amazon_ads" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +requests-oauthlib = "==1.3.1" +airbyte-cdk = "==0.62.0" +pendulum = "==2.1.2" + +[tool.poetry.scripts] +source-amazon-ads = "source_amazon_ads.run:run" + +[tool.poetry.group.dev.dependencies] +responses = "^0.23.1" +freezegun = "^1.2.0" +requests-mock = "^1.9.3" +pytest-mock = "^3.7.0" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt b/airbyte-integrations/connectors/source-amazon-ads/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py deleted file mode 100644 index 7d612fffeaaa..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.7.0", - "jsonschema~=3.2.0", - "responses~=0.23.1", - "freezegun~=1.2.0", -] - -setup( - name="source_amazon_ads", - description="Source implementation for Amazon Ads.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py new file mode 100644 index 000000000000..a8012240de66 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.config_migrations import MigrateStartDate + + +def run(): + source = SourceAmazonAds() + MigrateStartDate.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py index b7c375fefaf7..e7e2fa7cd07c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py @@ -51,7 +51,7 @@ class BrandsAdGroup(CatalogModel): campaignId: Decimal adGroupId: Decimal name: str - bid: int + bid: Decimal keywordId: Decimal keywordText: str nativeLanguageKeyword: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py index 92f2c0ea4892..83e845fb36ed 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py @@ -62,9 +62,15 @@ class DisplayBudgetRuleDetailsPerformanceMeasureCondition(CatalogModel): threshold: Decimal +class DisplayBudgetRuleDetailsRecurrenceIntraDaySchedule(CatalogModel): + startTime: str + endTime: str + + class DisplayBudgetRuleDetailsRecurrence(CatalogModel): type: str daysOfWeek: List[str] = None + intraDaySchedule: List[DisplayBudgetRuleDetailsRecurrenceIntraDaySchedule] = None threshold: Decimal diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 09b4d52205f3..0e703cb4ca3a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -127,6 +127,14 @@ advanced_auth: - auth_type predicate_value: oauth2.0 oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + region: + type: string + path_in_connector_config: + - region complete_oauth_output_specification: type: object additionalProperties: true diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/__init__.py new file mode 100644 index 000000000000..aabf455a3389 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/__init__.py @@ -0,0 +1,11 @@ +from .oauth_request_builder import OAuthRequestBuilder +from .profiles_request_builder import ProfilesRequestBuilder +from .sponsored_brands_request_builder import SponsoredBrandsRequestBuilder +from .attribution_report_request_builder import AttributionReportRequestBuilder +from .sponsored_display_report_request_builder import SponsoredDisplayReportRequestBuilder +from .report_check_status_request_builer import ReportCheckStatusRequestBuilder +from .report_download_request_builder import ReportDownloadRequestBuilder +from .sponsored_products_report_request_builder import SponsoredProductsReportRequestBuilder +from .sponsored_brands_video_report_request_builder import SponsoredBrandsVideoReportRequestBuilder +from .sponsored_brands_report_request_builder import SponsoredBrandsReportRequestBuilder +from .sponsored_brands_report_v3_request_builder import SponsoredBrandsV3ReportRequestBuilder diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py new file mode 100644 index 000000000000..c0c0572b42b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py @@ -0,0 +1,134 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import datetime +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +from source_amazon_ads.streams.attribution_report import BRAND_REFERRAL_BONUS, METRICS_MAP + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class AttributionReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def products_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, start_date: datetime.date, end_date: datetime.date, limit: int = 300 + ) -> "AttributionReportRequestBuilder": + return cls("attribution/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_report_type("PRODUCTS") \ + .with_metrics(METRICS_MAP["PRODUCTS"]) \ + .with_limit(limit) \ + .with_start_date(start_date) \ + .with_end_date(end_date) + + @classmethod + def performance_adgroup_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, start_date: datetime.date, end_date: datetime.date, limit: int = 300 + ) -> "AttributionReportRequestBuilder": + return cls("attribution/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_report_type("PERFORMANCE") \ + .with_metrics(METRICS_MAP["PERFORMANCE"] + [BRAND_REFERRAL_BONUS]) \ + .with_limit(limit) \ + .with_start_date(start_date) \ + .with_end_date(end_date) \ + .with_grouping("ADGROUP") + + @classmethod + def performance_campaign_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, start_date: datetime.date, end_date: datetime.date, limit: int = 300 + ) -> "AttributionReportRequestBuilder": + return cls("attribution/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_report_type("PERFORMANCE") \ + .with_metrics(METRICS_MAP["PERFORMANCE"] + [BRAND_REFERRAL_BONUS]) \ + .with_limit(limit) \ + .with_start_date(start_date) \ + .with_end_date(end_date) \ + .with_grouping("CAMPAIGN") + + @classmethod + def performance_creative_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, start_date: datetime.date, end_date: datetime.date, limit: int = 300 + ) -> "AttributionReportRequestBuilder": + return cls("attribution/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_report_type("PERFORMANCE") \ + .with_metrics(METRICS_MAP["PERFORMANCE"]) \ + .with_limit(limit) \ + .with_start_date(start_date) \ + .with_end_date(end_date) \ + .with_grouping("CREATIVE") + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._cursor_field: Optional[int] = "" + self._end_date: Optional[str] = None + self._grouping: Optional[str] = None + self._limit: Optional[int] = None + self._metrics: Optional[List[str]] = [] + self._report_type: Optional[str] = None + self._start_date: Optional[str] = None + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_type: + body["reportType"] = self._report_type + if self._limit: + body["count"] = self._limit + if self._metrics: + body["metrics"] = ",".join(self._metrics) + if self._start_date: + body["startDate"] = self._start_date + if self._end_date: + body["endDate"] = self._end_date + + body["cursorId"] = self._cursor_field + + if self._grouping: + body["groupBy"] = self._grouping + + return json.dumps(body) + + def with_cursor_field(self, cursor_field: str) -> "AttributionReportRequestBuilder": + self._cursor_field: int = cursor_field + return self + + def with_end_date(self, end_date: datetime.date) -> "AttributionReportRequestBuilder": + self._end_date: str = end_date.isoformat().replace("-", "") + return self + + def with_grouping(self, grouping: str) -> "AttributionReportRequestBuilder": + self._grouping: str = grouping + return self + + def with_limit(self, limit: int) -> "AttributionReportRequestBuilder": + self._limit: int = limit + return self + + def with_metrics(self, metrics: List[str]) -> "AttributionReportRequestBuilder": + self._metrics: str = metrics + return self + + def with_report_type(self, report_type: str) -> "AttributionReportRequestBuilder": + self._report_type: str = report_type + return self + + def with_start_date(self, start_date: datetime.date) -> "AttributionReportRequestBuilder": + self._start_date: str = start_date.isoformat().replace("-", "") + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/base_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/base_request_builder.py new file mode 100644 index 000000000000..e0b58efab6e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/base_request_builder.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import abc +from typing import Any, Dict, Optional + +from airbyte_cdk.test.mock_http import HttpRequest + +from .constants import BASE_URL + + +class AmazonAdsRequestBuilder(abc.ABC): + @property + @abc.abstractmethod + def url(self) -> str: + """""" + + @property + @abc.abstractmethod + def query_params(self) -> Dict[str, Any]: + """""" + + @property + @abc.abstractmethod + def headers(self) -> Dict[str, Any]: + """""" + + @property + @abc.abstractmethod + def request_body(self) -> Optional[str]: + """""" + + def build(self) -> HttpRequest: + return HttpRequest( + url=self.url, + query_params=self.query_params, + headers=self.headers, + body=self.request_body + ) + + +class AmazonAdsBaseRequestBuilder(AmazonAdsRequestBuilder): + def __init__(self, resource: str) -> None: + self._resource: str = resource + self._client_access_token: str = None + self._client_id: str = None + self._profile_id: str = None + + @property + def url(self) -> str: + return f"{BASE_URL}/{self._resource}" + + @property + def headers(self): + return (super().headers or {}) | { + "Amazon-Advertising-API-ClientId": self._client_id, + "Amazon-Advertising-API-Scope": self._profile_id, + "Authorization": f"Bearer {self._client_access_token}", + } + + def with_client_access_token(self, client_access_token: str) -> "AmazonAdsBaseRequestBuilder": + self._client_access_token: str = client_access_token + return self + + def with_client_id(self, client_id: str) -> "AmazonAdsBaseRequestBuilder": + self._client_id: str = client_id + return self + + def with_profile_id(self, profile_id: str) -> "AmazonAdsBaseRequestBuilder": + self._profile_id: str = str(profile_id) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/constants.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/constants.py new file mode 100644 index 000000000000..217e4a8dbb39 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/constants.py @@ -0,0 +1,4 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +BASE_URL = "https://advertising-api.amazon.com" +BASE_OAUTH_URL = "https://api.amazon.com" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/oauth_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/oauth_request_builder.py new file mode 100644 index 000000000000..9038bced8d06 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/oauth_request_builder.py @@ -0,0 +1,49 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional + +from .base_request_builder import AmazonAdsRequestBuilder +from .constants import BASE_OAUTH_URL + + +class OAuthRequestBuilder(AmazonAdsRequestBuilder): + @classmethod + def oauth_endpoint(cls, client_id: str, client_secred: str, refresh_token: str) -> "OAuthRequestBuilder": + return cls("auth/o2/token") \ + .with_client_id(client_id) \ + .with_client_secret(client_secred) \ + .with_refresh_token(refresh_token) + + def __init__(self, resource: str) -> None: + self._resource: str = resource + self._client_id: str = None + self._client_secret: str = None + self._refresh_token: str = None + + @property + def url(self) -> str: + return f"{BASE_OAUTH_URL}/{self._resource}" + + @property + def query_params(self) -> Dict[str, Any]: + return {} + + @property + def headers(self) -> Dict[str, Any]: + return {} + + @property + def request_body(self) -> Optional[str]: + return f"grant_type=refresh_token&client_id={self._client_id}&client_secret={self._client_secret}&refresh_token={self._refresh_token}" + + def with_client_id(self, client_id: str) -> "OAuthRequestBuilder": + self._client_id: str = client_id + return self + + def with_client_secret(self, client_secret: str) -> "OAuthRequestBuilder": + self._client_secret: str = client_secret + return self + + def with_refresh_token(self, refresh_token: str) -> "OAuthRequestBuilder": + self._refresh_token: str = refresh_token + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/profiles_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/profiles_request_builder.py new file mode 100644 index 000000000000..5c5fb98fef7a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/profiles_request_builder.py @@ -0,0 +1,55 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List, Optional + +from .base_request_builder import AmazonAdsRequestBuilder +from .constants import BASE_URL + + +class ProfilesRequestBuilder(AmazonAdsRequestBuilder): + @classmethod + def profiles_endpoint(cls, client_id: str, client_access_token: str) -> "ProfilesRequestBuilder": + return cls("v2/profiles") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_type_filter(["seller", "vendor"]) + + def __init__(self, resource: str) -> None: + self._resource: str = resource + self._client_id: str = None + self._client_access_token: str = None + self._profile_type_filter: Optional[List[str]] = None + + @property + def url(self) -> str: + url = f"{BASE_URL}/{self._resource}" + if self._profile_type_filter: + url = f"{url}?profileTypeFilter={','.join(self._profile_type_filter)}" + return url + + @property + def query_params(self) -> Dict[str, Any]: + return {} + + @property + def headers(self) -> Dict[str, Any]: + return { + "Amazon-Advertising-API-ClientId": self._client_id, + "Authorization": f"Bearer {self._client_access_token}", + } + + @property + def request_body(self) -> Optional[str]: + return None + + def with_profile_type_filter(self, profile_type_filter: List[str]) -> "ProfilesRequestBuilder": + self._profile_type_filter: List[str] = profile_type_filter + return self + + def with_client_id(self, client_id: str) -> "ProfilesRequestBuilder": + self._client_id: str = client_id + return self + + def with_client_access_token(self, client_access_token: str) -> "ProfilesRequestBuilder": + self._client_access_token: str = client_access_token + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_check_status_request_builer.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_check_status_request_builer.py new file mode 100644 index 000000000000..fcbd1ed257fc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_check_status_request_builer.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class ReportCheckStatusRequestBuilder(AmazonAdsBaseRequestBuilder): + + @classmethod + def check_v2_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls(f"v2/reports/{report_id}") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) + + @classmethod + def check_v3_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls(f"reporting/reports/{report_id}") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) + + @classmethod + def check_sponsored_display_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls.check_v2_report_status_endpoint(client_id, client_access_token, profile_id, report_id) + + @classmethod + def check_sponsored_brands_video_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls.check_v2_report_status_endpoint(client_id, client_access_token, profile_id, report_id) + + @classmethod + def check_sponsored_brands_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls.check_v2_report_status_endpoint(client_id, client_access_token, profile_id, report_id) + + @classmethod + def check_sponsored_products_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls.check_v3_report_status_endpoint(client_id, client_access_token, profile_id, report_id) + + @classmethod + def check_sponsored_brands_v3_report_status_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_id: str + ) -> "ReportCheckStatusRequestBuilder": + return cls.check_v3_report_status_endpoint(client_id, client_access_token, profile_id, report_id) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + return None diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_download_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_download_request_builder.py new file mode 100644 index 000000000000..3e234bc587a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/report_download_request_builder.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from .base_request_builder import AmazonAdsRequestBuilder + + +class ReportDownloadRequestBuilder(AmazonAdsRequestBuilder): + @classmethod + def download_endpoint(cls, report_id: str) -> "ReportDownloadRequestBuilder": + return cls(report_id) + + def __init__(self, report_id: str) -> None: + self._report_id: str = report_id + + @property + def url(self): + return ( + f"https://offline-report-storage-us-east-1-prod.s3.amazonaws.com" + f"/{self._report_id}/{self._report_id}.json" + ) + + @property + def headers(self): + return None + + @property + def query_params(self): + return None + + @property + def request_body(self): + return None diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_request_builder.py new file mode 100644 index 000000000000..c81423442747 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_request_builder.py @@ -0,0 +1,66 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +import pendulum + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredBrandsReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def _init_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_type: str, metrics: List[str], report_date: Optional[str] = None + ) -> "SponsoredBrandsReportRequestBuilder": + return cls(f"v2/hsa/{report_type}/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_metrics(metrics) \ + .with_report_date(report_date) + + @classmethod + def init_campaigns_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "campaigns", report_date, metrics) + + @classmethod + def init_ad_groups_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "adGroups", report_date, metrics) + + @classmethod + def init_keywords_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "keywords", report_date, metrics) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._metrics: List[str] = None + self._report_date: str = None + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_date: + body["reportDate"] = self._report_date + if self._metrics: + body["metrics"] = self._metrics + return json.dumps(body) + + def with_report_date(self, report_date: pendulum.date) -> "SponsoredBrandsReportRequestBuilder": + self._report_date = report_date.format("YYYYMMDD") + return self + + def with_metrics(self, metrics: List[str]) -> "SponsoredBrandsReportRequestBuilder": + self._metrics = ",".join(metrics) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py new file mode 100644 index 000000000000..f8ddae54d5e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py @@ -0,0 +1,101 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +import pendulum + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredBrandsV3ReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def _init_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_type: str, metrics: List[str], report_date: Optional[str] = None + ) -> "SponsoredBrandsV3ReportRequestBuilder": + return cls(f"reporting/reports") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_metrics(metrics) \ + .with_report_date(report_date) \ + .with_report_type(report_type) + + @classmethod + def init_purchased_asin_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsV3ReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "purchasedAsin", report_date, metrics) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._metrics: List[str] = None + self._report_date: str = None + self._report_type: str = None + + @property + def _report_config_group_by(self) -> List[str]: + return { + "purchasedAsin": ["purchasedAsin"], + }[self._report_type] + + @property + def _report_config_report_type_id(self) -> str: + return { + "purchasedAsin": "sbPurchasedProduct", + }[self._report_type] + + @property + def _report_config_filters(self) -> List[str]: + return { + "purchasedAsin": [], + }[self._report_type] + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_type and self._report_date: + body["name"] = f"{self._report_type} report {self._report_date}" + + if self._report_date: + body["startDate"] = self._report_date + body["endDate"] = self._report_date + + if self._report_type: + body["configuration"] = { + "adProduct": "SPONSORED_BRANDS", + "groupBy": self._report_config_group_by + } + + if self._metrics: + body["configuration"]["columns"] = self._metrics + + if self._report_type: + body["configuration"]["reportTypeId"] = self._report_config_report_type_id + body["configuration"]["filters"] = self._report_config_filters + + body["configuration"]["timeUnit"] = "SUMMARY" + body["configuration"]["format"] = "GZIP_JSON" + + return json.dumps(body) + + def with_report_date(self, report_date: pendulum.date) -> "SponsoredBrandsV3ReportRequestBuilder": + self._report_date = report_date.format("YYYY-MM-DD") + return self + + def with_report_type(self, report_type: str) -> "SponsoredBrandsV3ReportRequestBuilder": + self._report_type = report_type + return self + + def with_tactics(self, tactics: str) -> "SponsoredBrandsV3ReportRequestBuilder": + self._tactics = tactics + return self + + def with_metrics(self, metrics: List[str]) -> "SponsoredBrandsV3ReportRequestBuilder": + self._metrics = metrics + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_request_builder.py new file mode 100644 index 000000000000..400fe018e55f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_request_builder.py @@ -0,0 +1,66 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredBrandsRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def ad_groups_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, limit: Optional[int] = 100, start_index: Optional[int] = 0 + ) -> "SponsoredBrandsRequestBuilder": + return cls("sb/adGroups") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_limit(limit) \ + .with_start_index(start_index) + + @classmethod + def keywords_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, limit: Optional[int] = 100, start_index: Optional[int] = 0 + ) -> "SponsoredBrandsRequestBuilder": + return cls("sb/keywords") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_limit(limit) \ + .with_start_index(start_index) + + @classmethod + def campaigns_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, limit: Optional[int] = 100, start_index: Optional[int] = 0 + ) -> "SponsoredBrandsRequestBuilder": + return cls("sb/campaigns") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_limit(limit) \ + .with_start_index(start_index) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._limit: Optional[int] = None + self._start_index: Optional[int] = None + + @property + def query_params(self) -> Dict[str, Any]: + query_params = {} + if self._limit is not None: + query_params["count"] = self._limit + if self._start_index: + query_params["startIndex"] = self._start_index + return query_params + + @property + def request_body(self) ->Optional[str]: + return None + + def with_limit(self, limit: int) -> "SponsoredBrandsRequestBuilder": + self._limit: int = limit + return self + + def with_start_index(self, offset: int) -> "SponsoredBrandsRequestBuilder": + self._start_index: int = offset + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_video_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_video_report_request_builder.py new file mode 100644 index 000000000000..21a9d59a828d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_video_report_request_builder.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +import pendulum + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredBrandsVideoReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def _init_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_type: str, metrics: List[str], report_date: Optional[str] = None + ) -> "SponsoredBrandsVideoReportRequestBuilder": + return cls(f"v2/hsa/{report_type}/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_metrics(metrics) \ + .with_report_date(report_date) + + @classmethod + def init_campaigns_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsVideoReportRequestBuilder": + return cls.init_report_endpoint(client_id, client_access_token, profile_id, "campaigns", report_date, metrics) + + @classmethod + def init_ad_groups_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsVideoReportRequestBuilder": + return cls.init_report_endpoint(client_id, client_access_token, profile_id, "adGroups", report_date, metrics) + + @classmethod + def init_keywords_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredBrandsVideoReportRequestBuilder": + return cls.init_report_endpoint(client_id, client_access_token, profile_id, "keywords", report_date, metrics) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._metrics: List[str] = None + self._report_date: str = None + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_date: + body["reportDate"] = self._report_date + body["creativeType"] = "video" + if self._metrics: + body["metrics"] = self._metrics + return json.dumps(body) + + def with_report_date(self, report_date: pendulum.date) -> "SponsoredBrandsVideoReportRequestBuilder": + self._report_date = report_date.format("YYYYMMDD") + return self + + def with_metrics(self, metrics: List[str]) -> "SponsoredBrandsVideoReportRequestBuilder": + self._metrics = ",".join(metrics) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py new file mode 100644 index 000000000000..cfa25f28aa3b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +import pendulum + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredDisplayReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def _init_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_type: str, tactics: str, metrics: List[str], report_date: Optional[str] = None + ) -> "SponsoredDisplayReportRequestBuilder": + return cls(f"sd/{report_type}/report") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_tactics(tactics) \ + .with_metrics(metrics) \ + .with_report_date(report_date) + + @classmethod + def init_campaigns_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, tactics: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredDisplayReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "campaigns", report_date, tactics, metrics) + + @classmethod + def init_ad_groups_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, tactics: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredDisplayReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "adGroups", report_date, tactics, metrics) + + @classmethod + def init_product_ads_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, tactics: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredDisplayReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "productAds", report_date, tactics, metrics) + + @classmethod + def init_targets_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, tactics: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredDisplayReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "targets", report_date, tactics, metrics) + + @classmethod + def init_asins_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, tactics: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredDisplayReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "asins", report_date, tactics, metrics) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._metrics: List[str] = None + self._report_date: str = None + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_date: + body["reportDate"] = self._report_date + if self._tactics: + body["tactic"] = self._tactics + if self._metrics: + body["metrics"] = self._metrics + return json.dumps(body) + + def with_report_date(self, report_date: pendulum.date) -> "SponsoredDisplayReportRequestBuilder": + self._report_date = report_date.format("YYYYMMDD") + return self + + def with_tactics(self, tactics: str) -> "SponsoredDisplayReportRequestBuilder": + self._tactics = tactics + return self + + def with_metrics(self, metrics: List[str]) -> "SponsoredDisplayReportRequestBuilder": + self._metrics = ",".join(metrics) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py new file mode 100644 index 000000000000..b5bba3e2aded --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py @@ -0,0 +1,155 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from collections import OrderedDict +from typing import Any, Dict, List, Optional + +import pendulum + +from .base_request_builder import AmazonAdsBaseRequestBuilder + + +class SponsoredProductsReportRequestBuilder(AmazonAdsBaseRequestBuilder): + @classmethod + def _init_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, report_type: str, metrics: List[str], report_date: Optional[str] = None + ) -> "SponsoredProductsReportRequestBuilder": + return cls(f"reporting/reports") \ + .with_client_id(client_id) \ + .with_client_access_token(client_access_token) \ + .with_profile_id(profile_id) \ + .with_metrics(metrics) \ + .with_report_date(report_date) \ + .with_report_type(report_type) + + @classmethod + def init_campaigns_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "campaigns", report_date, metrics) + + @classmethod + def init_ad_groups_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "adGroups", report_date, metrics) + + @classmethod + def init_keywords_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "keywords", report_date, metrics) + + @classmethod + def init_targets_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "targets", report_date, metrics) + + @classmethod + def init_product_ads_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "productAds", report_date, metrics) + + @classmethod + def init_asins_keywords_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "asins_keywords", report_date, metrics) + + @classmethod + def init_asins_targets_report_endpoint( + cls, client_id: str, client_access_token: str, profile_id: str, metrics: List[str], report_date: Optional[str] + ) -> "SponsoredProductsReportRequestBuilder": + return cls._init_report_endpoint(client_id, client_access_token, profile_id, "asins_targets", report_date, metrics) + + def __init__(self, resource: str) -> None: + super().__init__(resource) + self._metrics: List[str] = None + self._report_date: str = None + self._report_type: str = None + + @property + def _report_config_group_by(self) -> List[str]: + return { + "campaigns": ["campaign"], + "adGroups": ["campaign", "adGroup"], + "keywords": ["targeting"], + "targets": ["targeting"], + "productAds": ["advertiser"], + "asins_keywords": ["asin"], + "asins_targets": ["asin"], + }[self._report_type] + + @property + def _report_config_report_type_id(self) -> str: + return { + "campaigns": "spCampaigns", + "adGroups": "spCampaigns", + "keywords": "spTargeting", + "targets": "spTargeting", + "productAds": "spAdvertisedProduct", + "asins_keywords": "spPurchasedProduct", + "asins_targets": "spPurchasedProduct", + }[self._report_type] + + @property + def _report_config_filters(self) -> List[str]: + return { + "campaigns": [], + "adGroups": [], + "keywords": [{"field": "keywordType", "values": ["BROAD", "PHRASE", "EXACT"]}], + "targets": [{"field": "keywordType", "values": ["TARGETING_EXPRESSION", "TARGETING_EXPRESSION_PREDEFINED"]}], + "productAds": [], + "asins_keywords": [], + "asins_targets": [], + }[self._report_type] + + @property + def query_params(self) -> Dict[str, Any]: + return None + + @property + def request_body(self) ->Optional[str]: + body: dict = OrderedDict() + if self._report_type and self._report_date: + body["name"] = f"{self._report_type} report {self._report_date}" + + if self._report_date: + body["startDate"] = self._report_date + body["endDate"] = self._report_date + + if self._report_type: + body["configuration"] = { + "adProduct": "SPONSORED_PRODUCTS", + "groupBy": self._report_config_group_by + } + + if self._metrics: + body["configuration"]["columns"] = self._metrics + + if self._report_type: + body["configuration"]["reportTypeId"] = self._report_config_report_type_id + body["configuration"]["filters"] = self._report_config_filters + + body["configuration"]["timeUnit"] = "SUMMARY" + body["configuration"]["format"] = "GZIP_JSON" + + return json.dumps(body) + + def with_report_date(self, report_date: pendulum.date) -> "SponsoredProductsReportRequestBuilder": + self._report_date = report_date.format("YYYY-MM-DD") + return self + + def with_report_type(self, report_type: str) -> "SponsoredProductsReportRequestBuilder": + self._report_type = report_type + return self + + def with_tactics(self, tactics: str) -> "SponsoredProductsReportRequestBuilder": + self._tactics = tactics + return self + + def with_metrics(self, metrics: List[str]) -> "SponsoredProductsReportRequestBuilder": + self._metrics = metrics + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/__init__.py new file mode 100644 index 000000000000..04cdc18afd8b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/__init__.py @@ -0,0 +1,8 @@ +from .sponsored_brands_response_builder import SponsoredBrandsResponseBuilder +from .profiles_response_builder import ProfilesResponseBuilder +from .oauth_response_builder import OAuthResponseBuilder +from .error_response_builder import ErrorResponseBuilder +from .attribution_report_response_builder import AttributionReportResponseBuilder +from .report_init_response_builder import ReportInitResponseBuilder +from .report_check_status_response_builder import ReportCheckStatusResponseBuilder +from .report_download_response_builder import ReportDownloadResponseBuilder diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/attribution_report_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/attribution_report_response_builder.py new file mode 100644 index 000000000000..a29b32b19762 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/attribution_report_response_builder.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Optional + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, PaginationStrategy, find_template + + +class AttributionReportResponseBuilder(HttpResponseBuilder): + @classmethod + def products_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "AttributionReportResponseBuilder": + return cls(find_template("attribution_report_products", __file__), FieldPath("reports"), pagination_strategy) + + @classmethod + def performance_adgroup_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "AttributionReportResponseBuilder": + return cls(find_template("attribution_report_performance_adgroup", __file__), FieldPath("reports"), pagination_strategy) + + @classmethod + def performance_campaign_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "AttributionReportResponseBuilder": + return cls(find_template("attribution_report_performance_campaign", __file__), FieldPath("reports"), pagination_strategy) + + @classmethod + def performance_creative_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "AttributionReportResponseBuilder": + return cls(find_template("attribution_report_performance_creative", __file__), FieldPath("reports"), pagination_strategy) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/error_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/error_response_builder.py new file mode 100644 index 000000000000..c23611559074 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/error_response_builder.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from typing import Any, Dict, Optional, Union + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + PaginationStrategy, + RecordBuilder, + find_template, +) + +from .records.fields import DictTemplatePath + + +class ErrorResponseBuilder(HttpResponseBuilder): + def __init__(self, template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Union[PaginationStrategy, None]): + super().__init__(template, records_path, pagination_strategy) + self._records: Dict[str, Any] = {} + + @classmethod + def non_breaking_error_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "ErrorResponseBuilder": + return cls(find_template("non_breaking_error", __file__), DictTemplatePath(), pagination_strategy) + + @classmethod + def breaking_error_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "ErrorResponseBuilder": + return cls(find_template("error", __file__), DictTemplatePath(), pagination_strategy) + + def with_record(self, record: RecordBuilder) -> HttpResponseBuilder: + self._records = record + return self + + def build(self) -> HttpResponse: + self._records_path.update(self._response, self._records.build()) + return HttpResponse(json.dumps(self._response), self._status_code) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/http_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/http_response_builder.py new file mode 100644 index 000000000000..bd8778db9cd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/http_response_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder + + +class AmazonAdsHttpResponseBuilder(HttpResponseBuilder): + def with_pagination(self) -> "AmazonAdsHttpResponseBuilder": + if not self._pagination_strategy: + super().with_pagination() + else: + self._pagination_strategy.update(self._records) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/oauth_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/oauth_response_builder.py new file mode 100644 index 000000000000..17b5cdb6fd01 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/oauth_response_builder.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + + +class OAuthResponseBuilder: + @classmethod + def token_response(cls, status_code: int = 200) -> "OAuthResponseBuilder": + return cls("oauth", status_code) + + def __init__(self, resource: str, status_code: int = 200) -> None: + self._status_code: int = status_code + self._resource: str = resource + + def with_status_code(self, status_code: int) -> "OAuthResponseBuilder": + self._status_code = status_code + return self + + def build(self) -> HttpResponse: + return HttpResponse(json.dumps(find_template(self._resource, __file__)), self._status_code) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/__init__.py new file mode 100644 index 000000000000..b7884e21612c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/__init__.py @@ -0,0 +1,2 @@ +from .count_based_pagination_strategy import CountBasedPaginationStrategy +from .cursor_based_pagination_strategy import CursorBasedPaginationStrategy diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/count_based_pagination_strategy.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/count_based_pagination_strategy.py new file mode 100644 index 000000000000..46f2e32127a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/count_based_pagination_strategy.py @@ -0,0 +1,18 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class CountBasedPaginationStrategy(PaginationStrategy): + @staticmethod + def update(response: List[Dict[str, Any]]) -> None: + if len(response) < 100: + response.extend([response.pop()] * (100 - len(response))) + elif len(response) > 100: + response_page = response[:100] + response.clear() + response.extend(response_page) + else: + pass diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/cursor_based_pagination_strategy.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/cursor_based_pagination_strategy.py new file mode 100644 index 000000000000..5dbab1380bfc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/pagination_strategies/cursor_based_pagination_strategy.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class CursorBasedPaginationStrategy(PaginationStrategy): + @staticmethod + def update(response: Dict[str, Any]) -> None: + response["cursorId"] = "next-page-token" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/profiles_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/profiles_response_builder.py new file mode 100644 index 000000000000..c3db4d96859b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/profiles_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template + +from .records.fields import ListTemplatePath + + +class ProfilesResponseBuilder(HttpResponseBuilder): + @classmethod + def profiles_response(cls) -> "ProfilesResponseBuilder": + return cls(find_template("profiles", __file__), ListTemplatePath(), None) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/__init__.py new file mode 100644 index 000000000000..7ef5d8d4aebf --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/__init__.py @@ -0,0 +1,7 @@ +from .profiles_record_builder import ProfilesRecordBuilder +from .sponsored_brands_record_builder import SponsoredBrandsRecordBuilder +from .error_record_builder import ErrorRecordBuilder +from .attribution_report_record_builder import AttributionReportRecordBuilder +from .report_init_response_record_builder import ReportInitResponseRecordBuilder +from .report_check_status_record_builder import ReportCheckStatusRecordBuilder +from .report_file_recod_builder import ReportFileRecordBuilder diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/attribution_report_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/attribution_report_record_builder.py new file mode 100644 index 000000000000..d537ff3f0b59 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/attribution_report_record_builder.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, RecordBuilder, find_template + + +class AttributionReportRecordBuilder(RecordBuilder): + _field_path = FieldPath("reports") + + @classmethod + def products_record(cls) -> "AttributionReportRecordBuilder": + return cls(cls._field_path.extract(find_template("attribution_report_products", __file__))[0], None, None) + + @classmethod + def performance_adgroup_record(cls) -> "AttributionReportRecordBuilder": + return cls(cls._field_path.extract(find_template("attribution_report_performance_adgroup", __file__))[0], None, None) + + @classmethod + def performance_campaign_record(cls) -> "AttributionReportRecordBuilder": + return cls(cls._field_path.extract(find_template("attribution_report_performance_campaign", __file__))[0], None, None) + + @classmethod + def performance_creative_record(cls) -> "AttributionReportRecordBuilder": + return cls(cls._field_path.extract(find_template("attribution_report_performance_creative", __file__))[0], None, None) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/error_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/error_record_builder.py new file mode 100644 index 000000000000..2028d58e7159 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/error_record_builder.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional, Union + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath, Path, RecordBuilder, find_template + + +class ErrorRecordBuilder(RecordBuilder): + def __init__( + self, + template: Dict[str, Any], + id_path: Optional[Path] = None, + cursor_path: Optional[Union[FieldPath, NestedPath]] = None, + error_message_path: Optional[Path] = None + ): + super().__init__(template, id_path, cursor_path) + self._error_message_path = error_message_path + + @classmethod + def non_breaking_error(cls) -> "ErrorRecordBuilder": + return cls(find_template("non_breaking_error", __file__), None, None, error_message_path=FieldPath("details")) + + @classmethod + def breaking_error(cls) -> "ErrorRecordBuilder": + return cls(find_template("error", __file__), None, None, error_message_path=FieldPath("message")) + + def with_error_message(self, message: str) -> "ErrorRecordBuilder": + self._set_field(self._error_message_path._path[0], self._error_message_path, message) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/__init__.py new file mode 100644 index 000000000000..b692e37af617 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/__init__.py @@ -0,0 +1,2 @@ +from .list_template_path import ListTemplatePath +from .dict_template_path import DictTemplatePath diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/dict_template_path.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/dict_template_path.py new file mode 100644 index 000000000000..c84a6b61629c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/dict_template_path.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import Path + + +class DictTemplatePath(Path): + def update(self, template: Dict[str, Any], value: Dict[str, Any]) -> None: + template.clear() + template.update(value) + + def write(self, template: Dict[str, Any], value: Dict[str, Any]) -> None: + template.update(value) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/list_template_path.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/list_template_path.py new file mode 100644 index 000000000000..7415114a85ed --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/fields/list_template_path.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List + +from airbyte_cdk.test.mock_http.response_builder import Path + + +class ListTemplatePath(Path): + def update(self, template: List[Dict[str, Any]], value: List[Dict[str, Any]]) -> None: + template.clear() + template.extend(value) + + def write(self, template: List[Dict[str, Any]], value: List[Dict[str, Any]]) -> None: + template.extend(value) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/profiles_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/profiles_record_builder.py new file mode 100644 index 000000000000..a8c979834254 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/profiles_record_builder.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, RecordBuilder, find_template + + +class ProfilesRecordBuilder(RecordBuilder): + @classmethod + def profiles_record(cls) -> "ProfilesRecordBuilder": + return cls(find_template("profiles", __file__)[0], FieldPath("profileId"), None) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_check_status_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_check_status_record_builder.py new file mode 100644 index 000000000000..8681c0eb3b09 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_check_status_record_builder.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional, Union + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath, Path, RecordBuilder, find_template + + +class ReportCheckStatusRecordBuilder(RecordBuilder): + @classmethod + def status_record(cls) -> "ReportCheckStatusRecordBuilder": + return cls( + find_template("report_status_response", __file__), + id_path=None, + status_path=FieldPath("status"), + url_path=FieldPath("url") + ) + + def __init__( + self, + template: Dict[str, Any], + id_path: Optional[Path] = None, + status_path: Optional[Path] = None, + url_path: Optional[Path] = None, + cursor_path: Optional[Union[FieldPath, NestedPath]] = None + ): + super().__init__(template, id_path, cursor_path) + self._status_path = status_path + self._url_path = url_path + + def with_status(self, status: str) -> "ReportCheckStatusRecordBuilder": + self._set_field("status", self._status_path, status) + return self + + def with_url(self, url: str) -> "ReportCheckStatusRecordBuilder": + self._set_field("status", self._url_path, url) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_file_recod_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_file_recod_builder.py new file mode 100644 index 000000000000..3e40f5a581c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_file_recod_builder.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, RecordBuilder, find_template + + +class ReportFileRecordBuilder(RecordBuilder): + @classmethod + def report_file_record(cls): + return cls(find_template("download_report_file", __file__)[0], FieldPath("campaignId"), None) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_init_response_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_init_response_record_builder.py new file mode 100644 index 000000000000..eec142972d1c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/report_init_response_record_builder.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, Optional, Union + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath, Path, RecordBuilder, find_template + + +class ReportInitResponseRecordBuilder(RecordBuilder): + @classmethod + def init_response_record(cls) -> "ReportInitResponseRecordBuilder": + return cls( + find_template("report_init_response", __file__), + id_path=FieldPath("reportId"), + status_path=FieldPath("status"), + cursor_path=None + ) + + def __init__( + self, + template: Dict[str, Any], + id_path: Optional[Path] = None, + status_path: Optional[Path] = None, + cursor_path: Optional[Union[FieldPath, NestedPath]] = None + ): + super().__init__(template, id_path, cursor_path) + self._status_path = status_path + + def with_status(self, status: str) -> "ReportInitResponseRecordBuilder": + self._set_field("status", self._status_path, status) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/sponsored_brands_record_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/sponsored_brands_record_builder.py new file mode 100644 index 000000000000..8bcd24507ab7 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/records/sponsored_brands_record_builder.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, RecordBuilder, find_template + + +class SponsoredBrandsRecordBuilder(RecordBuilder): + @classmethod + def ad_groups_record(cls) -> "SponsoredBrandsRecordBuilder": + return cls(find_template("sponsored_brands_ad_groups", __file__)[0], FieldPath("adGroupId"), None) + + @classmethod + def campaigns_record(cls) -> "SponsoredBrandsRecordBuilder": + return cls(find_template("sponsored_brands_campaigns", __file__)[0], FieldPath("campaignId"), None) + + @classmethod + def keywords_record(cls) -> "SponsoredBrandsRecordBuilder": + return cls(find_template("sponsored_brands_keywords", __file__)[0], FieldPath("adGroupId"), None) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_check_status_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_check_status_response_builder.py new file mode 100644 index 000000000000..8ba0dfc26aed --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_check_status_response_builder.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, RecordBuilder, find_template + +from .records.fields import DictTemplatePath + + +class ReportCheckStatusResponseBuilder(HttpResponseBuilder): + @classmethod + def check_status_response(cls) -> "ReportCheckStatusResponseBuilder": + return cls(find_template("report_status_response", __file__), DictTemplatePath(), None) + + def with_record(self, record: RecordBuilder) -> HttpResponseBuilder: + self._records = record + return self + + def build(self) -> HttpResponse: + self._records_path.update(self._response, self._records.build()) + return HttpResponse(json.dumps(self._response), self._status_code) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py new file mode 100644 index 000000000000..93f08ef88ec1 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import gzip +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template + +from .records.fields import ListTemplatePath + + +class ReportDownloadResponseBuilder(HttpResponseBuilder): + @classmethod + def download_report(cls) -> "ReportDownloadResponseBuilder": + return cls(find_template("download_report_file", __file__), ListTemplatePath(), None) + + def build(self) -> HttpResponse: + http_response = super().build() + http_response._body = gzip.compress(http_response._body.encode("iso-8859-1")) + return http_response diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_init_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_init_response_builder.py new file mode 100644 index 000000000000..470b7f72a527 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_init_response_builder.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, RecordBuilder, find_template + +from .records.fields import DictTemplatePath + + +class ReportInitResponseBuilder(HttpResponseBuilder): + @classmethod + def report_init_response(cls) -> "ReportInitResponseBuilder": + return cls(find_template("report_init_response", __file__), DictTemplatePath(), None) + + def with_record(self, record: RecordBuilder) -> HttpResponseBuilder: + self._records = record + return self + + def build(self) -> HttpResponse: + self._records_path.update(self._response, self._records.build()) + return HttpResponse(json.dumps(self._response), self._status_code) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/sponsored_brands_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/sponsored_brands_response_builder.py new file mode 100644 index 000000000000..2b07547fa989 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/sponsored_brands_response_builder.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Optional + +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, PaginationStrategy, find_template + +from .records.fields import ListTemplatePath + + +class SponsoredBrandsResponseBuilder(HttpResponseBuilder): + @classmethod + def ad_groups_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "SponsoredBrandsResponseBuilder": + return cls(find_template("sponsored_brands_ad_groups", __file__), ListTemplatePath(), pagination_strategy) + + @classmethod + def ad_groups_non_breaking_error_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "SponsoredBrandsResponseBuilder": + return cls(find_template("non_breaking_error", __file__), ListTemplatePath(), pagination_strategy) + + @classmethod + def campaigns_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "SponsoredBrandsResponseBuilder": + return cls(find_template("sponsored_brands_campaigns", __file__), ListTemplatePath(), pagination_strategy) + + @classmethod + def keywords_response(cls, pagination_strategy: Optional[PaginationStrategy] = None) -> "SponsoredBrandsResponseBuilder": + return cls(find_template("sponsored_brands_keywords", __file__), ListTemplatePath(), pagination_strategy) + + def with_pagination(self) -> "SponsoredBrandsResponseBuilder": + if not self._pagination_strategy: + super().with_pagination() + else: + self._pagination_strategy.update(self._records) + return self diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py new file mode 100644 index 000000000000..a9a989b8d8c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import datetime +from typing import Any, Dict, List + +from airbyte_cdk.test.mock_http.response_builder import find_template + +CLIENT_ID = "amzn.app-oa2-client.test" +CLIENT_SECRET = "test-secret" +REGION = "NA" +REPORT_WAIT_TIMEOUT = 120 +PROFILES = [1] + + +class ConfigBuilder: + def __init__(self) -> None: + oauth_fixture: Dict[str, Any] = find_template("oauth", __file__) + self._access_token: str = oauth_fixture["access_token"] + self._refresh_token: str = oauth_fixture["refresh_token"] + self._client_id: str = CLIENT_ID + self._client_secret: str = CLIENT_SECRET + self._region: str = REGION + self._report_wait_timeout: str = REPORT_WAIT_TIMEOUT + self._profiles: str = PROFILES + self._start_date: str = None + + def with_client_id(self, client_id: str) -> "ConfigBuilder": + self._client_id = client_id + return self + + def with_client_secret(self, client_secret: str) -> "ConfigBuilder": + self._client_secret = client_secret + return self + + def with_access_token(self, access_token: str) -> "ConfigBuilder": + self._access_token = access_token + return self + + def with_refresh_token(self, refresh_token: str) -> "ConfigBuilder": + self._refresh_token = refresh_token + return self + + def with_region(self, region: str) -> "ConfigBuilder": + self._region = region + return self + + def with_report_wait_timeout(self, report_wait_timeout: int) -> "ConfigBuilder": + self._report_wait_timeout = report_wait_timeout + return self + + def with_profiles(self, profiles: List[int]) -> "ConfigBuilder": + self._profiles = profiles + return self + + def with_start_date(self, start_date: datetime.date) -> "ConfigBuilder": + self._start_date = start_date.isoformat() + return self + + def build(self) -> Dict[str, Any]: + config = { + "client_id": self._client_id, + "client_secret": self._client_secret, + "access_token": self._access_token, + "refresh_token": self._refresh_token, + "region": self._region, + "report_wait_timeout": self._report_wait_timeout, + "profiles": self._profiles, + } + if self._start_date: + config["start_date"] = self._start_date + return config diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py new file mode 100644 index 000000000000..6548c3a39c9f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py @@ -0,0 +1,500 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from unittest import TestCase +from unittest.mock import patch +from zoneinfo import ZoneInfo + +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .ad_requests import AttributionReportRequestBuilder, OAuthRequestBuilder, ProfilesRequestBuilder +from .ad_responses import AttributionReportResponseBuilder, ErrorResponseBuilder, OAuthResponseBuilder, ProfilesResponseBuilder +from .ad_responses.pagination_strategies import CursorBasedPaginationStrategy +from .ad_responses.records import AttributionReportRecordBuilder, ErrorRecordBuilder, ProfilesRecordBuilder +from .config import ConfigBuilder +from .utils import get_log_messages_by_log_level, read_stream + +REPORTING_PERIOD = 90 +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=REPORTING_PERIOD) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestAttributionReportStreamsFullRefresh(TestCase): + @property + def _config(self): + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + return ConfigBuilder().with_start_date(_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date()).build() + + def _given_oauth_and_profiles(self, http_mocker: HttpMocker, config: dict) -> None: + http_mocker.post( + OAuthRequestBuilder.oauth_endpoint(client_id=config["client_id"], client_secred=config["client_secret"], refresh_token=config["refresh_token"]).build(), + OAuthResponseBuilder.token_response().build() + ) + http_mocker.get( + ProfilesRequestBuilder.profiles_endpoint(client_id=config["client_id"], client_access_token=config["access_token"]).build(), + ProfilesResponseBuilder.profiles_response().with_record(ProfilesRecordBuilder.profiles_record()).build() + ) + + @HttpMocker() + def test_given_non_breaking_error_when_read_products_then_stream_is_ignored(self, http_mocker): + """ + Check products stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.products_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + + output = read_stream("attribution_report_products", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_products_then_stream_is_ignored(self, http_mocker): + """ + Check products stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + breaking_error = ErrorRecordBuilder.breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.products_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("attribution_report_products", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_products_then_return_records(self, http_mocker): + """ + Check prodcts stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.products_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.products_response().with_record(AttributionReportRecordBuilder.products_record()).build() + ) + + output = read_stream("attribution_report_products", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_products_then_return_records(self, http_mocker): + """ + Check products stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.products_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.products_response(CursorBasedPaginationStrategy()) \ + .with_record(AttributionReportRecordBuilder.products_record()) \ + .with_pagination() \ + .build() + ) + http_mocker.post( + AttributionReportRequestBuilder.products_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).with_cursor_field("next-page-token").build(), + AttributionReportResponseBuilder.products_response().with_record(AttributionReportRecordBuilder.products_record()).build() + ) + + output = read_stream("attribution_report_products", SyncMode.full_refresh, self._config) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_non_breaking_error_when_read_performance_adgroup_then_stream_is_ignored(self, http_mocker): + """ + Check performance ad group stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_adgroup_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + + output = read_stream("attribution_report_performance_adgroup", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_performance_adgroup_then_stream_is_ignored(self, http_mocker): + """ + Check performance ad group stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + breaking_error = ErrorRecordBuilder.breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_adgroup_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("attribution_report_performance_adgroup", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_performance_adgroup_then_return_records(self, http_mocker): + """ + Check performance ad group stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_adgroup_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_adgroup_response().with_record(AttributionReportRecordBuilder.performance_adgroup_record()).build() + ) + + output = read_stream("attribution_report_performance_adgroup", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_performance_adgroup_then_return_records(self, http_mocker): + """ + Check performance ad group stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_adgroup_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_adgroup_response(CursorBasedPaginationStrategy()) \ + .with_record(AttributionReportRecordBuilder.performance_adgroup_record()) \ + .with_pagination() \ + .build() + ) + http_mocker.post( + AttributionReportRequestBuilder.performance_adgroup_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).with_cursor_field("next-page-token").build(), + AttributionReportResponseBuilder.performance_adgroup_response().with_record(AttributionReportRecordBuilder.performance_adgroup_record()).build() + ) + + output = read_stream("attribution_report_performance_adgroup", SyncMode.full_refresh, self._config) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_non_breaking_error_when_read_performance_campaign_then_stream_is_ignored(self, http_mocker): + """ + Check performance campaign stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_campaign_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + + output = read_stream("attribution_report_performance_campaign", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_performance_campaign_then_stream_is_ignored(self, http_mocker): + """ + Check performance campaign stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + breaking_error = ErrorRecordBuilder.breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_campaign_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("attribution_report_performance_campaign", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_performance_campaign_then_return_records(self, http_mocker): + """ + Check performance campaign stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_campaign_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_campaign_response().with_record(AttributionReportRecordBuilder.performance_campaign_record()).build() + ) + + output = read_stream("attribution_report_performance_campaign", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_performance_campaign_then_return_records(self, http_mocker): + """ + Check performance campaign stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_campaign_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_campaign_response(CursorBasedPaginationStrategy()) \ + .with_record(AttributionReportRecordBuilder.performance_campaign_record()) \ + .with_pagination() \ + .build() + ) + http_mocker.post( + AttributionReportRequestBuilder.performance_campaign_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).with_cursor_field("next-page-token").build(), + AttributionReportResponseBuilder.performance_campaign_response().with_record(AttributionReportRecordBuilder.performance_campaign_record()).build() + ) + + output = read_stream("attribution_report_performance_campaign", SyncMode.full_refresh, self._config) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_non_breaking_error_when_read_performance_creative_then_stream_is_ignored(self, http_mocker): + """ + Check performance creative stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_creative_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + + output = read_stream("attribution_report_performance_creative", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_performance_creative_then_stream_is_ignored(self, http_mocker): + """ + Check performance creative stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + breaking_error = ErrorRecordBuilder.breaking_error() + + http_mocker.post( + AttributionReportRequestBuilder.performance_creative_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("attribution_report_performance_creative", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_performance_creative_then_return_records(self, http_mocker): + """ + Check performance creative stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_creative_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_creative_response().with_record(AttributionReportRecordBuilder.performance_creative_record()).build() + ) + + output = read_stream("attribution_report_performance_creative", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_performance_creative_then_return_records(self, http_mocker): + """ + Check performance creative stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + + http_mocker.post( + AttributionReportRequestBuilder.performance_creative_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).build(), + AttributionReportResponseBuilder.performance_creative_response(CursorBasedPaginationStrategy()) \ + .with_record(AttributionReportRecordBuilder.performance_creative_record()) \ + .with_pagination() \ + .build() + ) + http_mocker.post( + AttributionReportRequestBuilder.performance_creative_endpoint( + self._config["client_id"], + self._config["access_token"], + self._config["profiles"][0], + start_date=_A_START_DATE.astimezone(ZoneInfo(profile_timezone)).date(), + end_date=_NOW.astimezone(ZoneInfo(profile_timezone)).date() + ).with_cursor_field("next-page-token").build(), + AttributionReportResponseBuilder.performance_creative_response().with_record(AttributionReportRecordBuilder.performance_creative_record()).build() + ) + + output = read_stream("attribution_report_performance_creative", SyncMode.full_refresh, self._config) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py new file mode 100644 index 000000000000..2d377a047928 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py @@ -0,0 +1,432 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import uuid +from unittest import TestCase + +import pendulum +import requests_mock +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequestMatcher +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode +from source_amazon_ads.streams.report_streams import brands_report, brands_video_report, display_report, products_report + +from .ad_requests import ( + OAuthRequestBuilder, + ProfilesRequestBuilder, + ReportCheckStatusRequestBuilder, + ReportDownloadRequestBuilder, + SponsoredBrandsReportRequestBuilder, + SponsoredBrandsV3ReportRequestBuilder, + SponsoredBrandsVideoReportRequestBuilder, + SponsoredDisplayReportRequestBuilder, + SponsoredProductsReportRequestBuilder, +) +from .ad_responses import ( + ErrorResponseBuilder, + OAuthResponseBuilder, + ProfilesResponseBuilder, + ReportCheckStatusResponseBuilder, + ReportDownloadResponseBuilder, + ReportInitResponseBuilder, +) +from .ad_responses.records import ( + ErrorRecordBuilder, + ProfilesRecordBuilder, + ReportCheckStatusRecordBuilder, + ReportFileRecordBuilder, + ReportInitResponseRecordBuilder, +) +from .config import ConfigBuilder +from .utils import get_log_messages_by_log_level, read_stream + + +class TestDisplayReportStreams(TestCase): + @property + def _config(self): + return ConfigBuilder().build() + + def _given_oauth_and_profiles(self, http_mocker: HttpMocker, config: dict) -> None: + """ + Authenticate and get profiles + """ + http_mocker.post( + OAuthRequestBuilder.oauth_endpoint(client_id=config["client_id"], client_secred=config["client_secret"], refresh_token=config["refresh_token"]).build(), + OAuthResponseBuilder.token_response().build() + ) + http_mocker.get( + ProfilesRequestBuilder.profiles_endpoint(client_id=config["client_id"], client_access_token=config["access_token"]).build(), + ProfilesResponseBuilder.profiles_response().with_record(ProfilesRecordBuilder.profiles_record()).build() + ) + + @HttpMocker() + def test_given_file_when_read_display_report_then_return_records(self, http_mocker): + """ + Check display report stream: normal stream read flow + In this test we prepare http mocker to handle all report types and tactics as well as workaround to handle gzipped file content + Request structure: + 1. Request report for start processing + 2. Check status and get a download link + 3. Download report file using the link + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + + for report_type, metrics in display_report.METRICS_MAP.items(): + for tactic in display_report.TACTICS: + report_id = str(uuid.uuid4()) + http_mocker.post( + SponsoredDisplayReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, tactic, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(202).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_display_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record(ReportFileRecordBuilder.report_file_record()).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + output = read_stream("sponsored_display_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 10 + + @HttpMocker() + def test_given_file_when_read_products_report_then_return_records(self, http_mocker): + """ + Check products report stream: normal stream read flow. + In this test we prepare http mocker to handle all report types based on metrics defined for the report stream + as well as workaround to handle gzipped file content. + Request structure: + 1. Request report for start processing + 2. Check status and get a download link + 3. Download report file using the link + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + + for report_type, metrics in products_report.METRICS_MAP.items(): + report_id = str(uuid.uuid4()) + http_mocker.post( + SponsoredProductsReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(200).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_products_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record( + ReportFileRecordBuilder.report_file_record() + ).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + output = read_stream("sponsored_products_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 7 + + @HttpMocker() + def test_given_file_when_read_brands_video_report_then_return_records(self, http_mocker): + """ + Check brands video report stream: normal stream read flow. + In this test we prepare http mocker to handle all report types based on metrics defined for the report stream + as well as workaround to handle gzipped file content + Request structure: + 1. Request report for start processing + 2. Check status and get a download link + 3. Download report file using the link + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + + for report_type, metrics in brands_video_report.METRICS_MAP.items(): + report_id = str(uuid.uuid4()) + http_mocker.post( + SponsoredBrandsVideoReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(202).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_brands_video_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record( + ReportFileRecordBuilder.report_file_record() + ).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + output = read_stream("sponsored_brands_video_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_file_when_read_brands_report_then_return_records(self, http_mocker): + """ + Check brands report stream: normal stream read flow. + In this test we prepare http mocker to handle all report types based on metrics defined for the report stream + as well as workaround to handle gzipped file content. + Request structure: + 1. Request report for start processing + 2. Check status and get a download link + 3. Download report file using the link + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + + for report_type, metrics in brands_report.METRICS_MAP.items(): + report_id = str(uuid.uuid4()) + http_mocker.post( + SponsoredBrandsReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(202).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_brands_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record( + ReportFileRecordBuilder.report_file_record() + ).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + output = read_stream("sponsored_brands_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_file_when_read_brands_v3_report_then_return_records(self, http_mocker): + """ + Check brands v3 report stream: normal stream read flow. + In this test we prepare http mocker to handle all report types based on metrics defined for the report stream + as well as workaround to handle gzipped file content. + Request structure: + 1. Request report for start processing + 2. Check status and get a download link + 3. Download report file using the link + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + + for report_type, metrics in brands_report.METRICS_MAP_V3.items(): + report_id = str(uuid.uuid4()) + http_mocker.post( + SponsoredBrandsV3ReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(200).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_brands_v3_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record( + ReportFileRecordBuilder.report_file_record() + ).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + output = read_stream("sponsored_brands_v3_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_known_error_when_read_brands_v3_report_then_skip_report(self, http_mocker): + """ + Check brands v3 stream: non-breaking errors are ignored. + When error of this kind happen, we warn and then keep syncing another reports if possible. + In this test all report init requests are failed with known error and skipped + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + ERRORS = [ + (400, "KDP authors do not have access to Sponsored Brands functionality"), + (401, "Not authorized to access scope 0001"), + (406, "Report date is too far in the past."), + ] + + for status_code, msg in ERRORS: + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + non_breaking_error = ErrorRecordBuilder.non_breaking_error().with_error_message(msg) + + for report_type, metrics in brands_report.METRICS_MAP_V3.items(): + http_mocker.post( + SponsoredBrandsV3ReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, metrics, start_date + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(status_code).build(), + ) + + output = read_stream("sponsored_brands_v3_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + expected_warning_log = ( + f"Unexpected HTTP status code {status_code} when registering purchasedAsin, " + f"SponsoredBrandsV3ReportStream for 1 profile: {json.dumps(non_breaking_error.build())}" + ) + assert any([expected_warning_log in warn for warn in warning_logs]) + + @HttpMocker() + def test_given_known_error_when_read_display_report_then_partially_skip_records(self, http_mocker): + """ + Check brands v3 stream: non-breaking errors are ignored. + When error of this kind happen, we warn and then keep syncing another reports if possible. + In this test half of report init requests are failed with known error and skipped while another half of reports successfully processed + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + ERRORS = [ + (400, "Tactic T00030 is not supported for report API in marketplace ABC00030."), + ] + + for status_code, msg in ERRORS: + profile_timezone = ProfilesRecordBuilder.profiles_record().build().get("timezone") + start_date = pendulum.today(tz=profile_timezone).date() + non_breaking_error = ErrorRecordBuilder.non_breaking_error().with_error_message(msg) + + for report_type, metrics in display_report.METRICS_MAP.items(): + report_id = str(uuid.uuid4()) + tactic = display_report.TACTICS[0] + http_mocker.post( + SponsoredDisplayReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, tactic, metrics, start_date + ).build(), + ReportInitResponseBuilder.report_init_response().with_record( + ReportInitResponseRecordBuilder.init_response_record().with_status("PENDING").with_id(report_id) + ).with_status_code(202).build() + ) + download_request_builder = ReportDownloadRequestBuilder.download_endpoint(report_id) + http_mocker.get( + ReportCheckStatusRequestBuilder.check_sponsored_display_report_status_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_id + ).build(), + ReportCheckStatusResponseBuilder.check_status_response().with_record( + ReportCheckStatusRecordBuilder.status_record().with_status("COMPLETED").with_url(download_request_builder.url) + ).build() + ) + + # a workaround to pass compressed document to the mocked response + gzip_file_report_response = ReportDownloadResponseBuilder.download_report().with_record(ReportFileRecordBuilder.report_file_record()).build() + request_matcher = HttpRequestMatcher(download_request_builder.build(), minimum_number_of_expected_match=1) + http_mocker._matchers.append(request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(request_matcher), + response_list=[{"content": gzip_file_report_response.body, "status_code": gzip_file_report_response.status_code}], + ) + + for report_type, metrics in display_report.METRICS_MAP.items(): + tactic = display_report.TACTICS[1] + http_mocker.post( + SponsoredDisplayReportRequestBuilder._init_report_endpoint( + self._config["client_id"], self._config["access_token"], self._config["profiles"][0], report_type, tactic, metrics, start_date + ).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(status_code).build(), + ) + + output = read_stream("sponsored_display_report_stream", SyncMode.full_refresh, self._config) + assert len(output.records) == 5 + + expected_warning_logs = [ + ( + f"Unexpected HTTP status code {status_code} when registering {report_type}, " + f"SponsoredDisplayReportStream for 1 profile: {json.dumps(non_breaking_error.build())}" + ) for report_type in display_report.METRICS_MAP.keys() + ] + for expected_warning_log in expected_warning_logs: + assert any( + [ + expected_warning_log in warn + for warn in get_log_messages_by_log_level(output.logs, LogLevel.WARN) + ] + ) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py new file mode 100644 index 000000000000..ffad23c72024 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py @@ -0,0 +1,262 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest import TestCase +from unittest.mock import patch + +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .ad_requests import OAuthRequestBuilder, ProfilesRequestBuilder, SponsoredBrandsRequestBuilder +from .ad_responses import ErrorResponseBuilder, OAuthResponseBuilder, ProfilesResponseBuilder, SponsoredBrandsResponseBuilder +from .ad_responses.pagination_strategies import CountBasedPaginationStrategy +from .ad_responses.records import ErrorRecordBuilder, ProfilesRecordBuilder, SponsoredBrandsRecordBuilder +from .config import ConfigBuilder +from .utils import get_log_messages_by_log_level, read_stream + + +class TestSponsoredBrandsStreamsFullRefresh(TestCase): + @property + def _config(self): + return ConfigBuilder().build() + + def _given_oauth_and_profiles(self, http_mocker: HttpMocker, config: dict) -> None: + """ + Authenticate and get profiles + """ + http_mocker.post( + OAuthRequestBuilder.oauth_endpoint(client_id=config["client_id"], client_secred=config["client_secret"], refresh_token=config["refresh_token"]).build(), + OAuthResponseBuilder.token_response().build() + ) + http_mocker.get( + ProfilesRequestBuilder.profiles_endpoint(client_id=config["client_id"], client_access_token=config["access_token"]).build(), + ProfilesResponseBuilder.profiles_response().with_record(ProfilesRecordBuilder.profiles_record()).build() + ) + + @HttpMocker() + def test_given_non_breaking_error_when_read_ad_groups_then_stream_is_ignored(self, http_mocker): + """ + Check ad groups stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + output = read_stream("sponsored_brands_ad_groups", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_ad_groups_then_stream_stop_syncing(self, http_mocker): + """ + Check ad groups stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + breaking_error = ErrorRecordBuilder.breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + with patch('time.sleep', return_value=None): + output = read_stream("sponsored_brands_ad_groups", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_ad_groups_then_return_records(self, http_mocker): + """ + Check ad groups stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.ad_groups_response().with_record(SponsoredBrandsRecordBuilder.ad_groups_record()).build() + ) + + output = read_stream("sponsored_brands_ad_groups", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_ad_groups_then_return_records(self, http_mocker): + """ + Check ad groups stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.ad_groups_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.ad_groups_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=100).build(), + SponsoredBrandsResponseBuilder.ad_groups_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.ad_groups_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.ad_groups_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=200).build(), + SponsoredBrandsResponseBuilder.ad_groups_response().with_record(SponsoredBrandsRecordBuilder.ad_groups_record()).build() + ) + + output = read_stream("sponsored_brands_ad_groups", SyncMode.full_refresh, self._config) + assert len(output.records) == 201 + + @HttpMocker() + def test_given_non_breaking_error_when_read_campaigns_then_stream_is_ignored(self, http_mocker): + """ + Check campaigns stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + output = read_stream("sponsored_brands_campaigns", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_campaigns_then_stream_stop_syncing(self, http_mocker): + """ + Check campaigns stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + breaking_error = ErrorRecordBuilder.breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + with patch('time.sleep', return_value=None): + output = read_stream("sponsored_brands_campaigns", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_campaigns_then_return_records(self, http_mocker): + """ + Check campaigns stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.campaigns_response().with_record(SponsoredBrandsRecordBuilder.campaigns_record()).build() + ) + + output = read_stream("sponsored_brands_campaigns", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_campaigns_then_return_records(self, http_mocker): + """ + Check campaigns stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.campaigns_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.campaigns_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=100).build(), + SponsoredBrandsResponseBuilder.campaigns_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.campaigns_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.campaigns_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=200).build(), + SponsoredBrandsResponseBuilder.campaigns_response().with_record(SponsoredBrandsRecordBuilder.campaigns_record()).build() + ) + + output = read_stream("sponsored_brands_campaigns", SyncMode.full_refresh, self._config) + assert len(output.records) == 201 + + @HttpMocker() + def test_given_non_breaking_error_when_read_keywords_then_stream_is_ignored(self, http_mocker): + """ + Check keywords stream: non-breaking errors are ignored + When error of this kind happen, we warn and then keep syncing another streams + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + non_breaking_error = ErrorRecordBuilder.non_breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.non_breaking_error_response().with_record(non_breaking_error).with_status_code(400).build() + ) + output = read_stream("sponsored_brands_keywords", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + warning_logs = get_log_messages_by_log_level(output.logs, LogLevel.WARN) + assert any([non_breaking_error.build().get("details") in worning for worning in warning_logs]) + + @HttpMocker() + def test_given_breaking_error_when_read_keywords_then_stream_stop_syncing(self, http_mocker): + """ + Check keywords stream: when unknown error happen we stop syncing with raising the error + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + breaking_error = ErrorRecordBuilder.breaking_error() + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + ErrorResponseBuilder.breaking_error_response().with_record(breaking_error).with_status_code(500).build() + ) + with patch('time.sleep', return_value=None): + output = read_stream("sponsored_brands_keywords", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any([breaking_error.build().get("message") in error for error in error_logs]) + + @HttpMocker() + def test_given_one_page_when_read_keywords_then_return_records(self, http_mocker): + """ + Check keywords stream: normal full refresh sync without pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.keywords_response().with_record(SponsoredBrandsRecordBuilder.keywords_record()).build() + ) + + output = read_stream("sponsored_brands_keywords", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_many_pages_when_read_keywords_then_return_records(self, http_mocker): + """ + Check keywords stream: normal full refresh sync with pagination + """ + self._given_oauth_and_profiles(http_mocker, self._config) + + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100).build(), + SponsoredBrandsResponseBuilder.keywords_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.keywords_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=100).build(), + SponsoredBrandsResponseBuilder.keywords_response(CountBasedPaginationStrategy()).with_record(SponsoredBrandsRecordBuilder.keywords_record()).with_pagination().build() + ) + http_mocker.get( + SponsoredBrandsRequestBuilder.keywords_endpoint(self._config["client_id"], self._config["access_token"], self._config["profiles"][0], limit=100, start_index=200).build(), + SponsoredBrandsResponseBuilder.keywords_response().with_record(SponsoredBrandsRecordBuilder.keywords_record()).build() + ) + + output = read_stream("sponsored_brands_keywords", SyncMode.full_refresh, self._config) + assert len(output.records) == 201 diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py new file mode 100644 index 000000000000..6d2828a3bf70 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import operator +from typing import Any, Dict, List, Optional + +from airbyte_cdk.models import AirbyteMessage +from airbyte_cdk.models import Level as LogLevel +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_protocol.models import SyncMode +from source_amazon_ads import SourceAmazonAds + + +def read_stream( + stream_name: str, + sync_mode: SyncMode, + config: Dict[str, Any], + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build() + return read(SourceAmazonAds(), config, catalog, state, expecting_exception) + + +def get_log_messages_by_log_level(logs: List[AirbyteMessage], log_level: LogLevel) -> List[str]: + return map(operator.attrgetter("log.message"), filter(lambda x: x.log.level == log_level, logs)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_adgroup.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_adgroup.json new file mode 100644 index 000000000000..3f957cc1b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_adgroup.json @@ -0,0 +1,22 @@ +{ + "reports": [ + { + "date": "string", + "brandName": "string", + "marketplace": "string", + "campaignId": "string", + "productAsin": "string", + "productConversionType": "string", + "advertiserName": "string", + "adGroupId": "string", + "creativeId": "string", + "productName": "string", + "productCategory": "string", + "productSubcategory": "string", + "productGroup": "string", + "publisher": "string" + } + ], + "size": 0, + "cursorId": "" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_campaign.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_campaign.json new file mode 100644 index 000000000000..3f957cc1b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_campaign.json @@ -0,0 +1,22 @@ +{ + "reports": [ + { + "date": "string", + "brandName": "string", + "marketplace": "string", + "campaignId": "string", + "productAsin": "string", + "productConversionType": "string", + "advertiserName": "string", + "adGroupId": "string", + "creativeId": "string", + "productName": "string", + "productCategory": "string", + "productSubcategory": "string", + "productGroup": "string", + "publisher": "string" + } + ], + "size": 0, + "cursorId": "" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_creative.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_creative.json new file mode 100644 index 000000000000..3f957cc1b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_performance_creative.json @@ -0,0 +1,22 @@ +{ + "reports": [ + { + "date": "string", + "brandName": "string", + "marketplace": "string", + "campaignId": "string", + "productAsin": "string", + "productConversionType": "string", + "advertiserName": "string", + "adGroupId": "string", + "creativeId": "string", + "productName": "string", + "productCategory": "string", + "productSubcategory": "string", + "productGroup": "string", + "publisher": "string" + } + ], + "size": 0, + "cursorId": "" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_products.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_products.json new file mode 100644 index 000000000000..3f957cc1b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/attribution_report_products.json @@ -0,0 +1,22 @@ +{ + "reports": [ + { + "date": "string", + "brandName": "string", + "marketplace": "string", + "campaignId": "string", + "productAsin": "string", + "productConversionType": "string", + "advertiserName": "string", + "adGroupId": "string", + "creativeId": "string", + "productName": "string", + "productCategory": "string", + "productSubcategory": "string", + "productGroup": "string", + "publisher": "string" + } + ], + "size": 0, + "cursorId": "" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/download_report_file.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/download_report_file.json new file mode 100644 index 000000000000..ea886db49a61 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/download_report_file.json @@ -0,0 +1,13 @@ +[ + { + "campaignId": 1, + "campaignName": "str", + "adGroupId": "str", + "adId": "str", + "targetId": "str", + "asin": "str", + "advertisedAsin": "str", + "keywordBid": "str", + "keywordId": "str" + } +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/error.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/error.json new file mode 100644 index 000000000000..e0beab50dbfa --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/error.json @@ -0,0 +1,3 @@ +{ + "message": "no way" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/non_breaking_error.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/non_breaking_error.json new file mode 100644 index 000000000000..b7a82340f900 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/non_breaking_error.json @@ -0,0 +1,5 @@ +{ + "code": "0", + "details": "we are in trouble, but lets keep going", + "requestId": "0" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/oauth.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/oauth.json new file mode 100644 index 000000000000..12f51ef7c7a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/oauth.json @@ -0,0 +1,6 @@ +{ + "access_token": "test-access-token", + "refresh_token": "test-refresh-token", + "token_type": "bearer", + "expires_in": 3600 +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/profiles.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/profiles.json new file mode 100644 index 000000000000..ea19614b6579 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/profiles.json @@ -0,0 +1,16 @@ +[ + { + "profileId": 1, + "countryCode": "CA", + "currencyCode": "CAD", + "dailyBudget": 0.0, + "timezone": "America/Los_Angeles", + "accountInfo": { + "marketplaceStringId": "A2EUQ1WTGCTBG2", + "id": "A3LUQZ2NBMFGO4", + "type": "seller", + "name": "string", + "validPaymentMethod": true + } + } +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_init_response.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_init_response.json new file mode 100644 index 000000000000..d280e044e1fb --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_init_response.json @@ -0,0 +1,4 @@ +{ + "reportId": "str", + "status": "str" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_status_response.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_status_response.json new file mode 100644 index 000000000000..d61e7ea2fb34 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/report_status_response.json @@ -0,0 +1,5 @@ +{ + "status": "str", + "localtion": "str", + "url": "str" +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_ad_groups.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_ad_groups.json new file mode 100644 index 000000000000..e7ef472e3783 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_ad_groups.json @@ -0,0 +1,7 @@ +[ + { + "campaignId": 1, + "adGroupId": 1, + "name": "string" + } +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_campaigns.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_campaigns.json new file mode 100644 index 000000000000..51b88a4d0895 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_campaigns.json @@ -0,0 +1,42 @@ +[ + { + "campaignId": 1, + "name": "string", + "budget": 0, + "budgetType": "lifetime", + "startDate": "string", + "endDate": "string", + "state": "enabled", + "servingStatus": "asinNotBuyable", + "portfolioId": 0, + "bidOptimization": true, + "bidMultiplier": 0, + "bidAdjustments": [ + { + "bidAdjustmentPredicate": "placementGroupHome", + "bidAdjustmentPercent": 50 + }, + { + "bidAdjustmentPredicate": "placementGroupDetailPage", + "bidAdjustmentPercent": 50 + }, + { + "bidAdjustmentPredicate": "placementGroupOther", + "bidAdjustmentPercent": 50 + } + ], + "adFormat": "productCollection", + "creative": { + "brandName": "string", + "brandLogoAssetID": "string", + "brandLogoUrl": "string", + "headline": "string", + "asins": ["string"], + "shouldOptimizeAsins": false + }, + "landingPage": { + "pageType": "productList", + "url": "string" + } + } +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_keywords.json b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_keywords.json new file mode 100644 index 000000000000..76c079cc1c68 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/resource/http/response/sponsored_brands_keywords.json @@ -0,0 +1,12 @@ +[ + { + "keywordId": 1, + "adGroupId": 1, + "campaignId": 1, + "keywordText": "string", + "nativeLanguageKeyword": "string", + "matchType": "broad", + "state": "enabled", + "bid": 0 + } +] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index f8759493fe74..c4278cb6df5e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -1,4 +1,5 @@ connector_image: airbyte/source-amazon-seller-partner:dev +test_strictness_level: high acceptance_tests: spec: tests: @@ -23,120 +24,114 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" - timeout_seconds: 2400 + timeout_seconds: 3600 ignored_fields: GET_MERCHANT_LISTINGS_ALL_DATA: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_FLAT_FILE_OPEN_LISTINGS_DATA: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_MERCHANTS_LISTINGS_FYP_REPORT: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_MERCHANT_LISTINGS_DATA: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_MERCHANT_LISTINGS_INACTIVE_DATA: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" GET_XML_BROWSE_TREE_DATA: - name: "dataEndTime" - bypass_reason: changes frequently + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + ListFinancialEvents: + - name: "PostedBefore" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no exact_order: no extra_records: yes - # TODO: Add records for empty streams - https://github.com/airbytehq/airbyte/issues/21555 empty_streams: - name: GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING - bypass_reason: "no access and no data" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_ORDER_REPORT_DATA_SHIPPING - bypass_reason: "no access and no data" - - name: GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_SELLER_FEEDBACK_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_LEDGER_DETAIL_VIEW_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_AFN_INVENTORY_DATA_BY_COUNTRY - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_VENDOR_SALES_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_SNS_FORECAST_DATA - bypass_reason: "no records" - - name: GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_AFN_INVENTORY_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_MERCHANT_CANCELLED_LISTINGS_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_LEDGER_SUMMARY_VIEW_DATA - bypass_reason: "no records" - - name: GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: VendorDirectFulfillmentShipping - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_VENDOR_INVENTORY_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_SNS_PERFORMANCE_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_INVENTORY_PLANNING_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_STORAGE_FEE_CHARGES_DATA - bypass_reason: "no records" - - name: GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_SALES_AND_TRAFFIC_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_STRANDED_INVENTORY_UI_DATA - bypass_reason: "no records" - - name: GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_FBA_REIMBURSEMENTS_DATA - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_VENDOR_REAL_TIME_INVENTORY_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: GET_VENDOR_TRAFFIC_REPORT - bypass_reason: "no records" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" + - name: VendorOrders + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json deleted file mode 100644 index 2ce8fbb81064..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT", - "json_schema": { - "title": "Brand Analytics Alternate Purchase Reports", - "description": "Brand Analytics Alternate Purchase Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "purchasedAsin": { - "type": ["null", "string"] - }, - "purchasedRank": { - "type": ["null", "integer"] - }, - "purchasedPct": { - "type": ["null", "number"] - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json deleted file mode 100644 index 4d7300e63157..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT", - "json_schema": { - "title": "Brand Analytics Item Comparison Reports", - "description": "Brand Analytics Item Comparison Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "comparedAsin": { - "type": ["null", "string"] - }, - "comparedRank": { - "type": ["null", "integer"] - }, - "comparedPct": { - "type": ["null", "number"] - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl index 05c70bd41237..9fc6ce2a88a2 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl @@ -1,53 +1,82 @@ -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Active", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254096} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254097} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2023-11-15"}, "emitted_at": 1690214254099} -{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "1000", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2023-11-15"}, "emitted_at": 1690217648401} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2023-11-15"}, "emitted_at": 1690219384531} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 11, 2022", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2023-11-15"}, "emitted_at": 1690219384532} -{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2023-11-15"}, "emitted_at": 1690220838938} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2023-11-15"}, "emitted_at": 1690223127427} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2023-11-15"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2023-11-15"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2023-11-15"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2023-11-15"}, "emitted_at": 1690223127429} -{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-4052057-4266618", "EarliestShipDate": "2022-07-25T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-26T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-22T20:25:05Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "7.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-09-01T13:16:42Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1691499338977} -{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "00860509139506", "LastUpdateDate": "2022-09-01T13:16:42Z", "AmazonOrderId": "112-4052057-4266618"}, "emitted_at": 1691499343416} -{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte T-Shirt Black", "FNSKU": "X0041NMBPF", "Merchant SKU": "IA-VREM-8L92", "ASIN": "B0CJ5Q3NLP", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "15.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "", "dataEndTime": "2023-11-22"}, "emitted_at": 1700672280561} -{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte Merch White", "FNSKU": "X003X1FG67", "Merchant SKU": "KW-J7BQ-WNKL", "ASIN": "B0CDLLJ5VV", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "10.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "0.1736 ft3", "dataEndTime": "2023-11-22"}, "emitted_at": 1700672280561} -{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte T-Shirt Black", "FNSKU": "X0041NMBPF", "Merchant SKU": "IA-VREM-8L92", "ASIN": "B0CJ5Q3NLP", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "15.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "", "dataEndTime": "2023-11-22"}, "emitted_at": 1700672342102} -{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte Merch White", "FNSKU": "X003X1FG67", "Merchant SKU": "KW-J7BQ-WNKL", "ASIN": "B0CDLLJ5VV", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "10.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "0.1736 ft3", "dataEndTime": "2023-11-22"}, "emitted_at": 1700672342102} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": "2023-10-16T22:51:31+00:00", "settlement-end-date": "2023-11-13T22:51:31+00:00", "deposit-date": "2023-11-15T22:51:31+00:00", "total-amount": "-39.99", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-11-13"}, "emitted_at": 1700672582202} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-16T22:51:31+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-27.54", "dataEndTime": "2023-11-13"}, "emitted_at": 1700672582203} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-11-09T18:44:35+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-11-13"}, "emitted_at": 1700672582203} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Successful charge", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-17T00:01:09+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "27.54 ", "dataEndTime": "2023-11-13"}, "emitted_at": 1700672582203} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18834943411", "settlement-start-date": "2023-10-02T22:51:31+00:00", "settlement-end-date": "2023-10-16T22:51:31+00:00", "deposit-date": "2023-10-18T22:51:31+00:00", "total-amount": "-27.54", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-16"}, "emitted_at": 1700672613327} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18834943411", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-09T20:49:19+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-10-16"}, "emitted_at": 1700672613327} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18834943411", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Previous Reserve Amount Balance", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-02T22:58:21+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "12.45", "dataEndTime": "2023-10-16"}, "emitted_at": 1700672613327} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "2023-09-18T22:51:31+00:00", "settlement-end-date": "2023-10-02T22:51:31+00:00", "deposit-date": "2023-10-04T22:51:31+00:00", "total-amount": "0.00", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644700} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Order", "order-id": "111-1308361-8778604", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "D7vNnKlKr", "marketplace-name": "Amazon.com", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "MFN", "posted-date": "2023-09-26T12:06:28+00:00", "order-item-code": "85435093931281", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "IA-VREM-8L92", "quantity-purchased": "1", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644700} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Order", "order-id": "111-1308361-8778604", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "D7vNnKlKr", "marketplace-name": "Amazon.com", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "MFN", "posted-date": "2023-09-26T12:06:28+00:00", "order-item-code": "85435093931281", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "IA-VREM-8L92", "quantity-purchased": "", "price-type": "Principal", "price-amount": "15.00", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644700} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Order", "order-id": "111-1308361-8778604", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "D7vNnKlKr", "marketplace-name": "Amazon.com", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "MFN", "posted-date": "2023-09-26T12:06:28+00:00", "order-item-code": "85435093931281", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "IA-VREM-8L92", "quantity-purchased": "", "price-type": "Tax", "price-amount": "0.86", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644700} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Order", "order-id": "111-1308361-8778604", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "D7vNnKlKr", "marketplace-name": "Amazon.com", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "MFN", "posted-date": "2023-09-26T12:06:28+00:00", "order-item-code": "85435093931281", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "IA-VREM-8L92", "quantity-purchased": "", "price-type": "MarketplaceFacilitatorTax-Principal", "price-amount": "-0.86", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644700} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Order", "order-id": "111-1308361-8778604", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "D7vNnKlKr", "marketplace-name": "Amazon.com", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "MFN", "posted-date": "2023-09-26T12:06:28+00:00", "order-item-code": "85435093931281", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "IA-VREM-8L92", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "Commission", "item-related-fee-amount": "-2.55", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644701} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-09-18T22:51:31+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644701} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Current Reserve Amount", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-02T22:58:21+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-12.45", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644701} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18654297941", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Successful charge", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-09-19T02:17:15+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "39.99 ", "dataEndTime": "2023-10-02"}, "emitted_at": 1700672644701} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18560892581", "settlement-start-date": "2023-08-21T22:51:30+00:00", "settlement-end-date": "2023-09-18T22:51:31+00:00", "deposit-date": "2023-09-20T22:51:31+00:00", "total-amount": "-39.99", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-09-18"}, "emitted_at": 1700672676034} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18560892581", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-08-21T22:51:30+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-44.00", "dataEndTime": "2023-09-18"}, "emitted_at": 1700672676035} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18560892581", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-09-09T19:10:06+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-09-18"}, "emitted_at": 1700672676035} -{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18560892581", "settlement-start-date": "", "settlement-end-date": "", "deposit-date": "", "total-amount": "", "currency": "", "transaction-type": "Successful charge", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-08-22T02:11:48+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "44.00 ", "dataEndTime": "2023-09-18"}, "emitted_at": 1700672676035} -{"stream": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2023-11-22"}, "emitted_at": 1700673220599} -{"stream": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2023-11-22"}, "emitted_at": 1700673282494} -{"stream": "ListFinancialEvents", "data": {"ShipmentEventList": [{"AmazonOrderId": "111-1308361-8778604", "MarketplaceName": "Amazon.com", "PostedDate": "2023-09-26T12:06:28Z", "ShipmentItemList": [{"SellerSKU": "IA-VREM-8L92", "OrderItemId": "85435093931281", "QuantityShipped": 1, "ItemChargeList": [{"ChargeType": "Principal", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 15.0}}, {"ChargeType": "Tax", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.86}}, {"ChargeType": "GiftWrap", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"ChargeType": "GiftWrapTax", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"ChargeType": "ShippingCharge", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"ChargeType": "ShippingTax", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}], "ItemFeeList": [{"FeeType": "Commission", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -2.55}}, {"FeeType": "FixedClosingFee", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"FeeType": "GiftwrapCommission", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"FeeType": "ShippingHB", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}, {"FeeType": "VariableClosingFee", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}}], "ItemTaxWithheldList": [{"TaxCollectionModel": "MarketplaceFacilitator", "TaxesWithheld": [{"ChargeType": "MarketplaceFacilitatorTax-Principal", "ChargeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -0.86}}]}]}]}], "ShipmentSettleEventList": [], "RefundEventList": [], "GuaranteeClaimEventList": [], "ChargebackEventList": [], "PayWithAmazonEventList": [], "ServiceProviderCreditEventList": [], "RetrochargeEventList": [], "RentalTransactionEventList": [], "PerformanceBondRefundEventList": [], "ProductAdsPaymentEventList": [], "ServiceFeeEventList": [{"FeeList": [{"FeeType": "FBAInboundTransportationFee", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -4.01}}]}, {"FeeList": [{"FeeType": "Subscription", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}}]}, {"FeeList": [{"FeeType": "Subscription", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}}]}, {"FeeList": [{"FeeType": "Subscription", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}}]}, {"FeeList": [{"FeeType": "Subscription", "FeeAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}}]}], "SellerDealPaymentEventList": [], "DebtRecoveryEventList": [{"DebtRecoveryType": "DebtPayment", "RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 44.0}, "DebtRecoveryItemList": [{"RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}, "OriginalAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "GroupBeginDate": "2023-08-07T22:51:31Z", "GroupEndDate": "2023-08-21T22:51:30Z"}, {"RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 4.01}, "OriginalAmount": {"CurrencyCode": "USD", "CurrencyAmount": -4.01}, "GroupBeginDate": "2022-08-08T22:51:31Z", "GroupEndDate": "2023-08-07T22:51:31Z"}], "ChargeInstrumentList": [{"Description": "MasterCard", "Tail": "4832", "Amount": {"CurrencyCode": "USD", "CurrencyAmount": 44.0}}]}, {"DebtRecoveryType": "DebtPayment", "RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}, "DebtRecoveryItemList": [{"RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}, "OriginalAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "GroupBeginDate": "2023-08-21T22:51:30Z", "GroupEndDate": "2023-09-18T22:51:31Z"}], "ChargeInstrumentList": [{"Description": "MasterCard", "Tail": "4832", "Amount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}}]}, {"DebtRecoveryType": "DebtPayment", "RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 27.54}, "DebtRecoveryItemList": [{"RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 27.54}, "OriginalAmount": {"CurrencyCode": "USD", "CurrencyAmount": -27.54}, "GroupBeginDate": "2023-10-02T22:51:31Z", "GroupEndDate": "2023-10-16T22:51:31Z"}], "ChargeInstrumentList": [{"Description": "MasterCard", "Tail": "4832", "Amount": {"CurrencyCode": "USD", "CurrencyAmount": 27.54}}]}, {"DebtRecoveryType": "DebtPayment", "RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}, "DebtRecoveryItemList": [{"RecoveryAmount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}, "OriginalAmount": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "GroupBeginDate": "2023-10-16T22:51:31Z", "GroupEndDate": "2023-11-13T22:51:31Z"}], "ChargeInstrumentList": [{"Description": "MasterCard", "Tail": "4832", "Amount": {"CurrencyCode": "USD", "CurrencyAmount": 39.99}}]}], "LoanServicingEventList": [], "AdjustmentEventList": [{"AdjustmentType": "ReserveCredit", "PostedDate": "2023-10-02T22:58:21Z", "AdjustmentAmount": {"CurrencyCode": "USD", "CurrencyAmount": 12.45}}, {"AdjustmentType": "ReserveDebit", "PostedDate": "2023-10-02T22:58:21Z", "AdjustmentAmount": {"CurrencyCode": "USD", "CurrencyAmount": -12.45}}], "SAFETReimbursementEventList": [], "SellerReviewEnrollmentPaymentEventList": [], "FBALiquidationEventList": [], "CouponPaymentEventList": [], "ImagingServicesFeeEventList": [], "NetworkComminglingTransactionEventList": [], "AffordabilityExpenseEventList": [], "AffordabilityExpenseReversalEventList": [], "RemovalShipmentEventList": [], "RemovalShipmentAdjustmentEventList": [], "TrialShipmentEventList": [], "TDSReimbursementEventList": [], "AdhocDisbursementEventList": [], "TaxWithholdingEventList": [], "ChargeRefundEventList": [], "FailedAdhocDisbursementEventList": [], "ValueAddedServiceChargeEventList": [], "CapacityReservationBillingEventList": []}, "emitted_at": 1700673369722} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "OjA9atYr_0qd8Aj7QZ0Lgwuyh2CfzzC3BxpBlmjWjQ4", "ProcessingStatus": "Open", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FinancialEventGroupStart": "2023-11-13T22:51:31Z"}, "emitted_at": 1700673439533} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "AedbAPByjFtRD7l9BGVAdv5J0TcNN_yVv3fE86WmPOw", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FundTransferDate": "2023-11-13T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -27.54}, "FinancialEventGroupStart": "2023-10-16T22:51:31Z", "FinancialEventGroupEnd": "2023-11-13T22:51:31Z"}, "emitted_at": 1700673439534} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "kknsejlnfUQp0IcWLl31D87wp1agIXGRvWH4XAwKif8", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -27.54}, "FundTransferDate": "2023-10-16T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "FinancialEventGroupStart": "2023-10-02T22:51:31Z", "FinancialEventGroupEnd": "2023-10-16T22:51:31Z"}, "emitted_at": 1700673439534} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "ufeBK7q-aynnP0RL06gMGe56ulqvLSar2gTJD0of53c", "ProcessingStatus": "Closed", "FundTransferStatus": "Failed", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "FundTransferDate": "2023-10-02T22:58:22Z", "AccountTail": "045", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FinancialEventGroupStart": "2023-09-18T22:51:31Z", "FinancialEventGroupEnd": "2023-10-02T22:51:31Z"}, "emitted_at": 1700673439534} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "0dFlFdmLOy4RYkE8EMtOEuQHTbRYpeGGr0wDnRFR4A0", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FundTransferDate": "2023-09-18T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -44.0}, "FinancialEventGroupStart": "2023-08-21T22:51:30Z", "FinancialEventGroupEnd": "2023-09-18T22:51:31Z"}, "emitted_at": 1700673439534} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "SwmeJZ8-W1YO2jMTsK7pOUliBFcUeVYOp84Zo98oA8A", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FundTransferDate": "2023-08-21T22:51:30Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -4.01}, "FinancialEventGroupStart": "2023-08-07T22:51:31Z", "FinancialEventGroupEnd": "2023-08-21T22:51:30Z"}, "emitted_at": 1700673439534} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "biM60XKT9qekhLpYdH9-ktjaaCDakRl5bhkXarpufys", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -4.01}, "FundTransferDate": "2023-08-07T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -58.86}, "FinancialEventGroupStart": "2022-08-08T22:51:31Z", "FinancialEventGroupEnd": "2023-08-07T22:51:31Z"}, "emitted_at": 1700673439534} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355646011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Bananas - en_US", "browseNodeStoreContextName": "Bananas - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011,20355646011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US,Bananas - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2023-11-22"}, "emitted_at": 1700673810076} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355647011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Grapes - en_US", "browseNodeStoreContextName": "Grapes - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011,20355647011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US,Grapes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2023-11-22"}, "emitted_at": 1700673810076} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Active", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478279} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05T08:09:12-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08T03:50:23-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11T01:48:47-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Airbyte Merch White", "item-description": "Airbyte T-short", "listing-id": "0803A3SAML1", "seller-sku": "KW-J7BQ-WNKL", "price": "10", "quantity": "", "open-date": "2023-08-03T02:26:19-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CDLLJ5VV", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CDLLJ5VV", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11T01:16:54-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05T08:00:10-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "1000", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968460244} +{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "IA-VREM-8L92", "asin": "B0CJ5Q3NLP", "price": "15.00", "quantity": "", "Business Price": "", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968460245} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-29", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968785470} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-11", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968785473} +{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968964616} +{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "Business Price": "", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968964618} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05T08:09:12-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137910} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08T03:50:23-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11T01:48:47-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Airbyte Merch White", "item-description": "Airbyte T-short", "listing-id": "0803A3SAML1", "seller-sku": "KW-J7BQ-WNKL", "price": "10", "quantity": "", "open-date": "2023-08-03T02:26:19-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CDLLJ5VV", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CDLLJ5VV", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11T01:16:54-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05T08:00:10-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "111-1225255-7785053", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-15T22:08:15Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-18T22:54:07Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-3632856-2922613", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-17T07:44:26Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-22T08:23:04Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-8462063-1469066", "EarliestShipDate": "2022-07-25T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-26T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-23T18:45:44Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-23T18:46:16Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-3281105-7707448", "EarliestShipDate": "2022-07-26T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-27T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-25T16:07:42Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-25T16:13:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-3669120-1845053", "EarliestShipDate": "2022-07-15T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-16T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-14T21:59:53Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "10.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-26T07:16:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-1507758-0081841", "EarliestShipDate": "2022-07-15T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-16T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-14T20:22:16Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "10.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-26T07:22:46Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-8121041-0876267", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-18T04:26:52Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "14.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-28T07:22:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "49158270219090", "LastUpdateDate": "2022-07-18T22:54:07Z", "AmazonOrderId": "111-1225255-7785053"}, "emitted_at": 1701969226265} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "37736574199610", "LastUpdateDate": "2022-07-22T08:23:04Z", "AmazonOrderId": "112-3632856-2922613"}, "emitted_at": 1701969227457} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "65706488326346", "LastUpdateDate": "2022-07-23T18:46:16Z", "AmazonOrderId": "113-8462063-1469066"}, "emitted_at": 1701969228659} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "08960455780074", "LastUpdateDate": "2022-07-25T16:13:14Z", "AmazonOrderId": "113-3281105-7707448"}, "emitted_at": 1701969229850} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "33405118899762", "LastUpdateDate": "2022-07-26T07:16:14Z", "AmazonOrderId": "112-3669120-1845053"}, "emitted_at": 1701969231047} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "36800179130578", "LastUpdateDate": "2022-07-26T07:22:46Z", "AmazonOrderId": "113-1507758-0081841"}, "emitted_at": 1701969232250} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "65043207929194", "LastUpdateDate": "2022-07-28T07:22:14Z", "AmazonOrderId": "113-8121041-0876267"}, "emitted_at": 1701969233471} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.09"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "26165617935794", "LastUpdateDate": "2022-07-28T07:24:14Z", "AmazonOrderId": "114-3041148-1777835"}, "emitted_at": 1701969234665} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "5.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "49051239848578", "LastUpdateDate": "2022-07-28T07:42:43Z", "AmazonOrderId": "112-3720233-8146637"}, "emitted_at": 1701969235896} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "36347967018074", "LastUpdateDate": "2022-07-28T07:44:16Z", "AmazonOrderId": "111-9754278-6869864"}, "emitted_at": 1701969237108} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "25578504674962", "LastUpdateDate": "2022-07-28T07:52:23Z", "AmazonOrderId": "114-4026932-3219457"}, "emitted_at": 1701969238326} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "00770178005186", "LastUpdateDate": "2022-07-28T08:07:41Z", "AmazonOrderId": "112-1098428-3787449"}, "emitted_at": 1701969239527} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "51815408701706", "LastUpdateDate": "2022-07-29T07:27:14Z", "AmazonOrderId": "112-8173974-4673832"}, "emitted_at": 1701969240733} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "true", "BuyerCancelReason": "REASON_LEFT_UNSPECIFIED"}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "41300609058346", "LastUpdateDate": "2022-07-29T07:50:18Z", "AmazonOrderId": "114-5642155-9428269"}, "emitted_at": 1701969241937} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "64356568394218", "LastUpdateDate": "2022-07-29T08:19:16Z", "AmazonOrderId": "113-8871452-8288246"}, "emitted_at": 1701969243138} +{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte T-Shirt Black", "FNSKU": "X0041NMBPF", "Merchant SKU": "IA-VREM-8L92", "ASIN": "B0CJ5Q3NLP", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "15.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969512824} +{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte Merch White", "FNSKU": "X003X1FG67", "Merchant SKU": "KW-J7BQ-WNKL", "ASIN": "B0CDLLJ5VV", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "10.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "0.1736 ft3", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969512826} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": "2023-11-13T22:51:31+00:00", "settlement-end-date": "2023-12-11T22:51:31+00:00", "deposit-date": "2023-12-13T22:51:31+00:00", "total-amount": "-39.99", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": null, "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-12-11"}, "emitted_at": 1707819017802} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-11-13T22:51:31+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-12-11"}, "emitted_at": 1707819017804} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-12-09T20:02:53+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-12-11"}, "emitted_at": 1707819017805} +{"stream": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701976405556} +{"stream": "ListFinancialEvents", "data": {"ShipmentEventList": [], "ShipmentSettleEventList": [], "RefundEventList": [], "GuaranteeClaimEventList": [], "ChargebackEventList": [], "PayWithAmazonEventList": [], "ServiceProviderCreditEventList": [], "RetrochargeEventList": [], "RentalTransactionEventList": [], "PerformanceBondRefundEventList": [], "ProductAdsPaymentEventList": [{"postedDate": "2022-07-28T20:06:07Z", "transactionType": "Charge", "invoiceId": "TR1T7Z7DR-1", "baseValue": {"CurrencyCode": "USD", "CurrencyAmount": -9.08}, "taxValue": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "transactionValue": {"CurrencyCode": "USD", "CurrencyAmount": -9.08}}], "ServiceFeeEventList": [], "SellerDealPaymentEventList": [], "DebtRecoveryEventList": [], "LoanServicingEventList": [], "AdjustmentEventList": [], "SAFETReimbursementEventList": [], "SellerReviewEnrollmentPaymentEventList": [], "FBALiquidationEventList": [], "CouponPaymentEventList": [], "ImagingServicesFeeEventList": [], "NetworkComminglingTransactionEventList": [], "AffordabilityExpenseEventList": [], "AffordabilityExpenseReversalEventList": [], "RemovalShipmentEventList": [], "RemovalShipmentAdjustmentEventList": [], "TrialShipmentEventList": [], "TDSReimbursementEventList": [], "AdhocDisbursementEventList": [], "TaxWithholdingEventList": [], "ChargeRefundEventList": [], "FailedAdhocDisbursementEventList": [], "ValueAddedServiceChargeEventList": [], "CapacityReservationBillingEventList": [], "PostedBefore": "2022-07-31T00:00:00Z"}, "emitted_at": 1701976465145} +{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "6uFLEEa3LQgyvcccMnVQ4Bj-I5zkOVNoM41q8leJzLk", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -58.86}, "FundTransferDate": "2022-08-08T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FinancialEventGroupStart": "2021-07-26T22:51:30Z", "FinancialEventGroupEnd": "2022-08-08T22:51:31Z"}, "emitted_at": 1701976502869} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355628011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Vegetables - en_US", "browseNodeStoreContextName": "Vegetables - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US", "hasChildren": "true", "childNodes": {"count": "3", "id": ["20355644011", "20355643011", "20355645011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355644011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Artichokes - en_US", "browseNodeStoreContextName": "Artichokes - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355644011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Artichokes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355643011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Celery - en_US", "browseNodeStoreContextName": "Celery - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355643011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Celery - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355645011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Eggplant - en_US", "browseNodeStoreContextName": "Eggplant - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355645011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Eggplant - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354445011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test2", "browseNodeStoreContextName": "Test2", "browsePathById": "19162063011,19162064011,21354445011", "browsePathByName": "Yggdrasil,Test2", "hasChildren": "true", "childNodes": {"count": "1", "id": ["21354444011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354444011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test1", "browseNodeStoreContextName": "Test1", "browsePathById": "19162063011,19162064011,21354445011,21354444011", "browsePathByName": "Yggdrasil,Test2,Test1", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676488} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8871452-8288246", "merchant-order-id": "", "purchase-date": "2022-07-18T18:52:47+00:00", "last-updated-date": "2022-07-29T08:19:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "KERRVILLE", "ship-state": "TX", "ship-postal-code": "78028-6411", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698682} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-5642155-9428269", "merchant-order-id": "", "purchase-date": "2022-07-18T17:15:20+00:00", "last-updated-date": "2022-07-29T07:50:18+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "SUNNY ISLES BEACH", "ship-state": "FL", "ship-postal-code": "33160-2404", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698685} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-8173974-4673832", "merchant-order-id": "", "purchase-date": "2022-07-18T19:42:56+00:00", "last-updated-date": "2022-07-29T07:27:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "MANSFIELD", "ship-state": "PA", "ship-postal-code": "16933-1252", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-1098428-3787449", "merchant-order-id": "", "purchase-date": "2022-07-17T17:49:26+00:00", "last-updated-date": "2022-07-28T08:07:41+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Winnebago", "ship-state": "IL", "ship-postal-code": "61088", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-4026932-3219457", "merchant-order-id": "", "purchase-date": "2022-07-17T17:53:01+00:00", "last-updated-date": "2022-07-28T07:52:23+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Clinton Township", "ship-state": "MI", "ship-postal-code": "48035", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "111-9754278-6869864", "merchant-order-id": "", "purchase-date": "2022-07-15T18:30:29+00:00", "last-updated-date": "2022-07-28T07:44:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "ROTONDA WEST", "ship-state": "FLORIDA", "ship-postal-code": "33947-1801", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3720233-8146637", "merchant-order-id": "", "purchase-date": "2022-07-18T02:30:11+00:00", "last-updated-date": "2022-07-28T07:42:43+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "5.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "WINFIELD", "ship-state": "MO", "ship-postal-code": "63389-2051", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-3041148-1777835", "merchant-order-id": "", "purchase-date": "2022-07-18T00:32:07+00:00", "last-updated-date": "2022-07-28T07:24:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "0.09", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BURBANK", "ship-state": "IL", "ship-postal-code": "60459-3101", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8121041-0876267", "merchant-order-id": "", "purchase-date": "2022-07-18T04:26:52+00:00", "last-updated-date": "2022-07-28T07:22:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "SEASIDE", "ship-state": "CA", "ship-postal-code": "93955-5450", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-1507758-0081841", "merchant-order-id": "", "purchase-date": "2022-07-14T20:22:16+00:00", "last-updated-date": "2022-07-26T07:22:46+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "PORT ARTHUR", "ship-state": "TX", "ship-postal-code": "77642-6487", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3669120-1845053", "merchant-order-id": "", "purchase-date": "2022-07-14T21:59:53+00:00", "last-updated-date": "2022-07-26T07:16:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "North Andover", "ship-state": "MA", "ship-postal-code": "01845", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-3281105-7707448", "merchant-order-id": "", "purchase-date": "2022-07-25T16:07:42+00:00", "last-updated-date": "2022-07-25T16:13:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "NEW YORK", "ship-state": "NY", "ship-postal-code": "10023-7107", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8462063-1469066", "merchant-order-id": "", "purchase-date": "2022-07-23T18:45:44+00:00", "last-updated-date": "2022-07-23T18:46:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "MARYSVILLE", "ship-state": "WA", "ship-postal-code": "98271-9030", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3632856-2922613", "merchant-order-id": "", "purchase-date": "2022-07-17T07:44:26+00:00", "last-updated-date": "2022-07-22T08:23:04+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BRONX", "ship-state": "NY", "ship-postal-code": "10475-4302", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "111-1225255-7785053", "merchant-order-id": "", "purchase-date": "2022-07-15T22:08:15+00:00", "last-updated-date": "2022-07-18T22:54:07+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "TAMPA", "ship-state": "FL", "ship-postal-code": "33615-4914", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698689} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-4470913-2725847", "PurchaseDate": "2022-07-29T08:14:41+00:00", "LastUpdatedDate": "2022-08-11T07:34:27+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "BRONX", "State": "NY", "PostalCode": "10462-5935", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "58620406098794", "ASIN": "B000VHRNUW", "SKU": "MP-V4RG-EDEY", "ProductName": "House Foods, Organic Firm Tofu, 14 oz", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "5.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188131} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-9288908-5020240", "PurchaseDate": "2022-07-29T04:44:18+00:00", "LastUpdatedDate": "2022-08-09T07:54:14+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "PERRY", "State": "UT", "PostalCode": "84302-4853", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "02473315381338", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": [{"Type": "Principal", "Amount": {"currency": "USD", "value": "35.0"}}, {"Type": "Tax", "Amount": {"currency": "USD", "value": "1.05"}}]}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "114-6340460-9317849", "PurchaseDate": "2022-07-28T20:31:53+00:00", "LastUpdatedDate": "2022-08-09T07:44:43+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "IRVINE", "State": "CA", "PostalCode": "92620-2213", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "07455825901354", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "7.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "114-9668619-2274637", "PurchaseDate": "2022-07-28T17:51:54+00:00", "LastUpdatedDate": "2022-08-09T07:12:17+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "BROOKSVILLE", "State": "ME", "PostalCode": "04617-3551", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "02835442928154", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "7.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-1503149-6333038", "PurchaseDate": "2022-07-28T16:36:33+00:00", "LastUpdatedDate": "2022-08-09T07:36:17+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "Woodbury", "State": "NJ", "PostalCode": "08096", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "17238515541858", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "14.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-4470913-2725847", "merchant-order-id": "", "purchase-date": "2022-07-29T08:14:41+00:00", "last-updated-date": "2022-08-11T07:34:27+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Organic Firm Tofu, 14 oz", "sku": "MP-V4RG-EDEY", "asin": "B000VHRNUW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "5.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BRONX", "ship-state": "NY", "ship-postal-code": "10462-5935", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513197} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-9288908-5020240", "merchant-order-id": "", "purchase-date": "2022-07-29T04:44:18+00:00", "last-updated-date": "2022-08-09T07:54:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "35.0", "item-tax": "1.05", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "PERRY", "ship-state": "UT", "ship-postal-code": "84302-4853", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513199} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "114-6340460-9317849", "merchant-order-id": "", "purchase-date": "2022-07-28T20:31:53+00:00", "last-updated-date": "2022-08-09T07:44:43+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "IRVINE", "ship-state": "CA", "ship-postal-code": "92620-2213", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513199} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "114-9668619-2274637", "merchant-order-id": "", "purchase-date": "2022-07-28T17:51:54+00:00", "last-updated-date": "2022-08-09T07:12:17+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BROOKSVILLE", "ship-state": "ME", "ship-postal-code": "04617-3551", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513200} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-1503149-6333038", "merchant-order-id": "", "purchase-date": "2022-07-28T16:36:33+00:00", "last-updated-date": "2022-08-09T07:36:17+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Woodbury", "ship-state": "NJ", "ship-postal-code": "08096", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513200} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json index b572d1137add..9bf0b19e4d41 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json @@ -3,10 +3,10 @@ "type": "STREAM", "stream": { "stream_state": { - "LastUpdateDate": "2121-07-01T00:00:00Z" + "dataEndTime": "2121-07-01" }, "stream_descriptor": { - "name": "OrderItems" + "name": "GET_AFN_INVENTORY_DATA" } } }, @@ -14,10 +14,10 @@ "type": "STREAM", "stream": { "stream_state": { - "LastUpdateDate": "2121-07-01T00:00:00Z" + "dataEndTime": "2121-07-01" }, "stream_descriptor": { - "name": "Orders" + "name": "GET_AFN_INVENTORY_DATA_BY_COUNTRY" } } }, @@ -28,7 +28,7 @@ "dataEndTime": "2121-07-01" }, "stream_descriptor": { - "name": "GET_AFN_INVENTORY_DATA" + "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL" } } }, @@ -39,7 +39,370 @@ "dataEndTime": "2121-07-01" }, "stream_descriptor": { - "name": "GET_AFN_INVENTORY_DATA_BY_COUNTRY" + "name": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "queryEndDate": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_INVENTORY_PLANNING_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_REIMBURSEMENTS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_SNS_FORECAST_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_SNS_PERFORMANCE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_STORAGE_FEE_CHARGES_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_OPEN_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "Date": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_LEDGER_DETAIL_VIEW_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_LEDGER_SUMMARY_VIEW_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANTS_LISTINGS_FYP_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_CANCELLED_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_ALL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_INACTIVE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_ORDER_REPORT_DATA_SHIPPING" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "queryEndDate": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_SALES_AND_TRAFFIC_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_SELLER_FEEDBACK_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_STRANDED_INVENTORY_UI_DATA" } } }, @@ -53,5 +416,137 @@ "name": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_INVENTORY_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_REAL_TIME_INVENTORY_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_SALES_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_TRAFFIC_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdatedDate": "2121-07-01T00:00:00+00:00" + }, + "stream_descriptor": { + "name": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_XML_BROWSE_TREE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "FinancialEventGroupStart": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "ListFinancialEventGroups" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "PostedBefore": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "ListFinancialEvents" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdateDate": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "OrderItems" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdateDate": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "Orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdBefore": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "VendorDirectFulfillmentShipping" + } + } } ] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index d53252191baf..ee7f33aa3ce5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions +from source_amazon_seller_partner.run import run if __name__ == "__main__": - source = SourceAmazonSellerPartner() - MigrateAccountType.migrate(sys.argv[1:], source) - MigrateReportOptions.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index bac07225e362..ffbc4a51e268 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -8,20 +8,24 @@ data: - https://sandbox.sellingpartnerapi-fe.amazon.com - https://sandbox.sellingpartnerapi-na.amazon.com ab_internal: - ql: 200 - sl: 100 + ql: 400 + sl: 200 connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 2.5.0 + dockerImageTag: 3.5.0 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner icon: amazonsellerpartner.svg license: MIT name: Amazon Seller Partner + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-seller-partner registries: cloud: enabled: true @@ -39,6 +43,13 @@ data: 2.0.0: message: "Deprecated FBA reports will be removed permanently from Cloud and Brand Analytics Reports will be removed temporarily. Updates on Brand Analytics Reports can be tracked here: [#32353](https://github.com/airbytehq/airbyte/issues/32353)" upgradeDeadline: "2023-12-11" + 3.0.0: + message: + Streams 'GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL' and 'GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL' now have updated schemas. + Streams 'GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL', 'GET_LEDGER_DETAIL_VIEW_DATA', 'GET_MERCHANTS_LISTINGS_FYP_REPORT', + 'GET_STRANDED_INVENTORY_UI_DATA', and 'GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE' now have date-time formatted fields. + Users will need to refresh the source schemas and reset these streams after upgrading. + upgradeDeadline: "2024-01-12" supportLevel: community tags: - language:python diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 9b4396a6c472..e75c0a55146b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -5,22 +5,35 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "xmltodict~=0.12"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "xmltodict~=0.12", "dateparser==1.2.0"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "freezegun==1.2.2"] setup( + entry_points={ + "console_scripts": [ + "source-amazon-seller-partner=source_amazon_seller_partner.run:run", + ], + }, name="source_amazon_seller_partner", description="Source implementation for Amazon Seller Partner.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index fd0dc7e33b7b..4bdaab19df35 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from typing import Any, Mapping import pendulum diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py index 093fda40bd45..0267e3af07a1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json import logging from typing import Any, List, Mapping diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py index e8dc0e56c1c8..4b0e1e99bbe5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """ Country marketplaceId Country code Canada A2EUQ1WTGCTBG2 CA diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py new file mode 100644 index 000000000000..538cf70c8afc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_seller_partner import SourceAmazonSellerPartner +from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions + + +def run(): + source = SourceAmazonSellerPartner() + MigrateAccountType.migrate(sys.argv[1:], source) + MigrateReportOptions.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json index f40efcd6a819..5e771d793471 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json @@ -9,7 +9,7 @@ "asin": { "type": ["null", "string"] }, "condition-type": { "type": ["null", "string"] }, "Warehouse-Condition-code": { "type": ["null", "string"] }, - "Quantity Available": { "type": ["null", "number"] }, + "Quantity Available": { "type": ["null", "string"] }, "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json index d7cc3190ed77..24c7bdabbfc9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json @@ -9,7 +9,7 @@ "asin": { "type": ["null", "string"] }, "condition-type": { "type": ["null", "string"] }, "country": { "type": ["null", "string"] }, - "quantity-for-local-fulfillment": { "type": ["null", "number"] }, + "quantity-for-local-fulfillment": { "type": ["null", "string"] }, "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json index 552f03141420..e04864960619 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json @@ -48,7 +48,10 @@ "ship-promotion-discount": { "type": ["null", "string"] }, "carrier": { "type": ["null", "string"] }, "tracking-number": { "type": ["null", "string"] }, - "estimated-arrival-date": { "type": ["null", "string"] }, + "estimated-arrival-date": { + "type": ["null", "string"], + "format": "date-time" + }, "fulfillment-center-id": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, "sales-channel": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json deleted file mode 100644 index 48f44907c094..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "title": "Brand Analytics Alternate Purchase Reports", - "description": "Brand Analytics Alternate Purchase Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "purchasedAsin": { - "type": ["null", "string"] - }, - "purchasedRank": { - "type": ["null", "integer"] - }, - "purchasedPct": { - "type": ["null", "number"] - }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json deleted file mode 100644 index 901f2647849d..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "title": "Brand Analytics Item Comparison Reports", - "description": "Brand Analytics Item Comparison Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "comparedAsin": { - "type": ["null", "string"] - }, - "comparedRank": { - "type": ["null", "integer"] - }, - "comparedPct": { - "type": ["null", "number"] - }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json index 655e4af9eb76..4ba25d9eefa1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json @@ -27,6 +27,10 @@ "dataEndTime": { "type": ["null", "string"], "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json index c7cdcecd23de..06f26422fd69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json @@ -41,6 +41,10 @@ "dataEndTime": { "type": ["null", "string"], "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json index 99c2e5e97e31..65880173e0dc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json @@ -28,6 +28,10 @@ "dataEndTime": { "type": ["null", "string"], "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json index f49d4c25db77..9b5dd9d9bb53 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json @@ -7,96 +7,102 @@ "amazon-order-id": { "type": "string" }, - "asin": { - "type": ["null", "string"] + "merchant-order-id": { + "type": "string" }, - "currency": { + "purchase-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "last-updated-date": { + "type": "string", + "format": "date-time" + }, + "order-status": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, - "gift-wrap-price": { + "sales-channel": { "type": ["null", "string"] }, - "gift-wrap-tax": { + "order-channel": { "type": ["null", "string"] }, - "is-business-order": { + "ship-service-level": { "type": ["null", "string"] }, - "item-price": { + "product-name": { "type": ["null", "string"] }, - "item-promotion-discount": { + "sku": { "type": ["null", "string"] }, - "item-status": { + "asin": { "type": ["null", "string"] }, - "item-tax": { + "item-status": { "type": ["null", "string"] }, - "last-updated-date": { - "type": "string", - "format": "date-time" - }, - "merchant-order-id": { + "quantity": { "type": ["null", "string"] }, - "order-channel": { + "currency": { "type": ["null", "string"] }, - "order-status": { + "item-price": { "type": ["null", "string"] }, - "price-designation": { + "item-tax": { "type": ["null", "string"] }, - "product-name": { + "shipping-price": { "type": ["null", "string"] }, - "promotion-ids": { + "shipping-tax": { "type": ["null", "string"] }, - "purchase-date": { - "type": ["null", "string"], - "format": "date-time" + "gift-wrap-price": { + "type": ["null", "string"] }, - "purchase-order-number": { + "gift-wrap-tax": { "type": ["null", "string"] }, - "quantity": { + "item-promotion-discount": { "type": ["null", "string"] }, - "sales-channel": { + "ship-promotion-discount": { "type": ["null", "string"] }, "ship-city": { "type": ["null", "string"] }, - "ship-country": { + "ship-state": { "type": ["null", "string"] }, "ship-postal-code": { "type": ["null", "string"] }, - "ship-promotion-discount": { + "ship-country": { "type": ["null", "string"] }, - "ship-service-level": { + "promotion-ids": { "type": ["null", "string"] }, - "ship-state": { + "cpf": { "type": ["null", "string"] }, - "shipping-price": { + "is-business-order": { "type": ["null", "string"] }, - "shipping-tax": { + "purchase-order-number": { "type": ["null", "string"] }, - "sku": { + "price-designation": { + "type": ["null", "string"] + }, + "signature-confirmation-recommended": { "type": ["null", "string"] }, "dataEndTime": { diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json index 533abd6bfe0c..d64b129e6afc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json @@ -4,78 +4,63 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "order-id": { + "amazon-order-id": { "type": ["null", "string"] }, - "order-item-id": { + "merchant-order-id": { "type": ["null", "string"] }, "purchase-date": { "type": ["null", "string"], "format": "date-time" }, - "payments-date": { + "last-updated-date": { "type": ["null", "string"], "format": "date-time" }, - "buyer-email": { + "order-status": { "type": ["null", "string"] }, - "buyer-name": { + "fulfillment-channel": { "type": ["null", "string"] }, - "sku": { - "type": ["null", "string"] - }, - "product-name": { - "type": ["null", "string"] - }, - "quantity-purchased": { + "sales-channel": { "type": ["null", "string"] }, - "currency": { - "type": ["null", "string"] - }, - "item-price": { - "type": ["null", "string"] - }, - "shipping-price": { - "type": ["null", "string"] - }, - "item-tax": { + "order-channel": { "type": ["null", "string"] }, "ship-service-level": { "type": ["null", "string"] }, - "recipient-name": { + "product-name": { "type": ["null", "string"] }, - "ship-address-1": { + "sku": { "type": ["null", "string"] }, - "ship-address-2": { + "asin": { "type": ["null", "string"] }, - "ship-address-3": { + "item-status": { "type": ["null", "string"] }, - "ship-city": { + "quantity": { "type": ["null", "string"] }, - "ship-state": { + "currency": { "type": ["null", "string"] }, - "ship-postal-code": { + "item-price": { "type": ["null", "string"] }, - "ship-country": { + "item-tax": { "type": ["null", "string"] }, - "gift-wrap-type": { + "shipping-price": { "type": ["null", "string"] }, - "gift-message-text": { + "shipping-tax": { "type": ["null", "string"] }, "gift-wrap-price": { @@ -87,22 +72,25 @@ "item-promotion-discount": { "type": ["null", "string"] }, - "item-promotion-id": { + "ship-promotion-discount": { "type": ["null", "string"] }, - "shipping-promotion-discount": { + "ship-city": { "type": ["null", "string"] }, - "shipping-promotion-id": { + "ship-state": { "type": ["null", "string"] }, - "delivery-instructions": { + "ship-postal-code": { "type": ["null", "string"] }, - "order-channel": { + "ship-country": { + "type": ["null", "string"] + }, + "promotion-ids": { "type": ["null", "string"] }, - "order-channel-instance": { + "cpf": { "type": ["null", "string"] }, "is-business-order": { @@ -114,77 +102,7 @@ "price-designation": { "type": ["null", "string"] }, - "buyer-company-name": { - "type": ["null", "string"] - }, - "licensee-name": { - "type": ["null", "string"] - }, - "license-number": { - "type": ["null", "string"] - }, - "license-state": { - "type": ["null", "string"] - }, - "license-expiration-date": { - "type": ["null", "string"], - "format": "date-time" - }, - "Address-Type": { - "type": ["null", "string"] - }, - "Number-of-items": { - "type": ["null", "string"] - }, - "is-global-express": { - "type": ["null", "string"] - }, - "default-ship-from-address-name": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-1": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-2": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-3": { - "type": ["null", "string"] - }, - "default-ship-from-address-city": { - "type": ["null", "string"] - }, - "default-ship-from-address-state": { - "type": ["null", "string"] - }, - "default-ship-from-address-country": { - "type": ["null", "string"] - }, - "default-ship-from-address-postal-code": { - "type": ["null", "string"] - }, - "actual-ship-from-address-name": { - "type": ["null", "string"] - }, - "actual-ship-from-address-1": { - "type": ["null", "string"] - }, - "actual-ship-from-address-field-2": { - "type": ["null", "string"] - }, - "actual-ship-from-address-field-3": { - "type": ["null", "string"] - }, - "actual-ship-from-address-city": { - "type": ["null", "string"] - }, - "actual-ship-from-address-state": { - "type": ["null", "string"] - }, - "actual-ship-from-address-country": { - "type": ["null", "string"] - }, - "actual-ship-from-address-postal-code": { + "signature-confirmation-recommended": { "type": ["null", "string"] }, "dataEndTime": { diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json index 22a73e031942..1fd4d4de1f8b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json @@ -18,7 +18,7 @@ "Country": { "type": ["null", "string"] }, "Reconciled Quantity": { "type": ["null", "string"] }, "Unreconciled Quantity": { "type": ["null", "string"] }, - "Date and Time": { "type": ["null", "string"] }, + "Date and Time": { "type": ["null", "string"], "format": "date-time" }, "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json index 5b7d9aa7c908..88a3305e3165 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json @@ -4,7 +4,7 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "Date": { "type": ["null", "string"], "format": "date-time" }, + "Date": { "type": ["null", "string"], "format": "date" }, "FNSKU": { "type": ["null", "string"] }, "ASIN": { "type": ["null", "string"] }, "MSKU": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json index 09687d89cbfb..820f76581b69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json @@ -10,7 +10,7 @@ "ASIN": { "type": ["null", "string"] }, "Product name": { "type": ["null", "string"] }, "Condition": { "type": ["null", "string"] }, - "Status Change Date": { "type": ["null", "string"] }, + "Status Change Date": { "type": ["null", "string"], "format": "date" }, "Issue Description": { "type": ["null", "string"] }, "dataEndTime": { "type": ["null", "string"], "format": "date" } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json index d41c4cb69df1..d5863d6d3532 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json @@ -14,6 +14,9 @@ "childAsin": { "type": ["null", "string"] }, + "sku": { + "type": ["null", "string"] + }, "salesByAsin": { "type": "object", "properties": { diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json index ad8c53f56a3f..ffa44e5ee2a9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json @@ -5,9 +5,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "settlement-id": { "type": ["null", "string"] }, - "settlement-start-date": { "type": ["null", "string"] }, - "settlement-end-date": { "type": ["null", "string"] }, - "deposit-date": { "type": ["null", "string"] }, + "settlement-start-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "settlement-end-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "deposit-date": { "type": ["null", "string"], "format": "date-time" }, "total-amount": { "type": ["null", "string"] }, "currency": { "type": ["null", "string"] }, "transaction-type": { "type": ["null", "string"] }, @@ -21,7 +27,7 @@ "order-fee-type": { "type": ["null", "string"] }, "order-fee-amount": { "type": ["null", "string"] }, "fulfillment-id": { "type": ["null", "string"] }, - "posted-date": { "type": ["null", "string"] }, + "posted-date": { "type": ["null", "string"], "format": "date-time" }, "order-item-code": { "type": ["null", "string"] }, "merchant-order-item-id": { "type": ["null", "string"] }, "merchant-adjustment-item-id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json index 914ad310a1bd..dc6a332fcea4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json @@ -89,6 +89,10 @@ "dataEndTime": { "type": ["null", "string"], "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json index 266c115ccfd4..98afa6df8bc8 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json @@ -4,46 +4,27 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "netPureProductMarginAggregate": { - "type": ["null", "array"], - "items": { - "type": ["object"] - }, - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "netPureProductMargin": { - "type": ["null", "number"] - } - } + "startDate": { + "type": ["null", "string"], + "format": "date" }, - "netPureProductMarginByAsin": { - "type": ["null", "array"], - "items": { - "type": ["object"] - }, - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "netPureProductMargin": { - "type": ["null", "number"] - } - } + "endDate": { + "type": ["null", "string"], + "format": "date" + }, + "asin": { + "type": ["null", "string"] + }, + "netPureProductMargin": { + "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json index d927776b6d43..37623e3f5487 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json @@ -4,49 +4,27 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "reportSpecification": { - "type": ["null", "object"], - "properties": { - "reportType": { - "type": ["null", "string"] - }, - "dataStartTime": { - "type": ["null", "string"], - "format": "date-time" - }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date-time" - }, - "marketplaceIds": { - "type": ["null", "array"], - "items": { - "type": ["string"] - } - } - } + "startTime": { + "type": ["null", "string"], + "format": "date-time" }, - "reportData": { - "type": ["null", "array"], - "items": { - "type": ["object"] - }, - "properties": { - "startTime": { - "type": ["null", "string"], - "format": "date-time" - }, - "endTime": { - "type": ["null", "string"], - "format": "date-time" - }, - "asin": { - "type": ["null", "string"] - }, - "highlyAvailableInventory": { - "type": ["null", "integer"] - } - } + "endTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "asin": { + "type": ["null", "string"] + }, + "highlyAvailableInventory": { + "type": ["null", "integer"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json index 1fe51d9b53d6..f0fc0eac9364 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json @@ -17,6 +17,14 @@ }, "glanceViews": { "type": ["null", "integer"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json index aaaf073bf2e1..dc194dbe53ff 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json @@ -984,6 +984,10 @@ } } } + }, + "PostedBefore": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json index 56bae0c6f47b..73c80399a196 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json @@ -234,6 +234,10 @@ } } } + }, + "createdBefore": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorOrders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorOrders.json new file mode 100644 index 000000000000..c05d713e5e9e --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorOrders.json @@ -0,0 +1,354 @@ +{ + "title": "Vendor Orders", + "description": "All vendor purchase orders that were updated after a specified date", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "purchaseOrderNumber": { + "type": ["null", "string"] + }, + "purchaseOrderState": { + "type": ["null", "string"] + }, + "orderDetails": { + "type": ["null", "object"], + "properties": { + "purchaseOrderDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "purchaseOrderChangedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "purchaseOrderStateChangedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "purchaseOrderType": { + "type": ["null", "string"] + }, + "importDetails": { + "type": ["null", "object"], + "properties": { + "methodOfPayment": { + "type": ["null", "string"] + }, + "internationalCommercialTerms": { + "type": ["null", "string"] + }, + "portOfDelivery": { + "type": ["null", "string"] + }, + "importContainers": { + "type": ["null", "string"] + }, + "shippingInstructions": { + "type": ["null", "string"] + } + } + }, + "dealCode": { + "type": ["null", "string"] + }, + "paymentMethod": { + "type": ["null", "string"] + }, + "buyingParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxInfo": { + "type": ["null", "object"], + "properties": { + "taxType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + } + } + } + } + }, + "sellingParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxInfo": { + "type": ["null", "object"], + "properties": { + "taxType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + } + } + } + } + }, + "shipToParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxInfo": { + "type": ["null", "object"], + "properties": { + "taxType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + } + } + } + } + }, + "billToParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxInfo": { + "type": ["null", "object"], + "properties": { + "taxType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + } + } + } + } + }, + "shipWindow": { + "type": ["null", "string"] + }, + "deliveryWindow": { + "type": ["null", "string"] + }, + "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "itemSequenceNumber": { + "type": ["null", "string"] + }, + "amazonProductIdentifier": { + "type": ["null", "string"] + }, + "vendorProductIdentifier": { + "type": ["null", "string"] + }, + "orderedQuantity": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "unitOfMeasure": { + "type": ["null", "string"] + }, + "unitSize": { + "type": ["null", "integer"] + } + } + }, + "isBackOrderAllowed": { + "type": ["null", "boolean"] + }, + "netCost": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, + "listPrice": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + } + } + } + } + } + }, + "changedBefore": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 74dfdbd13738..91c2e1bd80ec 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import traceback from os import getenv from typing import Any, List, Mapping, Optional, Tuple @@ -10,12 +12,11 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError from source_amazon_seller_partner.auth import AWSAuthenticator from source_amazon_seller_partner.constants import get_marketplaces from source_amazon_seller_partner.streams import ( - BrandAnalyticsAlternatePurchaseReports, - BrandAnalyticsItemComparisonReports, BrandAnalyticsMarketBasketReports, BrandAnalyticsRepeatPurchaseReports, BrandAnalyticsSearchTermsReports, @@ -63,6 +64,7 @@ StrandedInventoryUiReport, VendorDirectFulfillmentShipping, VendorInventoryReports, + VendorOrders, VendorSalesReports, VendorTrafficReport, XmlAllOrdersDataByOrderDataGeneral, @@ -92,7 +94,7 @@ def _get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: "authenticator": auth, "replication_start_date": start_date, "marketplace_id": marketplace_id, - "period_in_days": config.get("period_in_days", 90), + "period_in_days": config.get("period_in_days", 30), "replication_end_date": config.get("replication_end_date"), } return stream_kwargs @@ -111,23 +113,26 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> self.validate_replication_dates(config) self.validate_stream_report_options(config) stream_kwargs = self._get_stream_kwargs(config) - orders_stream = Orders(**stream_kwargs) - next(orders_stream.read_records(sync_mode=SyncMode.full_refresh)) + + if config.get("account_type", "Seller") == "Seller": + stream_to_check = Orders(**stream_kwargs) + next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) + else: + stream_to_check = VendorOrders(**stream_kwargs) + stream_slices = list(stream_to_check.stream_slices(sync_mode=SyncMode.full_refresh)) + next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0])) return True, None except Exception as e: - # Validate Orders stream without data + # Validate stream without data if isinstance(e, StopIteration): return True, None - # Additional check, since Vendor-only accounts within Amazon Seller API will not pass the test without this exception - if "403 Client Error" in str(e): - stream_to_check = VendorSalesReports(**stream_kwargs) - next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - - error_message = e.response.json().get("error_description") if isinstance(e, HTTPError) else e - return False, error_message + if isinstance(e, HTTPError): + return False, e.response.json().get("error_description") + else: + error_message = "Caught unexpected exception during the check" + raise AirbyteTracedException(internal_message=error_message, message=error_message, exception=e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ @@ -178,6 +183,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: FbaInventoryPlaningReport, LedgerSummaryViewReport, FbaReimbursementsReports, + VendorOrders, ] # TODO: Remove after Brand Analytics will be enabled in CLOUD: https://github.com/airbytehq/airbyte/issues/32353 @@ -186,8 +192,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: BrandAnalyticsMarketBasketReports, BrandAnalyticsSearchTermsReports, BrandAnalyticsRepeatPurchaseReports, - BrandAnalyticsAlternatePurchaseReports, - BrandAnalyticsItemComparisonReports, SellerAnalyticsSalesAndTrafficReports, VendorSalesReports, VendorInventoryReports, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index ad555c619aff..9f84b550d8e7 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -109,7 +109,7 @@ "period_in_days": { "title": "Period In Days", "type": "integer", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", + "description": "For syncs spanning a large date range, this option is used to request data in a smaller fixed window to improve sync reliability. This time window can be configured granularly by day.", "default": 90, "minimum": 1, "order": 9 @@ -125,14 +125,13 @@ "required": ["stream_name", "options_list"], "properties": { "stream_name": { + "title": "Stream Name", "type": "string", "order": 0, "enum": [ "GET_AFN_INVENTORY_DATA", "GET_AFN_INVENTORY_DATA_BY_COUNTRY", "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", - "GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT", - "GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT", "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT", "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT", "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 0dead4f2c943..83cd4b4663f4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -2,14 +2,19 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import csv import gzip -import json as json_lib +import json +import logging +import os import time from abc import ABC, abstractmethod +from enum import Enum from io import StringIO from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union +import dateparser import pendulum import requests import xmltodict @@ -21,14 +26,17 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils.traced_exception import AirbyteTracedException -REPORTS_API_VERSION = "2021-06-30" # 2020-09-04 +REPORTS_API_VERSION = "2021-06-30" ORDERS_API_VERSION = "v0" VENDORS_API_VERSION = "v1" FINANCES_API_VERSION = "v0" +VENDOR_ORDERS_API_VERSION = "v1" DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" DATE_FORMAT = "%Y-%m-%d" +IS_TESTING = os.environ.get("DEPLOYMENT_MODE") == "testing" + class AmazonSPStream(HttpStream, ABC): data_field = "payload" @@ -61,6 +69,12 @@ def request_headers(self, *args, **kwargs) -> Mapping[str, Any]: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None + def retry_factor(self) -> float: + """ + Override for testing purposes + """ + return 0 if IS_TESTING else super().retry_factor + class IncrementalAmazonSPStream(AmazonSPStream, ABC): page_size = 100 @@ -96,14 +110,16 @@ def request_params( if next_page_token: return dict(next_page_token) - params = {self.replication_start_date_field: self._replication_start_date, self.page_size_field: self.page_size} + start_date = self._replication_start_date + params = {self.replication_start_date_field: start_date, self.page_size_field: self.page_size} - if self._replication_start_date and self.cursor_field: + if self.cursor_field: start_date = max(stream_state.get(self.cursor_field, self._replication_start_date), self._replication_start_date) - params.update({self.replication_start_date_field: start_date}) + start_date = min(start_date, pendulum.now("utc").to_date_string()) + params[self.replication_start_date_field] = start_date if self._replication_end_date: - params[self.replication_end_date_field] = self._replication_end_date + params[self.replication_end_date_field] = max(self._replication_end_date, start_date) return params @@ -114,29 +130,39 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return {self.next_page_token_field: next_page_token} def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargss + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, ) -> Iterable[Mapping]: """ - :return an iterable containing each record in the response + Return an iterable containing each record in the response """ yield from response.json().get(self.data_field, []) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. + Return the latest state by comparing the cursor value in the latest record with the stream's + most recent state object and returning an updated state object. """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} + latest_record_state = latest_record[self.cursor_field] + if stream_state := current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_record_state, stream_state)} + return {self.cursor_field: latest_record_state} + + +class ReportProcessingStatus(str, Enum): + CANCELLED = "CANCELLED" + DONE = "DONE" + FATAL = "FATAL" + IN_PROGRESS = "IN_PROGRESS" + IN_QUEUE = "IN_QUEUE" class ReportsAmazonSPStream(HttpStream, ABC): - max_wait_seconds = 3600 """ - API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reports_2020-09-04.md - API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/reports-api-model/reports_2020-09-04.json + API docs: https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference Report streams are intended to work as following: - create a new report; @@ -147,16 +173,19 @@ class ReportsAmazonSPStream(HttpStream, ABC): - yield the report document (if report processing status is `DONE`) """ - replication_start_date_limit_in_days = 90 + max_wait_seconds = 3600 + replication_start_date_limit_in_days = 365 primary_key = None path_prefix = f"reports/{REPORTS_API_VERSION}" sleep_seconds = 30 data_field = "payload" result_key = None - availability_sla_days = ( - 1 # see data availability sla at https://developer-docs.amazon.com/sp-api/docs/report-type-values#vendor-retail-analytics-reports - ) + + # see data availability sla at + # https://developer-docs.amazon.com/sp-api/docs/report-type-values#vendor-retail-analytics-reports + availability_sla_days = 1 + availability_strategy = None def __init__( self, @@ -174,7 +203,7 @@ def __init__( self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id - self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work as well + self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work self._report_options = report_options self._http_method = "GET" @@ -189,6 +218,15 @@ def http_method(self) -> str: def http_method(self, value: str): self._http_method = value + @property + def retry_factor(self) -> float: + """ + Set this 60.0 due to + https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference#post-reports2021-06-30reports + Override to 0 for integration testing purposes + """ + return 0 if IS_TESTING else 60.0 + @property def url_base(self) -> str: return self._url_base @@ -228,7 +266,7 @@ def _create_report( create_report_request = self._create_prepared_request( path=f"{self.path_prefix}/reports", headers=dict(request_headers, **self.authenticator.get_auth_header()), - data=json_lib.dumps(report_data), + data=json.dumps(report_data), ) report_response = self._send_request(create_report_request, {}) self.http_method = "GET" # rollback @@ -245,19 +283,33 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: return report_payload + def _retrieve_report_result(self, report_document_id: str) -> requests.Response: + request_headers = self.request_headers() + request = self._create_prepared_request( + path=self.path(document_id=report_document_id), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(), + ) + return self._send_request(request, {}) + @default_backoff_handler(factor=5, max_tries=5) def download_and_decompress_report_document(self, payload: dict) -> str: """ Unpacks a report document """ - report = requests.get(payload.get("url")) - report.raise_for_status() + + download_report_request = self._create_prepared_request(path=payload.get("url")) + report = self._send_request(download_report_request, {}) if "compressionAlgorithm" in payload: return gzip.decompress(report.content).decode("iso-8859-1") return report.content.decode("iso-8859-1") def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, ) -> Iterable[Mapping]: payload = response.json() @@ -275,14 +327,11 @@ def report_options(self) -> Optional[Mapping[str, Any]]: def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = max(pendulum.parse(self._replication_start_date), pendulum.now("utc").subtract(days=90)) - end_date = pendulum.now("utc") + now = pendulum.now("utc") + start_date = pendulum.parse(self._replication_start_date) + end_date = now if self._replication_end_date: - # if replication_start_date is older than 90 days(from current date), we are overriding the value above. - # when replication_end_date is present, we should use the user provided replication_start_date. - # user may provide a date range which is older than 90 days. end_date = min(end_date, pendulum.parse(self._replication_end_date)) - start_date = pendulum.parse(self._replication_start_date) if stream_state: state = stream_state.get(self.cursor_field) @@ -306,11 +355,10 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: """ Create and retrieve the report. - Decrypt and parse the report is its fully proceed, then yield the report document records. + Decrypt and parse the report if it's fully processed, then yield the report document records. """ report_payload = {} stream_slice = stream_slice or {} - is_processed = False start_time = pendulum.now("utc") seconds_waited = 0 try: @@ -318,62 +366,123 @@ def read_records( except DefaultBackoffException as e: logger.warning(f"The report for stream '{self.name}' was cancelled due to several failed retry attempts. {e}") return [] + except requests.exceptions.HTTPError as e: + if e.response.status_code == requests.codes.FORBIDDEN: + logger.warning( + f"The endpoint {e.response.url} returned {e.response.status_code}: {e.response.reason}. " + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate." + ) + return [] + raise e # create and retrieve the report - while not is_processed and seconds_waited < self.max_wait_seconds: + processed = False + while not processed and seconds_waited < self.max_wait_seconds: report_payload = self._retrieve_report(report_id=report_id) seconds_waited = (pendulum.now("utc") - start_time).seconds - is_processed = report_payload.get("processingStatus") not in ["IN_QUEUE", "IN_PROGRESS"] - time.sleep(self.sleep_seconds) + processed = report_payload.get("processingStatus") not in (ReportProcessingStatus.IN_QUEUE, ReportProcessingStatus.IN_PROGRESS) + if not processed: + time.sleep(self.sleep_seconds) - is_done = report_payload.get("processingStatus") == "DONE" - is_cancelled = report_payload.get("processingStatus") == "CANCELLED" - is_fatal = report_payload.get("processingStatus") == "FATAL" + processing_status = report_payload.get("processingStatus") report_end_date = pendulum.parse(report_payload.get("dataEndTime", stream_slice.get("dataEndTime"))) - if is_done: + if processing_status == ReportProcessingStatus.DONE: # retrieve and decrypt the report document document_id = report_payload["reportDocumentId"] - request_headers = self.request_headers() - request = self._create_prepared_request( - path=self.path(document_id=document_id), - headers=dict(request_headers, **self.authenticator.get_auth_header()), - params=self.request_params(), - ) - response = self._send_request(request, {}) + response = self._retrieve_report_result(document_id) + for record in self.parse_response(response, stream_state, stream_slice): if report_end_date: record["dataEndTime"] = report_end_date.strftime(DATE_FORMAT) yield record - elif is_fatal: - raise AirbyteTracedException(message=f"The report for stream '{self.name}' was not created - skip reading") - elif is_cancelled: - logger.warning(f"The report for stream '{self.name}' was cancelled or there is no data to return") + elif processing_status == ReportProcessingStatus.FATAL: + # retrieve and decrypt the report document + try: + document_id = report_payload["reportDocumentId"] + response = self._retrieve_report_result(document_id) + + document = self.download_and_decompress_report_document(response.json()) + error_response = json.loads(document) + except Exception as e: + logging.error(f"Failed to retrieve the report result document for stream '{self.name}'. Exception: {e}") + error_response = "Failed to retrieve the report result document." + + raise AirbyteTracedException( + internal_message=( + f"Failed to retrieve the report '{self.name}' for period " + f"{stream_slice['dataStartTime']}-{stream_slice['dataEndTime']}. " + f"This will be read during the next sync. Error: {error_response}" + ) + ) + elif processing_status == ReportProcessingStatus.CANCELLED: + logger.warning(f"The report for stream '{self.name}' was cancelled or there is no data to return.") else: - raise Exception(f"Unknown response for stream `{self.name}`. Response body {report_payload}") + raise Exception(f"Unknown response for stream '{self.name}'. Response body: {report_payload}.") -class MerchantListingsReports(ReportsAmazonSPStream): - name = "GET_MERCHANT_LISTINGS_ALL_DATA" +class IncrementalReportsAmazonSPStream(ReportsAmazonSPStream): + @property + def cursor_field(self) -> Union[str, List[str]]: + return "dataEndTime" + def _transform_report_record_cursor_value(self, date_string: str) -> str: + """ + Parse report date field based using transformer defined in the stream class + """ + return ( + self.transformer._custom_normalizer(date_string, self.get_json_schema()["properties"][self.cursor_field]) + if self.transformer._custom_normalizer + else date_string + ) -class NetPureProductMarginReport(ReportsAmazonSPStream): - name = "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT" + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's + most recent state object and returning an updated state object. + """ + latest_record_state = self._transform_report_record_cursor_value(latest_record[self.cursor_field]) + if stream_state := current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_record_state, stream_state)} + return {self.cursor_field: latest_record_state} -class RapidRetailAnalyticsInventoryReport(ReportsAmazonSPStream): - name = "GET_VENDOR_REAL_TIME_INVENTORY_REPORT" +class MerchantReports(IncrementalReportsAmazonSPStream, ABC): + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) + + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value and field_schema.get("format") == "date-time": + # open-date field is returned in format "2022-07-11 01:34:18 PDT" + transformed_value = dateparser.parse(original_value).isoformat() + return transformed_value + return original_value + + return transform_function + +class MerchantListingsReports(MerchantReports): + name = "GET_MERCHANT_LISTINGS_ALL_DATA" + primary_key = "listing-id" -class FlatFileOrdersReports(ReportsAmazonSPStream): + +class FlatFileOrdersReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780 """ name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + primary_key = "amazon-order-id" + cursor_field = "last-updated-date" -class FbaStorageFeesReports(ReportsAmazonSPStream): +class FbaStorageFeesReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/G202086720 """ @@ -381,39 +490,44 @@ class FbaStorageFeesReports(ReportsAmazonSPStream): name = "GET_FBA_STORAGE_FEE_CHARGES_DATA" -class FulfilledShipmentsReports(ReportsAmazonSPStream): +class FulfilledShipmentsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200453120 """ name = "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL" + # You can request up to one month of data in a single report + # https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports replication_start_date_limit_in_days = 30 -class FlatFileOpenListingsReports(ReportsAmazonSPStream): +class FlatFileOpenListingsReports(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_OPEN_LISTINGS_DATA" -class FbaOrdersReports(ReportsAmazonSPStream): +class FbaOrdersReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110 """ name = "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" + cursor_field = "last-updated-date" -class FlatFileActionableOrderDataShipping(ReportsAmazonSPStream): +class FlatFileActionableOrderDataShipping(IncrementalReportsAmazonSPStream): """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping + Field definitions: + https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping """ name = "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING" -class OrderReportDataShipping(ReportsAmazonSPStream): +class OrderReportDataShipping(IncrementalReportsAmazonSPStream): """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping + Field definitions: + https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping """ name = "GET_ORDER_REPORT_DATA_SHIPPING" @@ -433,7 +547,7 @@ def parse_document(self, document): return result -class FbaShipmentsReports(ReportsAmazonSPStream): +class FbaShipmentsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100 """ @@ -441,7 +555,7 @@ class FbaShipmentsReports(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA" -class FbaReplacementsReports(ReportsAmazonSPStream): +class FbaReplacementsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/200453300 """ @@ -449,7 +563,7 @@ class FbaReplacementsReports(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA" -class RestockInventoryReports(ReportsAmazonSPStream): +class RestockInventoryReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/202105670 """ @@ -457,11 +571,15 @@ class RestockInventoryReports(ReportsAmazonSPStream): name = "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT" -class GetXmlBrowseTreeData(ReportsAmazonSPStream): +class GetXmlBrowseTreeData(IncrementalReportsAmazonSPStream): def parse_document(self, document): try: parsed = xmltodict.parse( - document, dict_constructor=dict, attr_prefix="", cdata_key="text", force_list={"attribute", "id", "refinementField"} + document, + dict_constructor=dict, + attr_prefix="", + cdata_key="text", + force_list={"attribute", "id", "refinementField"}, ) except Exception as e: self.logger.warning(f"Unable to parse the report for the stream {self.name}, error: {str(e)}") @@ -470,33 +588,40 @@ def parse_document(self, document): return parsed.get("Result", {}).get("Node", []) name = "GET_XML_BROWSE_TREE_DATA" + primary_key = "browseNodeId" -class FbaEstimatedFbaFeesTxtReport(ReportsAmazonSPStream): +class FbaEstimatedFbaFeesTxtReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA" -class FbaFulfillmentCustomerShipmentPromotionReport(ReportsAmazonSPStream): +class FbaFulfillmentCustomerShipmentPromotionReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA" -class FbaMyiUnsuppressedInventoryReport(ReportsAmazonSPStream): +class FbaMyiUnsuppressedInventoryReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA" -class MerchantListingsReport(ReportsAmazonSPStream): +class MerchantListingsReport(MerchantReports): name = "GET_MERCHANT_LISTINGS_DATA" + primary_key = "listing-id" -class MerchantListingsInactiveData(ReportsAmazonSPStream): +class MerchantListingsInactiveData(MerchantReports): name = "GET_MERCHANT_LISTINGS_INACTIVE_DATA" + primary_key = "listing-id" -class StrandedInventoryUiReport(ReportsAmazonSPStream): +class StrandedInventoryUiReport(IncrementalReportsAmazonSPStream): name = "GET_STRANDED_INVENTORY_UI_DATA" -class XmlAllOrdersDataByOrderDataGeneral(ReportsAmazonSPStream): +class XmlAllOrdersDataByOrderDataGeneral(IncrementalReportsAmazonSPStream): + name = "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + primary_key = "AmazonOrderID" + cursor_field = "LastUpdatedDate" + def parse_document(self, document): try: parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message", "OrderItem"}) @@ -512,50 +637,66 @@ def parse_document(self, document): return result - name = "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" - -class MerchantListingsReportBackCompat(ReportsAmazonSPStream): +class MerchantListingsReportBackCompat(MerchantReports): name = "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT" + primary_key = "listing-id" -class MerchantCancelledListingsReport(ReportsAmazonSPStream): +class MerchantCancelledListingsReport(IncrementalReportsAmazonSPStream): name = "GET_MERCHANT_CANCELLED_LISTINGS_DATA" -class MerchantListingsFypReport(ReportsAmazonSPStream): +class MerchantListingsFypReport(IncrementalReportsAmazonSPStream): name = "GET_MERCHANTS_LISTINGS_FYP_REPORT" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) + + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value and field_schema.get("format") == "date": + try: + transformed_value = pendulum.from_format(original_value, "MMM D[,] YYYY").to_date_string() + return transformed_value + except ValueError: + pass + return original_value -class FbaSnsForecastReport(ReportsAmazonSPStream): + return transform_function + + +class FbaSnsForecastReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_SNS_FORECAST_DATA" -class FbaSnsPerformanceReport(ReportsAmazonSPStream): +class FbaSnsPerformanceReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_SNS_PERFORMANCE_DATA" -class FlatFileArchivedOrdersDataByOrderDate(ReportsAmazonSPStream): +class FlatFileArchivedOrdersDataByOrderDate(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE" + cursor_field = "last-updated-date" -class FlatFileReturnsDataByReturnDate(ReportsAmazonSPStream): +class FlatFileReturnsDataByReturnDate(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE" + # You can request up to 60 days of data in a single report + # https://developer-docs.amazon.com/sp-api/docs/report-type-values-returns replication_start_date_limit_in_days = 60 -class FbaInventoryPlaningReport(ReportsAmazonSPStream): +class FbaInventoryPlaningReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_INVENTORY_PLANNING_DATA" -class LedgerSummaryViewReport(ReportsAmazonSPStream): - name = "GET_LEDGER_SUMMARY_VIEW_DATA" - - class AnalyticsStream(ReportsAmazonSPStream): def parse_document(self, document): - parsed = json_lib.loads(document) + parsed = json.loads(document) return parsed.get(self.result_key, []) def _report_data( @@ -604,36 +745,121 @@ def _augmented_data(self, report_options) -> Mapping[str, Any]: } -class BrandAnalyticsMarketBasketReports(AnalyticsStream): +class IncrementalAnalyticsStream(AnalyticsStream): + fixed_period_in_days = 0 + + @property + def cursor_field(self) -> Union[str, List[str]]: + return "endDate" + + def _report_data( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Mapping[str, Any]: + data = super()._report_data(sync_mode, cursor_field, stream_slice, stream_state) + if stream_slice: + data_times = {} + if stream_slice.get("dataStartTime"): + data_times["dataStartTime"] = stream_slice["dataStartTime"] + if stream_slice.get("dataEndTime"): + data_times["dataEndTime"] = stream_slice["dataEndTime"] + data.update(data_times) + + return data + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, + ) -> Iterable[Mapping]: + payload = response.json() + + document = self.download_and_decompress_report_document(payload) + document_records = self.parse_document(document) + + # Not all (partial) responses include the request date, so adding it manually here + for record in document_records: + if stream_slice.get("dataEndTime"): + record["queryEndDate"] = pendulum.parse(stream_slice["dataEndTime"]).strftime("%Y-%m-%d") + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's + most recent state object and returning an updated state object. + """ + latest_record_state = latest_record[self.cursor_field] + if stream_state := current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_record_state, stream_state)} + return {self.cursor_field: latest_record_state} + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + start_date = pendulum.parse(self._replication_start_date) + end_date = pendulum.now("utc").subtract(days=self.availability_sla_days) + + if self._replication_end_date: + end_date = pendulum.parse(self._replication_end_date) + + if stream_state: + state = stream_state.get(self.cursor_field) + start_date = pendulum.parse(state) + + start_date = min(start_date, end_date) + + while start_date < end_date: + # If request only returns data on day level + if self.fixed_period_in_days != 0: + slice_range = self.fixed_period_in_days + else: + slice_range = self.period_in_days + + end_date_slice = start_date.add(days=slice_range) + yield { + "dataStartTime": start_date.strftime(DATE_TIME_FORMAT), + "dataEndTime": min(end_date_slice.subtract(seconds=1), end_date).strftime(DATE_TIME_FORMAT), + } + start_date = end_date_slice + + +class NetPureProductMarginReport(IncrementalAnalyticsStream): + name = "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT" + result_key = "netPureProductMarginByAsin" + + +class RapidRetailAnalyticsInventoryReport(IncrementalAnalyticsStream): + name = "GET_VENDOR_REAL_TIME_INVENTORY_REPORT" + result_key = "reportData" + cursor_field = "endTime" + + +class BrandAnalyticsMarketBasketReports(IncrementalAnalyticsStream): name = "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT" result_key = "dataByAsin" -class BrandAnalyticsSearchTermsReports(AnalyticsStream): +class BrandAnalyticsSearchTermsReports(IncrementalAnalyticsStream): """ Field definitions: https://sellercentral.amazon.co.uk/help/hub/reference/G5NXWNY8HUD3VDCW """ name = "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT" result_key = "dataByDepartmentAndSearchTerm" + cursor_field = "queryEndDate" -class BrandAnalyticsRepeatPurchaseReports(AnalyticsStream): +class BrandAnalyticsRepeatPurchaseReports(IncrementalAnalyticsStream): name = "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT" result_key = "dataByAsin" -class BrandAnalyticsAlternatePurchaseReports(AnalyticsStream): - name = "GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT" - result_key = "dataByAsin" - - -class BrandAnalyticsItemComparisonReports(AnalyticsStream): - name = "GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT" - result_key = "dataByAsin" - - -class VendorInventoryReports(AnalyticsStream): +class VendorInventoryReports(IncrementalAnalyticsStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#vendor-retail-analytics-reports """ @@ -643,26 +869,26 @@ class VendorInventoryReports(AnalyticsStream): availability_sla_days = 3 -class VendorTrafficReport(AnalyticsStream): +class VendorTrafficReport(IncrementalAnalyticsStream): name = "GET_VENDOR_TRAFFIC_REPORT" result_key = "trafficByAsin" -class IncrementalReportsAmazonSPStream(ReportsAmazonSPStream): - @property - @abstractmethod - def cursor_field(self) -> Union[str, List[str]]: - pass +class SellerAnalyticsSalesAndTrafficReports(IncrementalAnalyticsStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#seller-retail-analytics-reports + """ - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} + name = "GET_SALES_AND_TRAFFIC_REPORT" + result_key = "salesAndTrafficByAsin" + cursor_field = "queryEndDate" + fixed_period_in_days = 1 + + +class VendorSalesReports(IncrementalAnalyticsStream): + name = "GET_VENDOR_SALES_REPORT" + result_key = "salesByAsin" + availability_sla_days = 4 # Data is only available after 4 days class SellerFeedbackReports(IncrementalReportsAmazonSPStream): @@ -670,7 +896,8 @@ class SellerFeedbackReports(IncrementalReportsAmazonSPStream): Field definitions: https://sellercentral.amazon.com/help/hub/reference/G202125660 """ - # The list of MarketplaceIds can be found here https://docs.developer.amazonservices.com/en_UK/dev_guide/DG_Endpoints.html + # The list of MarketplaceIds can be found here: + # https://docs.developer.amazonservices.com/en_UK/dev_guide/DG_Endpoints.html MARKETPLACE_DATE_FORMAT_MAP = dict( # eu A2VIGQ35RCS4UG="D/M/YY", # AE @@ -710,20 +937,22 @@ def __init__(self, *args, **kwargs): def get_transform_function(self): def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: - if original_value and "format" in field_schema and field_schema["format"] == "date": + if original_value and field_schema.get("format") == "date": date_format = self.MARKETPLACE_DATE_FORMAT_MAP.get(self.marketplace_id) if not date_format: - raise KeyError(f"Date format not found for Markeplace ID: {self.marketplace_id}") - transformed_value = pendulum.from_format(original_value, date_format).to_date_string() - return transformed_value + raise KeyError(f"Date format not found for Marketplace ID: {self.marketplace_id}") + try: + transformed_value = pendulum.from_format(original_value, date_format).to_date_string() + return transformed_value + except ValueError: + pass return original_value return transform_function # csv header field names for this report differ per marketplace (are localized to marketplace language) - # but columns come in the same order - # so we set fieldnames to our custom ones + # but columns come in the same order, so we set fieldnames to our custom ones # and raise error if original and custom header field count does not match @staticmethod def parse_document(document): @@ -738,21 +967,21 @@ def parse_document(document): class FbaAfnInventoryReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports - Report has a long-running issue (fails when requested frequently): https://github.com/amzn/selling-partner-api-docs/issues/2231 + Report has a long-running issue (fails when requested frequently): + https://github.com/amzn/selling-partner-api-docs/issues/2231 """ name = "GET_AFN_INVENTORY_DATA" - cursor_field = "dataEndTime" class FbaAfnInventoryByCountryReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports - Report has a long-running issue (fails when requested frequently): https://github.com/amzn/selling-partner-api-docs/issues/2231 + Report has a long-running issue (fails when requested frequently): + https://github.com/amzn/selling-partner-api-docs/issues/2231 """ name = "GET_AFN_INVENTORY_DATA_BY_COUNTRY" - cursor_field = "dataEndTime" class FlatFileOrdersReportsByLastUpdate(IncrementalReportsAmazonSPStream): @@ -761,12 +990,14 @@ class FlatFileOrdersReportsByLastUpdate(IncrementalReportsAmazonSPStream): """ name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL" + primary_key = "amazon-order-id" cursor_field = "last-updated-date" + replication_start_date_limit_in_days = 30 class Orders(IncrementalAmazonSPStream): """ - API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/orders-api/ordersV0.md + API docs: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/orders-api-model/ordersV0.json """ @@ -787,11 +1018,15 @@ def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - params.update({"MarketplaceIds": self.marketplace_id}) + params["MarketplaceIds"] = self.marketplace_id return params def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, ) -> Iterable[Mapping]: yield from response.json().get(self.data_field, {}).get(self.name, []) @@ -803,7 +1038,7 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return self.default_backoff_time -class OrderItems(AmazonSPStream, ABC): +class OrderItems(IncrementalAmazonSPStream): """ API docs: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems API model: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#orderitemslist @@ -815,6 +1050,8 @@ class OrderItems(AmazonSPStream, ABC): parent_cursor_field = "LastUpdateDate" next_page_token_field = "NextToken" stream_slice_cursor_field = "AmazonOrderId" + replication_start_date_field = "LastUpdatedAfter" + replication_end_date_field = "LastUpdatedBefore" page_size_field = None default_backoff_time = 10 default_stream_slice_delay_time = 1 @@ -838,19 +1075,12 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite orders = Orders(**self.stream_kwargs) for order_record in orders.read_records(sync_mode=SyncMode.incremental, stream_state=stream_state): self.cached_state[self.parent_cursor_field] = order_record[self.parent_cursor_field] - self.logger.info(f"OrderItems stream slice for order {order_record[self.stream_slice_cursor_field]}") time.sleep(self.default_stream_slice_delay_time) yield { self.stream_slice_cursor_field: order_record[self.stream_slice_cursor_field], self.parent_cursor_field: order_record[self.parent_cursor_field], } - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = self.cached_state[self.parent_cursor_field] - if current_stream_state.get(self.parent_cursor_field): - return {self.parent_cursor_field: max(latest_benchmark, current_stream_state[self.parent_cursor_field])} - return {self.parent_cursor_field: latest_benchmark} - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: stream_data = response.json() next_page_token = stream_data.get("payload").get(self.next_page_token_field) @@ -865,10 +1095,13 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return self.default_backoff_time def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, ) -> Iterable[Mapping]: order_items_list = response.json().get(self.data_field, {}) - self.logger.info(f"order_items_list efim {order_items_list}") if order_items_list.get(self.next_page_token_field) is None: self.cached_state[self.parent_cursor_field] = stream_slice[self.parent_cursor_field] for order_item in order_items_list.get(self.name, []): @@ -890,125 +1123,91 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.transformer.registerCustomTransform(self.get_transform_function()) - def get_transform_function(self): - def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + @staticmethod + def get_transform_function(): + def transform_function(original_value: str, field_schema: Dict[str, Any]) -> str: if original_value and field_schema.get("format") == "date": - transformed_value = pendulum.from_format(original_value, "MM/DD/YYYY").to_date_string() - return transformed_value + date_format = "MM/YYYY" if len(original_value) <= 7 else "MM/DD/YYYY" + try: + transformed_value = pendulum.from_format(original_value, date_format).to_date_string() + return transformed_value + except ValueError: + pass return original_value return transform_function -class IncrementalAnalyticsStream(AnalyticsStream): +class LedgerSummaryViewReport(LedgerDetailedViewReports): + name = "GET_LEDGER_SUMMARY_VIEW_DATA" - fixed_period_in_days = 0 + +class VendorFulfillment(IncrementalAmazonSPStream, ABC): + primary_key = "purchaseOrderNumber" + next_page_token_field = "nextToken" + page_size_field = "limit" @property @abstractmethod - def cursor_field(self) -> Union[str, List[str]]: + def records_path(self) -> str: pass - def _report_data( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Mapping[str, Any]: - data = super()._report_data(sync_mode, cursor_field, stream_slice, stream_state) - if stream_slice: - data_times = {} - if stream_slice.get("dataStartTime"): - data_times["dataStartTime"] = stream_slice["dataStartTime"] - if stream_slice.get("dataEndTime"): - data_times["dataEndTime"] = stream_slice["dataEndTime"] - data.update(data_times) - - return data - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping]: - - payload = response.json() - - document = self.download_and_decompress_report_document(payload) - document_records = self.parse_document(document) - - # Not all (partial) responses include the request date, so adding it manually here - for record in document_records: - if stream_slice.get("dataEndTime"): - record["queryEndDate"] = pendulum.parse(stream_slice["dataEndTime"]).strftime("%Y-%m-%d") - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + stream_data = response.json() + next_page_token = stream_data.get(self.data_field, {}).get("pagination", {}).get(self.next_page_token_field) + if next_page_token: + return {self.next_page_token_field: next_page_token} def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = pendulum.parse(self._replication_start_date) - end_date = pendulum.now("utc").subtract(days=self.availability_sla_days) - - if self._replication_end_date: - end_date = pendulum.parse(self._replication_end_date) + end_date = pendulum.parse(self._replication_end_date) if self._replication_end_date else pendulum.now("utc") - if stream_state: - state = stream_state.get(self.cursor_field) - start_date = pendulum.parse(state) + stream_state = stream_state or {} + if state_value := stream_state.get(self.cursor_field): + start_date = max(start_date, pendulum.parse(state_value)) start_date = min(start_date, end_date) - slices = [] - while start_date < end_date: - # If request only returns data on day level - if self.fixed_period_in_days != 0: - slice_range = self.fixed_period_in_days - else: - slice_range = self.period_in_days - - end_date_slice = start_date.add(days=slice_range) - slices.append( - { - "dataStartTime": start_date.strftime(DATE_TIME_FORMAT), - "dataEndTime": min(end_date_slice.subtract(seconds=1), end_date).strftime(DATE_TIME_FORMAT), - } - ) + end_date_slice = start_date.add(days=7) + yield { + self.replication_start_date_field: start_date.strftime(DATE_TIME_FORMAT), + self.replication_end_date_field: min(end_date_slice, end_date).strftime(DATE_TIME_FORMAT), + } start_date = end_date_slice - return slices - - -class SellerAnalyticsSalesAndTrafficReports(IncrementalAnalyticsStream): - """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#seller-retail-analytics-reports - """ - - name = "GET_SALES_AND_TRAFFIC_REPORT" - result_key = "salesAndTrafficByAsin" - cursor_field = "queryEndDate" - fixed_period_in_days = 1 + def request_params( + self, + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + stream_slice = stream_slice or {} + if next_page_token: + stream_slice.update(next_page_token) + return stream_slice -class VendorSalesReports(IncrementalAnalyticsStream): - name = "GET_VENDOR_SALES_REPORT" - result_key = "salesByAsin" - cursor_field = "endDate" - availability_sla_days = 4 # Data is only available after 4 days + def parse_response( + self, + response: requests.Response, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping]: + params = self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + for record in response.json().get(self.data_field, {}).get(self.records_path, []): + record[self.replication_end_date_field] = params.get(self.replication_end_date_field) + yield record -class VendorDirectFulfillmentShipping(AmazonSPStream): +class VendorDirectFulfillmentShipping(VendorFulfillment): """ - API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/vendor-direct-fulfillment-shipping-api/vendorDirectFulfillmentShippingV1.md + API docs: https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/vendor-direct-fulfillment-shipping-api-model/vendorDirectFulfillmentShippingV1.json Returns a list of shipping labels created during the time frame that you specify. @@ -1017,37 +1216,35 @@ class VendorDirectFulfillmentShipping(AmazonSPStream): """ name = "VendorDirectFulfillmentShipping" - primary_key = None + records_path = "shippingLabels" replication_start_date_field = "createdAfter" replication_end_date_field = "createdBefore" - next_page_token_field = "nextToken" - page_size_field = "limit" - time_format = "%Y-%m-%dT%H:%M:%SZ" + cursor_field = "createdBefore" - def path(self, **kwargs) -> str: + def path(self, **kwargs: Any) -> str: return f"vendor/directFulfillment/shipping/{VENDORS_API_VERSION}/shippingLabels" - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - if not next_page_token: - end_date = pendulum.now("utc").strftime(self.time_format) - if self._replication_end_date: - end_date = self._replication_end_date - start_date = max(pendulum.parse(self._replication_start_date), pendulum.parse(end_date).subtract(days=7, hours=1)).strftime( - self.time_format - ) +class VendorOrders(VendorFulfillment): + """ + API docs: + https://developer-docs.amazon.com/sp-api/docs/vendor-orders-api-v1-reference#get-vendorordersv1purchaseorders - params.update({self.replication_start_date_field: start_date, self.replication_end_date_field: end_date}) - return params + API model: + https://github.com/amzn/selling-partner-api-models/blob/main/models/vendor-orders-api-model/vendorOrders.json + """ - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - yield from response.json().get(self.data_field, {}).get("shippingLabels", []) + name = "VendorOrders" + records_path = "orders" + replication_start_date_field = "changedAfter" + replication_end_date_field = "changedBefore" + cursor_field = "changedBefore" + def path(self, **kwargs: Any) -> str: + return f"vendor/orders/{VENDOR_ORDERS_API_VERSION}/purchaseOrders" -class FinanceStream(AmazonSPStream, ABC): + +class FinanceStream(IncrementalAmazonSPStream, ABC): next_page_token_field = "NextToken" page_size_field = "MaxResultsPerPage" page_size = 100 @@ -1080,6 +1277,10 @@ def request_params( DATE_TIME_FORMAT ) + stream_state = stream_state or {} + if stream_state_value := stream_state.get(self.cursor_field): + start_date = max(stream_state_value, start_date) + # logging to make sure user knows taken start date logger.info("start date used: %s", start_date) @@ -1106,47 +1307,76 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: class ListFinancialEventGroups(FinanceStream): """ - API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/finances-api/financesV0.md#listfinancialeventgroups + API docs: https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/finances-api-model/financesV0.json """ name = "ListFinancialEventGroups" + primary_key = "FinancialEventGroupId" replication_start_date_field = "FinancialEventGroupStartedAfter" replication_end_date_field = "FinancialEventGroupStartedBefore" + cursor_field = "FinancialEventGroupStart" def path(self, **kwargs) -> str: return f"finances/{FINANCES_API_VERSION}/financialEventGroups" - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, + ) -> Iterable[Mapping]: yield from response.json().get(self.data_field, {}).get("FinancialEventGroupList", []) class ListFinancialEvents(FinanceStream): """ - API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/finances-api/financesV0.md#listfinancialevents + API docs: https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/finances-api-model/financesV0.json """ name = "ListFinancialEvents" replication_start_date_field = "PostedAfter" replication_end_date_field = "PostedBefore" + cursor_field = "PostedBefore" def path(self, **kwargs) -> str: return f"finances/{FINANCES_API_VERSION}/financialEvents" - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - yield from [response.json().get(self.data_field, {}).get("FinancialEvents", {})] - + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Any, + ) -> Iterable[Mapping]: + params = self.request_params(stream_state) + events = response.json().get(self.data_field, {}).get("FinancialEvents", {}) + events[self.replication_end_date_field] = params.get(self.replication_end_date_field) + yield from [events] -class FbaCustomerReturnsReports(ReportsAmazonSPStream): +class FbaCustomerReturnsReports(IncrementalReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA" class FlatFileSettlementV2Reports(IncrementalReportsAmazonSPStream): - name = "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE" - cursor_field = "dataEndTime" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) + + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value == "" and field_schema.get("format") == "date-time": + return None + return original_value + + return transform_function def _create_report( self, @@ -1155,7 +1385,6 @@ def _create_report( stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, ) -> Mapping[str, Any]: - # For backwards return {"reportId": stream_slice.get("report_id")} @@ -1170,7 +1399,6 @@ def stream_slices( You can search for these reports using the getReports operation. ``` """ - strict_start_date = pendulum.now("utc").subtract(days=90) utc_now = pendulum.now("utc").date().to_date_string() @@ -1215,7 +1443,7 @@ def stream_slices( complete = True -class FbaReimbursementsReports(ReportsAmazonSPStream): +class FbaReimbursementsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/G200732720 """ diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py index fda5a0442cfd..fbc299d456e2 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py index e51ccf7e2aa0..88ae651494b9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py @@ -2,10 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import os from typing import Any, Dict import pytest +os.environ["DEPLOYMENT_MODE"] = "testing" + @pytest.fixture def report_init_kwargs() -> Dict[str, Any]: @@ -17,3 +21,8 @@ def report_init_kwargs() -> Dict[str, Any]: "report_options": None, "replication_end_date": None, } + + +@pytest.fixture +def http_mocker() -> None: + """This fixture is needed to pass http_mocker parameter from the @HttpMocker decorator to a test""" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py new file mode 100644 index 000000000000..d1decff994b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +from datetime import datetime +from typing import Dict + +import pendulum + +ACCESS_TOKEN = "test_access_token" +LWA_APP_ID = "amazon_app_id" +LWA_CLIENT_SECRET = "amazon_client_secret" +MARKETPLACE_ID = "ATVPDKIKX0DER" +REFRESH_TOKEN = "amazon_refresh_token" + +CONFIG_START_DATE = "2023-01-01T00:00:00Z" +CONFIG_END_DATE = "2023-01-30T00:00:00Z" +NOW = pendulum.now(tz="utc") +TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, str] = { + "refresh_token": REFRESH_TOKEN, + "lwa_app_id": LWA_APP_ID, + "lwa_client_secret": LWA_CLIENT_SECRET, + "replication_start_date": CONFIG_START_DATE, + "replication_end_date": CONFIG_END_DATE, + "aws_environment": "PRODUCTION", + "region": "US", + "account_type": "Seller", + } + + def with_start_date(self, start_date: datetime) -> ConfigBuilder: + self._config["replication_start_date"] = start_date.strftime(TIME_FORMAT) + return self + + def with_end_date(self, end_date: datetime) -> ConfigBuilder: + self._config["replication_end_date"] = end_date.strftime(TIME_FORMAT) + return self + + def build(self) -> Dict[str, str]: + return self._config diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py new file mode 100644 index 000000000000..79b1528ed038 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + +NEXT_TOKEN_STRING = "MDAwMDAwMDAwMQ==" + + +class VendorFulfillmentPaginationStrategy(PaginationStrategy): + def update(self, response: Dict[str, Any]) -> None: + response["payload"]["pagination"] = {} + response["payload"]["pagination"]["nextToken"] = NEXT_TOKEN_STRING diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py new file mode 100644 index 000000000000..d14097dc3cd1 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/request_builder.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +import json +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS, HttpRequest + +from .config import ACCESS_TOKEN, LWA_APP_ID, LWA_CLIENT_SECRET, MARKETPLACE_ID, NOW, REFRESH_TOKEN + + +class RequestBuilder: + @classmethod + def auth_endpoint(cls) -> RequestBuilder: + request_headers = {"Content-Type": "application/x-www-form-urlencoded"} + request_body = ( + f"grant_type=refresh_token&client_id={LWA_APP_ID}&" f"client_secret={LWA_CLIENT_SECRET}&refresh_token={REFRESH_TOKEN}" + ) + return cls("auth/o2/token").with_base_url("https://api.amazon.com").with_headers(request_headers).with_body(request_body) + + @classmethod + def create_report_endpoint(cls, report_name: str) -> RequestBuilder: + request_body = { + "reportType": report_name, + "marketplaceIds": [MARKETPLACE_ID], + "dataStartTime": "2023-01-01T00:00:00Z", + "dataEndTime": "2023-01-30T00:00:00Z", + } + return cls("reports/2021-06-30/reports").with_body(json.dumps(request_body)) + + @classmethod + def check_report_status_endpoint(cls, report_id: str) -> RequestBuilder: + return cls(f"reports/2021-06-30/reports/{report_id}") + + @classmethod + def get_document_download_url_endpoint(cls, document_id: str) -> RequestBuilder: + return cls(f"reports/2021-06-30/documents/{document_id}") + + @classmethod + def download_document_endpoint(cls, url: str) -> RequestBuilder: + return cls("").with_base_url(url).with_headers(None) + + @classmethod + def vendor_direct_fulfillment_shipping_endpoint(cls) -> RequestBuilder: + return cls("vendor/directFulfillment/shipping/v1/shippingLabels") + + @classmethod + def vendor_orders_endpoint(cls) -> RequestBuilder: + return cls("vendor/orders/v1/purchaseOrders") + + def __init__(self, resource: str) -> None: + self._resource = resource + self._base_url = "https://sellingpartnerapi-na.amazon.com" + self._headers = { + "content-type": "application/json", + "host": self._base_url.replace("https://", ""), + "user-agent": "python-requests", + "x-amz-access-token": ACCESS_TOKEN, + "x-amz-date": NOW.strftime("%Y%m%dT%H%M%SZ"), + } + self._query_params = ANY_QUERY_PARAMS + self._body = None + + def with_base_url(self, base_url: str) -> RequestBuilder: + self._base_url = base_url + return self + + def with_headers(self, headers: Optional[Union[str, Mapping[str, str]]]) -> RequestBuilder: + self._headers = headers + return self + + def with_query_params(self, query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> RequestBuilder: + self._query_params = query_params + return self + + def with_body(self, body: Union[str, bytes, Mapping[str, Any]]) -> RequestBuilder: + self._body = body + return self + + def _url(self) -> str: + return f"{self._base_url}/{self._resource}" if self._resource else self._base_url + + def build(self) -> HttpRequest: + return HttpRequest(url=self._url(), query_params=self._query_params, headers=self._headers, body=self._body) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/response_builder.py new file mode 100644 index 000000000000..c557e6bab9b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/response_builder.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus +from typing import Any, Mapping, Optional + +from airbyte_cdk.test.mock_http import HttpResponse + + +def response_with_status(status_code: HTTPStatus, body: Optional[Mapping[str, Any]] = None) -> HttpResponse: + body = body or {} + return HttpResponse(body=json.dumps(body), status_code=status_code) + + +def build_response(body: Mapping[str, Any], status_code: HTTPStatus) -> HttpResponse: + return HttpResponse(body=json.dumps(body), status_code=status_code) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py new file mode 100644 index 000000000000..bfdbfa241e73 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -0,0 +1,515 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import gzip +from http import HTTPStatus +from typing import List, Optional + +import freezegun +import pytest +import requests_mock +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher +from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode +from source_amazon_seller_partner.streams import ReportProcessingStatus + +from .config import CONFIG_END_DATE, CONFIG_START_DATE, MARKETPLACE_ID, NOW, ConfigBuilder +from .request_builder import RequestBuilder +from .response_builder import build_response, response_with_status +from .utils import assert_message_in_log_output, config, find_template, get_stream_by_name, mock_auth, read_output + +_DOCUMENT_DOWNLOAD_URL = "https://test.com/download" +_REPORT_ID = "6789087632" +_REPORT_DOCUMENT_ID = "report_document_id" + +DEFAULT_EXPECTED_NUMBER_OF_RECORDS = 2 # every test file in resource/http/response contains 2 records +STREAMS = ( + ("GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING", "csv"), + ("GET_ORDER_REPORT_DATA_SHIPPING", "xml"), + ("GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", "csv"), + ("GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", "csv"), + ("GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", "csv"), + ("GET_SELLER_FEEDBACK_DATA", "csv"), + ("GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA", "csv"), + ("GET_LEDGER_DETAIL_VIEW_DATA", "csv"), + ("GET_AFN_INVENTORY_DATA_BY_COUNTRY", "csv"), + ("GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE", "csv"), + ("GET_VENDOR_SALES_REPORT", "json"), + ("GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT", "json"), + ("GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", "csv"), + ("GET_FBA_SNS_FORECAST_DATA", "csv"), + ("GET_AFN_INVENTORY_DATA", "csv"), + ("GET_MERCHANT_CANCELLED_LISTINGS_DATA", "csv"), + ("GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA", "csv"), + ("GET_LEDGER_SUMMARY_VIEW_DATA", "csv"), + ("GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", "json"), + ("GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT", "json"), + ("GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE", "csv"), + ("GET_VENDOR_INVENTORY_REPORT", "json"), + ("GET_FBA_SNS_PERFORMANCE_DATA", "csv"), + ("GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", "csv"), + ("GET_FBA_INVENTORY_PLANNING_DATA", "csv"), + ("GET_FBA_STORAGE_FEE_CHARGES_DATA", "csv"), + ("GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA", "csv"), + ("GET_STRANDED_INVENTORY_UI_DATA", "csv"), + ("GET_FBA_REIMBURSEMENTS_DATA", "csv"), + ("GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", "json"), + ("GET_VENDOR_REAL_TIME_INVENTORY_REPORT", "json"), + ("GET_VENDOR_TRAFFIC_REPORT", "json"), +) + + +def _create_report_request(report_name: str) -> RequestBuilder: + """ + A POST request needed to start generating a report on Amazon SP platform. + Performed in ReportsAmazonSPStream._create_report method. + """ + + return RequestBuilder.create_report_endpoint(report_name) + + +def _check_report_status_request(report_id: str) -> RequestBuilder: + """ + A GET request needed to check the report generating status. + Performed in ReportsAmazonSPStream._retrieve_report method. + """ + + return RequestBuilder.check_report_status_endpoint(report_id) + + +def _get_document_download_url_request(document_id: str) -> RequestBuilder: + """ + A GET request which returns a URL for the report download. + """ + + return RequestBuilder.get_document_download_url_endpoint(document_id) + + +def _download_document_request(url: str) -> RequestBuilder: + """ + A GET request which actually downloads the report. + Performed in ReportsAmazonSPStream.download_and_decompress_report_document method. + """ + + return RequestBuilder.download_document_endpoint(url) + + +def _create_report_response(report_id: str, status_code: Optional[HTTPStatus] = HTTPStatus.ACCEPTED) -> HttpResponse: + response_body = {"reportId": report_id} + return build_response(response_body, status_code=status_code) + + +def _check_report_status_response( + report_name: str, + processing_status: Optional[ReportProcessingStatus] = ReportProcessingStatus.DONE, + report_document_id: Optional[str] = None, +) -> HttpResponse: + if processing_status == ReportProcessingStatus.DONE and not report_document_id: + raise ValueError("report_document_id value should be passed when processing_status is 'DONE'.") + + response_body = { + "reportType": report_name, + "processingStatus": processing_status, + "marketplaceIds": [MARKETPLACE_ID], + "reportId": _REPORT_ID, + "dataEndTime": CONFIG_END_DATE, + "createdTime": CONFIG_START_DATE, + "dataStartTime": CONFIG_START_DATE, + "reportDocumentId": report_document_id, + } + if processing_status == ReportProcessingStatus.DONE: + response_body.update( + { + "processingEndTime": CONFIG_START_DATE, + "processingStartTime": CONFIG_START_DATE, + } + ) + + return build_response(response_body, status_code=HTTPStatus.OK) + + +def _get_document_download_url_response( + document_download_url: str, report_document_id: str, compressed: Optional[bool] = False +) -> HttpResponse: + response_body = {"reportDocumentId": report_document_id, "url": document_download_url} + if compressed: + # See https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference#compressionalgorithm + response_body["compressionAlgorithm"] = "GZIP" + return build_response(response_body, status_code=HTTPStatus.OK) + + +def _download_document_response(stream_name: str, data_format: Optional[str] = "csv", compressed: Optional[bool] = False) -> HttpResponse: + response_body = find_template(stream_name, __file__, data_format) + if compressed: + response_body = gzip.compress(response_body.encode("iso-8859-1")) + return HttpResponse(body=response_body, status_code=HTTPStatus.OK) + + +def _download_document_error_response(compressed: Optional[bool] = False) -> HttpResponse: + response_body = '{"errorDetails":"Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again."}' + if compressed: + response_body = gzip.compress(response_body.encode("iso-8859-1")) + return HttpResponse(body=response_body, status_code=HTTPStatus.OK) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh: + @staticmethod + def _read(stream_name: str, config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=stream_name, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_when_read_then_return_records(self, stream_name: str, data_format: str, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_compressed_report_when_read_then_return_records( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID, compressed=True), + ) + + # a workaround to pass compressed document to the mocked response + document_request = _download_document_request(_DOCUMENT_DOWNLOAD_URL).build() + document_response = _download_document_response(stream_name, data_format=data_format, compressed=True) + document_request_matcher = HttpRequestMatcher(document_request, minimum_number_of_expected_match=1) + http_mocker._matchers.append(document_request_matcher) + + http_mocker._mocker.get( + requests_mock.ANY, + additional_matcher=http_mocker._matches_wrapper(document_request_matcher), + response_list=[{"content": document_response.body, "status_code": document_response.status_code}], + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_status_500_then_200_when_create_report_then_retry_and_return_records( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + http_mocker.post( + _create_report_request(stream_name).build(), + [response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), _create_report_response(_REPORT_ID)], + ) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_status_500_then_200_when_retrieve_report_then_retry_and_return_records( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ], + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_status_500_then_200_when_get_document_url_then_retry_and_return_records( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ], + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_status_500_then_200_when_download_document_then_retry_and_return_records( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _download_document_response(stream_name, data_format=data_format), + ], + ) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_access_forbidden_when_read_then_no_records_and_error_logged( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), response_with_status(status_code=HTTPStatus.FORBIDDEN)) + + output = self._read(stream_name, config()) + message_on_access_forbidden = ( + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate." + ) + assert_message_in_log_output(message_on_access_forbidden, output) + assert len(output.records) == 0 + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_status_cancelled_when_read_then_stream_completed_successfully_and_warn_about_cancellation( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, processing_status=ReportProcessingStatus.CANCELLED), + ) + + message_on_report_cancelled = f"The report for stream '{stream_name}' was cancelled or there is no data to return." + + output = self._read(stream_name, config()) + assert_message_in_log_output(message_on_report_cancelled, output) + assert len(output.records) == 0 + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_status_fatal_when_read_then_exception_raised( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response( + stream_name, processing_status=ReportProcessingStatus.FATAL, report_document_id=_REPORT_DOCUMENT_ID + ), + ) + + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _download_document_error_response(), + ], + ) + + output = self._read(stream_name, config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.config_error + assert ( + f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{CONFIG_END_DATE}. This will be read during the next sync. Error: {{'errorDetails': 'Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again.'}}" + ) in output.errors[-1].trace.error.message + + @pytest.mark.parametrize( + ("stream_name", "date_field", "expected_date_value"), + ( + ("GET_SELLER_FEEDBACK_DATA", "date", "2020-10-20"), + ("GET_LEDGER_DETAIL_VIEW_DATA", "Date", "2021-11-21"), + ("GET_LEDGER_SUMMARY_VIEW_DATA", "Date", "2022-12-22"), + ), + ) + @HttpMocker() + def test_given_report_with_incorrect_date_format_when_read_then_formatted( + self, stream_name: str, date_field: str, expected_date_value: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get(_download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), _download_document_response(stream_name)) + + output = self._read(stream_name, config()) + assert len(output.records) == DEFAULT_EXPECTED_NUMBER_OF_RECORDS + assert output.records[0].record.data.get(date_field) == expected_date_value + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_error_500_on_create_report_when_read_then_no_records_and_error_logged( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post( + _create_report_request(stream_name).build(), + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + ) + + message_on_backoff_exception = f"The report for stream '{stream_name}' was cancelled due to several failed retry attempts." + + output = self._read(stream_name, config()) + assert_message_in_log_output(message_on_backoff_exception, output) + assert len(output.records) == 0 + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental: + default_cursor_field = "dataEndTime" + + @staticmethod + def _read( + stream_name: str, + config_: ConfigBuilder, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: bool = False, + ) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=stream_name, + sync_mode=SyncMode.incremental, + state=state, + expecting_exception=expecting_exception, + ) + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_when_read_then_default_cursor_field_added_to_every_record( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, config()) + assert all([self.default_cursor_field in record.record.data for record in output.records]) + + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_report_when_read_then_state_message_produced_and_state_match_latest_record( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ) -> None: + _config = config() + mock_auth(http_mocker) + + http_mocker.post(_create_report_request(stream_name).build(), _create_report_response(_REPORT_ID)) + http_mocker.get( + _check_report_status_request(_REPORT_ID).build(), + _check_report_status_response(stream_name, report_document_id=_REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _get_document_download_url_request(_REPORT_DOCUMENT_ID).build(), + _get_document_download_url_response(_DOCUMENT_DOWNLOAD_URL, _REPORT_DOCUMENT_ID), + ) + http_mocker.get( + _download_document_request(_DOCUMENT_DOWNLOAD_URL).build(), + _download_document_response(stream_name, data_format=data_format), + ) + + output = self._read(stream_name, _config) + assert len(output.state_messages) == 1 + + cursor_field = get_stream_by_name(stream_name, _config.build()).cursor_field + cursor_value_from_state_message = output.most_recent_state.get(stream_name, {}).get(cursor_field) + cursor_value_from_latest_record = output.records[-1].record.data.get(cursor_field) + assert cursor_value_from_state_message == cursor_value_from_latest_record diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py new file mode 100644 index 000000000000..7706c715d3c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py @@ -0,0 +1,235 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from http import HTTPStatus +from typing import List, Optional + +import freezegun +import pendulum +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode + +from .config import NOW, TIME_FORMAT, ConfigBuilder +from .pagination import NEXT_TOKEN_STRING, VendorFulfillmentPaginationStrategy +from .request_builder import RequestBuilder +from .response_builder import response_with_status +from .utils import config, mock_auth, read_output + +_START_DATE = pendulum.datetime(year=2023, month=1, day=1) +_END_DATE = pendulum.datetime(year=2023, month=1, day=5) +_REPLICATION_START_FIELD = "createdAfter" +_REPLICATION_END_FIELD = "createdBefore" +_CURSOR_FIELD = "createdBefore" +_STREAM_NAME = "VendorDirectFulfillmentShipping" + + +def _vendor_direct_fulfillment_shipping_request() -> RequestBuilder: + return RequestBuilder.vendor_direct_fulfillment_shipping_endpoint().with_query_params( + { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + } + ) + + +def _vendor_direct_fulfillment_shipping_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=NestedPath(["payload", "shippingLabels"]), + pagination_strategy=VendorFulfillmentPaginationStrategy(), + ) + + +def _shipping_label_record() -> RecordBuilder: + return create_record_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=NestedPath(["payload", "shippingLabels"]), + record_id_path=FieldPath("purchaseOrderNumber"), + ) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh: + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + _vendor_direct_fulfillment_shipping_response().with_pagination().with_record(_shipping_label_record()).build(), + ) + query_params_with_next_page_token = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + "nextToken": NEXT_TOKEN_STRING, + } + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_with_next_page_token).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_two_slices_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + end_date = _START_DATE.add(days=8) + mock_auth(http_mocker) + + query_params_first_slice = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _START_DATE.add(days=7).strftime(TIME_FORMAT), + } + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_first_slice).build(), + _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).build(), + ) + + query_params_second_slice = { + _REPLICATION_START_FIELD: query_params_first_slice[_REPLICATION_END_FIELD], + _REPLICATION_END_FIELD: end_date.strftime(TIME_FORMAT), + } + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_second_slice).build(), + _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(end_date)) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_http_status_500_then_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).build(), + ], + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.config_error + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental: + @staticmethod + def _read( + config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.incremental, + state=state, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + _vendor_direct_fulfillment_shipping_response().with_record(_shipping_label_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + expected_cursor_value = _END_DATE.strftime(TIME_FORMAT) + assert output.records[0].record.data[_CURSOR_FIELD] == expected_cursor_value + + @HttpMocker() + def test_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.state_messages) == 1 + + cursor_value_from_state_message = output.most_recent_state.get(_STREAM_NAME, {}).get(_CURSOR_FIELD) + cursor_value_from_latest_record = output.records[-1].record.data.get(_CURSOR_FIELD) + assert cursor_value_from_state_message == cursor_value_from_latest_record + + @HttpMocker() + def test_given_state_when_read_then_state_value_is_created_after_query_param(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + state_value = _START_DATE.add(days=1).strftime(TIME_FORMAT) + + query_params_first_read = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + } + query_params_incremental_read = {_REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT)} + + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_first_read).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), + ) + http_mocker.get( + _vendor_direct_fulfillment_shipping_request().with_query_params(query_params_incremental_read).build(), + _vendor_direct_fulfillment_shipping_response() + .with_record(_shipping_label_record()) + .with_record(_shipping_label_record()) + .build(), + ) + + output = self._read( + config_=config().with_start_date(_START_DATE).with_end_date(_END_DATE), + state=StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_value}).build(), + ) + assert output.most_recent_state == {_STREAM_NAME: {_CURSOR_FIELD: _END_DATE.strftime(TIME_FORMAT)}} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py new file mode 100644 index 000000000000..691bd32608de --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py @@ -0,0 +1,214 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from http import HTTPStatus +from typing import List, Optional + +import freezegun +import pendulum +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode + +from .config import NOW, TIME_FORMAT, ConfigBuilder +from .pagination import NEXT_TOKEN_STRING, VendorFulfillmentPaginationStrategy +from .request_builder import RequestBuilder +from .response_builder import response_with_status +from .utils import config, mock_auth, read_output + +_START_DATE = pendulum.datetime(year=2023, month=1, day=1) +_END_DATE = pendulum.datetime(year=2023, month=1, day=5) +_REPLICATION_START_FIELD = "changedAfter" +_REPLICATION_END_FIELD = "changedBefore" +_CURSOR_FIELD = "changedBefore" +_STREAM_NAME = "VendorOrders" + + +def _vendor_orders_request() -> RequestBuilder: + return RequestBuilder.vendor_orders_endpoint().with_query_params( + { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + } + ) + + +def _vendor_orders_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=NestedPath(["payload", "orders"]), + pagination_strategy=VendorFulfillmentPaginationStrategy(), + ) + + +def _order_record() -> RecordBuilder: + return create_record_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=NestedPath(["payload", "orders"]), + record_id_path=FieldPath("purchaseOrderNumber"), + ) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh: + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get(_vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build()) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_orders_request().build(), + _vendor_orders_response().with_pagination().with_record(_order_record()).build(), + ) + query_params_with_next_page_token = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + "nextToken": NEXT_TOKEN_STRING, + } + http_mocker.get( + _vendor_orders_request().with_query_params(query_params_with_next_page_token).build(), + _vendor_orders_response().with_record(_order_record()).with_record(_order_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_two_slices_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + end_date = _START_DATE.add(days=8) + mock_auth(http_mocker) + + query_params_first_slice = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _START_DATE.add(days=7).strftime(TIME_FORMAT), + } + http_mocker.get( + _vendor_orders_request().with_query_params(query_params_first_slice).build(), + _vendor_orders_response().with_record(_order_record()).build(), + ) + + query_params_second_slice = { + _REPLICATION_START_FIELD: query_params_first_slice[_REPLICATION_END_FIELD], + _REPLICATION_END_FIELD: end_date.strftime(TIME_FORMAT), + } + http_mocker.get( + _vendor_orders_request().with_query_params(query_params_second_slice).build(), + _vendor_orders_response().with_record(_order_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(end_date)) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_http_status_500_then_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_orders_request().build(), + [ + response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR), + _vendor_orders_response().with_record(_order_record()).build(), + ], + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get(_vendor_orders_request().build(), response_with_status(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.config_error + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental: + @staticmethod + def _read( + config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.incremental, + state=state, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get(_vendor_orders_request().build(), _vendor_orders_response().with_record(_order_record()).build()) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + expected_cursor_value = _END_DATE.strftime(TIME_FORMAT) + assert output.records[0].record.data[_CURSOR_FIELD] == expected_cursor_value + + @HttpMocker() + def test_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + http_mocker.get( + _vendor_orders_request().build(), + _vendor_orders_response().with_record(_order_record()).with_record(_order_record()).build(), + ) + + output = self._read(config().with_start_date(_START_DATE).with_end_date(_END_DATE)) + assert len(output.state_messages) == 1 + + cursor_value_from_state_message = output.most_recent_state.get(_STREAM_NAME, {}).get(_CURSOR_FIELD) + cursor_value_from_latest_record = output.records[-1].record.data.get(_CURSOR_FIELD) + assert cursor_value_from_state_message == cursor_value_from_latest_record + + @HttpMocker() + def test_given_state_when_read_then_state_value_is_created_after_query_param(self, http_mocker: HttpMocker) -> None: + mock_auth(http_mocker) + state_value = _START_DATE.add(days=1).strftime(TIME_FORMAT) + + query_params_first_read = { + _REPLICATION_START_FIELD: _START_DATE.strftime(TIME_FORMAT), + _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT), + } + query_params_incremental_read = {_REPLICATION_START_FIELD: state_value, _REPLICATION_END_FIELD: _END_DATE.strftime(TIME_FORMAT)} + + http_mocker.get( + _vendor_orders_request().with_query_params(query_params_first_read).build(), + _vendor_orders_response().with_record(_order_record()).with_record(_order_record()).build(), + ) + http_mocker.get( + _vendor_orders_request().with_query_params(query_params_incremental_read).build(), + _vendor_orders_response().with_record(_order_record()).with_record(_order_record()).build(), + ) + + output = self._read( + config_=config().with_start_date(_START_DATE).with_end_date(_END_DATE), + state=StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_value}).build(), + ) + assert output.most_recent_state == {_STREAM_NAME: {_CURSOR_FIELD: _END_DATE.strftime(TIME_FORMAT)}} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py new file mode 100644 index 000000000000..4631a8adfd48 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus +from typing import Any, List, Mapping, Optional + +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import _get_unit_test_folder +from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, Level, SyncMode +from source_amazon_seller_partner import SourceAmazonSellerPartner + +from .config import ACCESS_TOKEN, ConfigBuilder +from .request_builder import RequestBuilder +from .response_builder import build_response + + +def config() -> ConfigBuilder: + return ConfigBuilder() + + +def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(stream_name, sync_mode).build() + + +def source() -> SourceAmazonSellerPartner: + return SourceAmazonSellerPartner() + + +def read_output( + config_builder: ConfigBuilder, + stream_name: str, + sync_mode: SyncMode, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: Optional[bool] = False, +) -> EntrypointOutput: + _catalog = catalog(stream_name, sync_mode) + _config = config_builder.build() + return read(source(), _config, _catalog, state, expecting_exception) + + +def get_stream_by_name(stream_name: str, config_: Mapping[str, Any]) -> Stream: + streams = [stream for stream in source().streams(config_) if stream.name == stream_name] + if not streams: + raise ValueError("Please provide a valid stream name") + return streams[0] + + +def find_template(resource: str, execution_folder: str, template_format: Optional[str] = "csv") -> str: + response_template_filepath = str( + _get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}.{template_format}" + ) + with open(response_template_filepath, "r") as template_file: + if template_file == "json": + return json.load(template_file) + else: + return template_file.read() + + +def mock_auth(http_mocker: HttpMocker) -> None: + response_body = {"access_token": ACCESS_TOKEN, "expires_in": 3600, "token_type": "bearer"} + http_mocker.post(RequestBuilder.auth_endpoint().build(), build_response(response_body, status_code=HTTPStatus.OK)) + + +def assert_message_in_log_output(message: str, entrypoint_output: EntrypointOutput, log_level: Optional[Level] = Level.WARN) -> None: + assert any( + message in airbyte_message.log.message for airbyte_message in entrypoint_output.logs if airbyte_message.log.level == log_level + ) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA.csv new file mode 100644 index 000000000000..3d90f23a5731 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA.csv @@ -0,0 +1,3 @@ +seller-sku fulfillment-channel-sku asin condition-type Warehouse-Condition-code Quantity Available +seller-sku-1 fulfillment-channel-sku-1 asin-1 condition-type-1 Warehouse-Condition-code-1 11 +seller-sku-1 fulfillment-channel-sku-2 asin-2 condition-type-2 Warehouse-Condition-code-2 22 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA_BY_COUNTRY.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA_BY_COUNTRY.csv new file mode 100644 index 000000000000..73b819c535e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AFN_INVENTORY_DATA_BY_COUNTRY.csv @@ -0,0 +1,3 @@ +seller-sku fulfillment-channel-sku asin condition-type country quantity-for-local-fulfillment +seller-sku-1 fulfillment-channel-sku-1 asin-1 condition-type-1 11 +seller-sku-2 fulfillment-channel-sku-2 asin-2 condition-type-1 22 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.csv new file mode 100644 index 000000000000..e22e1ed5b267 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.csv @@ -0,0 +1,3 @@ +amazon-order-id merchant-order-id shipment-id shipment-item-id amazon-order-item-id merchant-order-item-id purchase-date payments-date shipment-date reporting-date buyer-email buyer-name buyer-phone-number sku product-name quantity-shipped currency item-price item-tax shipping-price shipping-tax gift-wrap-price gift-wrap-tax ship-service-level recipient-name ship-address-1 ship-address-2 ship-address-3 ship-city ship-state ship-postal-code ship-country ship-phone-number bill-address-1 bill-address-2 bill-address-3 bill-city bill-state bill-postal-code bill-country item-promotion-discount ship-promotion-discount carrier tracking-number estimated-arrival-date fulfillment-center-id fulfillment-channel sales-channel +amazon-order-id-1 merchant-order-id-1 shipment-id shipment-item-id-1 amazon-order-item-id-1 merchant-order-item-id-1 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 buyer-email-1 buyer-name-1 buyer-phone-number-1 sku-1 product-name-1 11 USD 111.0 10 12.00 15.99 16.99 0 ship-service-level-1 recipient-name-1 ship-address-1-1 ship-address-2-1 ship-address-3-1 ship-city-1 ship-state-1 ship-postal-code-1 ship-country-1 ship-phone-number-1 bill-address-1-1 bill-address-2-1 bill-address-3-1 bill-city-1 bill-state-1 bill-postal-code-1 bill-country-1 0.0 0.0 carrier-1 tracking-number-1 2022-07-05T08:09:12-07:00 fulfillment-center-id-1 fulfillment-channel sales-channel-1 +amazon-order-id-2 merchant-order-id-2 shipment-id shipment-item-id-2 amazon-order-item-id-2 merchant-order-item-id-2 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 2022-07-05T08:09:12-07:00 buyer-email-2 buyer-name-2 buyer-phone-number-2 sku-2 product-name-2 22 CAD 222.0 20 25.00 15.99 16.99 0 ship-service-level-2 recipient-name-2 ship-address-1-2 ship-address-2-2 ship-address-3-2 ship-city-2 ship-state-2 ship-postal-code-2 ship-country-2 ship-phone-number-2 bill-address-1-2 bill-address-2-2 bill-address-3-2 bill-city-2 bill-state-2 bill-postal-code-2 bill-country-2 0.0 0.0 carrier-2 tracking-number-2 2022-07-05T08:09:12-07:00 fulfillment-center-id-2 fulfillment-channel sales-channel-2 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json new file mode 100644 index 000000000000..3e5ab5a52328 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json @@ -0,0 +1,29 @@ +{ + "reportSpecification": { + "reportType": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT", + "reportOptions": { + "reportPeriod": "WEEK" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "dataByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "purchasedWithAsin": "B1A345B78C", + "purchasedWithRank": 1, + "combinationPct": 0.028 + }, + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "purchasedWithAsin": "B1D345E78F", + "purchasedWithRank": 2, + "combinationPct": 0.0229 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json new file mode 100644 index 000000000000..ff52ea200bbc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json @@ -0,0 +1,39 @@ +{ + "reportSpecification": { + "reportType": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT", + "reportOptions": { + "reportPeriod": "WEEK" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "dataByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "orders": 1256, + "uniqueCustomers": 1201, + "repeatCustomersPctTotal": 0.0083, + "repeatPurchaseRevenue": { + "amount": 2246.13, + "currencyCode": "USD" + }, + "repeatPurchaseRevenuePctTotal": 0.0217 + }, + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B234567890", + "orders": 2561, + "uniqueCustomers": 43, + "repeatCustomersPctTotal": 0.1234, + "repeatPurchaseRevenue": { + "amount": 1234.56, + "currencyCode": "USD" + }, + "repeatPurchaseRevenuePctTotal": 0.0465 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json new file mode 100644 index 000000000000..d0579b35ba2f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json @@ -0,0 +1,31 @@ +{ + "reportSpecification": { + "reportType": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", + "reportOptions": { + "reportPeriod": "WEEK" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-12", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "dataByDepartmentAndSearchTerm": [ + { + "departmentName": "Amazon.com", + "searchTerm": "search term rank one", + "searchFrequencyRank": 1, + "clickedAsin": "B123456789", + "clickShareRank": 1, + "clickShare": 0.0771, + "conversionShare": 0.0874 + }, + { + "departmentName": "Amazon.com", + "searchTerm": "search term rank one", + "searchFrequencyRank": 1, + "clickedAsin": "B987654321", + "clickShareRank": 2, + "clickShare": 0.0726, + "conversionShare": 0.0974 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.csv new file mode 100644 index 000000000000..404688f64a25 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.csv @@ -0,0 +1,3 @@ +sku fnsku asin product-name product-group brand fulfilled-by has-local-inventory your-price sales-price longest-side median-side shortest-side length-and-girth unit-of-dimension item-package-weight unit-of-weight product-size-weight-band currency estimated-fee-total estimated-referral-fee-per-unit estimated-variable-closing-fee expected-domestic-fulfilment-fee-per-unit expected-efn-fulfilment-fee-per-unit-uk expected-efn-fulfilment-fee-per-unit-de expected-efn-fulfilment-fee-per-unit-fr expected-efn-fulfilment-fee-per-unit-it expected-efn-fulfilment-fee-per-unit-es expected-efn-fulfilment-fee-per-unit-se +sku-1 fnsku-1 asin-1 product-name-1 product-group-1 brand-1 fulfilled-by-1 false 111.0 112.0 10 9 8 10 MM 11 LB product-size-weight-band-1 USD 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +sku-2 fnsku-2 asin-2 product-name-2 product-group-2 brand-2 fulfilled-by-2 false 222.0 223.0 20 19 18 20 MM 22 KG product-size-weight-band-2 CAD 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.csv new file mode 100644 index 000000000000..882e088d079b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.csv @@ -0,0 +1,3 @@ +return-date order-id sku asin fnsku product-name quantity fulfillment-center-id detailed-disposition reason status license-plate-number customer-comments +return-date order-id sku asin fnsku product-name quantity fulfillment-center-id detailed-disposition reason status license-plate-number customer-comments +return-date order-id sku asin fnsku product-name quantity fulfillment-center-id detailed-disposition reason status license-plate-number customer-comments diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.csv new file mode 100644 index 000000000000..998dc7477725 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.csv @@ -0,0 +1,3 @@ +shipment-date currency item-promotion-discount item-promotion-id description promotion-rule-value amazon-order-id shipment-id shipment-item-id +shipment-date currency item-promotion-discount item-promotion-id description promotion-rule-value amazon-order-id shipment-id shipment-item-id +shipment-date currency item-promotion-discount item-promotion-id description promotion-rule-value amazon-order-id shipment-id shipment-item-id diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.csv new file mode 100644 index 000000000000..ad5ee9b9902c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.csv @@ -0,0 +1,3 @@ +shipment-date sku asin fulfillment-center-id original-fulfillment-center-id quantity replacement-reason-code replacement-amazon-order-id original-amazon-order-id +shipment-date sku asin fulfillment-center-id original-fulfillment-center-id quantity replacement-reason-code replacement-amazon-order-id original-amazon-order-id +shipment-date sku asin fulfillment-center-id original-fulfillment-center-id quantity replacement-reason-code replacement-amazon-order-id original-amazon-order-id diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.csv new file mode 100644 index 000000000000..152f07f84f5d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.csv @@ -0,0 +1,3 @@ +request-date order-id order-type service-speed order-status last-updated-date sku fnsku disposition requested-quantity cancelled-quantity disposed-quantity shipped-quantity in-process-quantity removal-fee currency +request-date order-id order-type service-speed order-status last-updated-date sku fnsku disposition requested-quantity cancelled-quantity disposed-quantity shipped-quantity in-process-quantity removal-fee currency +request-date order-id order-type service-speed order-status last-updated-date sku fnsku disposition requested-quantity cancelled-quantity disposed-quantity shipped-quantity in-process-quantity removal-fee currency diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.csv new file mode 100644 index 000000000000..96f6732ebb9a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.csv @@ -0,0 +1,3 @@ +request-date order-id shipment-date sku fnsku disposition shipped-quantity carrier tracking-number removal-order-type +request-date order-id shipment-date sku fnsku disposition shipped-quantity carrier tracking-number removal-order-type +request-date order-id shipment-date sku fnsku disposition shipped-quantity carrier tracking-number removal-order-type diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_INVENTORY_PLANNING_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_INVENTORY_PLANNING_DATA.csv new file mode 100644 index 000000000000..1ec5e6746a3a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_INVENTORY_PLANNING_DATA.csv @@ -0,0 +1,3 @@ +snapshot-date sku fnsku asin product-name condition available pending-removal-quantity inv-age-0-to-90-days inv-age-91-to-180-days inv-age-181-to-270-days inv-age-271-to-365-days inv-age-365-plus-days currency qty-to-be-charged-ltsf-11-mo projected-ltsf-11-mo qty-to-be-charged-ltsf-12-mo estimated-ltsf-next-charge units-shipped-t7 units-shipped-t30 units-shipped-t60 units-shipped-t90 alert your-price sales-price lowest-price-new-plus-shipping lowest-price-used recommended-action healthy-inventory-level recommended-sales-price recommended-sale-duration-days recommended-removal-quantity estimated-cost-savings-of-recommended-actions sell-through item-volume volume-unit-measurement storage-type storage-volume marketplace product-group sales-rank days-of-supply estimated-excess-quantity weeks-of-cover-t30 weeks-of-cover-t90 featuredoffer-price sales-shipped-last-7-days sales-shipped-last-30-days sales-shipped-last-60-days sales-shipped-last-90-days inv-age-0-to-30-days inv-age-31-to-60-days inv-age-61-to-90-days inv-age-181-to-330-days inv-age-331-to-365-days estimated-storage-cost-next-month inbound-quantity inbound-working inbound-shipped inbound-received no-sale-last-6-months reserved-quantity unfulfillable-quantity +snapshot-date sku fnsku asin product-name condition available pending-removal-quantity inv-age-0-to-90-days inv-age-91-to-180-days inv-age-181-to-270-days inv-age-271-to-365-days inv-age-365-plus-days currency qty-to-be-charged-ltsf-11-mo projected-ltsf-11-mo qty-to-be-charged-ltsf-12-mo estimated-ltsf-next-charge units-shipped-t7 units-shipped-t30 units-shipped-t60 units-shipped-t90 alert your-price sales-price lowest-price-new-plus-shipping lowest-price-used recommended-action healthy-inventory-level recommended-sales-price recommended-sale-duration-days recommended-removal-quantity estimated-cost-savings-of-recommended-actions sell-through item-volume volume-unit-measurement storage-type storage-volume marketplace product-group sales-rank days-of-supply estimated-excess-quantity weeks-of-cover-t30 weeks-of-cover-t90 featuredoffer-price sales-shipped-last-7-days sales-shipped-last-30-days sales-shipped-last-60-days sales-shipped-last-90-days inv-age-0-to-30-days inv-age-31-to-60-days inv-age-61-to-90-days inv-age-181-to-330-days inv-age-331-to-365-days estimated-storage-cost-next-month inbound-quantity inbound-working inbound-shipped inbound-received no-sale-last-6-months reserved-quantity unfulfillable-quantity +snapshot-date sku fnsku asin product-name condition available pending-removal-quantity inv-age-0-to-90-days inv-age-91-to-180-days inv-age-181-to-270-days inv-age-271-to-365-days inv-age-365-plus-days currency qty-to-be-charged-ltsf-11-mo projected-ltsf-11-mo qty-to-be-charged-ltsf-12-mo estimated-ltsf-next-charge units-shipped-t7 units-shipped-t30 units-shipped-t60 units-shipped-t90 alert your-price sales-price lowest-price-new-plus-shipping lowest-price-used recommended-action healthy-inventory-level recommended-sales-price recommended-sale-duration-days recommended-removal-quantity estimated-cost-savings-of-recommended-actions sell-through item-volume volume-unit-measurement storage-type storage-volume marketplace product-group sales-rank days-of-supply estimated-excess-quantity weeks-of-cover-t30 weeks-of-cover-t90 featuredoffer-price sales-shipped-last-7-days sales-shipped-last-30-days sales-shipped-last-60-days sales-shipped-last-90-days inv-age-0-to-30-days inv-age-31-to-60-days inv-age-61-to-90-days inv-age-181-to-330-days inv-age-331-to-365-days estimated-storage-cost-next-month inbound-quantity inbound-working inbound-shipped inbound-received no-sale-last-6-months reserved-quantity unfulfillable-quantity diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.csv new file mode 100644 index 000000000000..039a6d93362f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.csv @@ -0,0 +1,3 @@ +sku fnsku asin product-name condition your-price mfn-listing-exists mfn-fulfillable-quantity afn-listing-exists afn-warehouse-quantity afn-fulfillable-quantity afn-unsellable-quantity afn-reserved-quantity afn-total-quantity per-unit-volume afn-inbound-working-quantity afn-inbound-shipped-quantity afn-inbound-receiving-quantity afn-researching-quantity afn-reserved-future-supply afn-future-supply-buyable +sku fnsku asin product-name condition your-price mfn-listing-exists mfn-fulfillable-quantity afn-listing-exists afn-warehouse-quantity afn-fulfillable-quantity afn-unsellable-quantity afn-reserved-quantity afn-total-quantity per-unit-volume afn-inbound-working-quantity afn-inbound-shipped-quantity afn-inbound-receiving-quantity afn-researching-quantity afn-reserved-future-supply afn-future-supply-buyable +sku fnsku asin product-name condition your-price mfn-listing-exists mfn-fulfillable-quantity afn-listing-exists afn-warehouse-quantity afn-fulfillable-quantity afn-unsellable-quantity afn-reserved-quantity afn-total-quantity per-unit-volume afn-inbound-working-quantity afn-inbound-shipped-quantity afn-inbound-receiving-quantity afn-researching-quantity afn-reserved-future-supply afn-future-supply-buyable diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_REIMBURSEMENTS_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_REIMBURSEMENTS_DATA.csv new file mode 100644 index 000000000000..50b8dff481c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_REIMBURSEMENTS_DATA.csv @@ -0,0 +1,3 @@ +approval-date reimbursement-id case-id amazon-order-id reason sku fnsku asin product-name condition currency-unit amount-per-unit amount-total quantity-reimbursed-cash quantity-reimbursed-inventory quantity-reimbursed-total original-reimbursement-id original-reimbursement-type +approval-date reimbursement-id case-id amazon-order-id reason sku fnsku asin product-name condition currency-unit amount-per-unit amount-total quantity-reimbursed-cash quantity-reimbursed-inventory quantity-reimbursed-total original-reimbursement-id original-reimbursement-type +approval-date reimbursement-id case-id amazon-order-id reason sku fnsku asin product-name condition currency-unit amount-per-unit amount-total quantity-reimbursed-cash quantity-reimbursed-inventory quantity-reimbursed-total original-reimbursement-id original-reimbursement-type diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_FORECAST_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_FORECAST_DATA.csv new file mode 100644 index 000000000000..5e0a7eed1897 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_FORECAST_DATA.csv @@ -0,0 +1,3 @@ +offer-state snapshot-date sku fnsku asin estimated-avg-sns-discount-next-8-weeks product-name country active-subscriptions week-1-start-date scheduled-sns-units-week-1 scheduled-sns-units-week-2 scheduled-sns-units-week-3 scheduled-sns-units-week-4 scheduled-sns-units-week-5 scheduled-sns-units-week-6 scheduled-sns-units-week-7 scheduled-sns-units-week-8 +offer-state snapshot-date sku fnsku asin estimated-avg-sns-discount-next-8-weeks product-name country active-subscriptions week-1-start-date scheduled-sns-units-week-1 scheduled-sns-units-week-2 scheduled-sns-units-week-3 scheduled-sns-units-week-4 scheduled-sns-units-week-5 scheduled-sns-units-week-6 scheduled-sns-units-week-7 scheduled-sns-units-week-8 +offer-state snapshot-date sku fnsku asin estimated-avg-sns-discount-next-8-weeks product-name country active-subscriptions week-1-start-date scheduled-sns-units-week-1 scheduled-sns-units-week-2 scheduled-sns-units-week-3 scheduled-sns-units-week-4 scheduled-sns-units-week-5 scheduled-sns-units-week-6 scheduled-sns-units-week-7 scheduled-sns-units-week-8 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_PERFORMANCE_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_PERFORMANCE_DATA.csv new file mode 100644 index 000000000000..1d3f83abf13f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_SNS_PERFORMANCE_DATA.csv @@ -0,0 +1,3 @@ +offer-state snapshot-date sku fnsku asin product-name country week-1-start-date sns-units-shipped-week-1 oos-rate-week-1 sns-sale-price-week-1 sns-discount-week-1 sns-units-shipped-week-2 oos-rate-week-2 sns-sale-price-week-2 sns-discount-week-2 sns-units-shipped-week-3 oos-rate-week-3 sns-sale-price-week-3 sns-discount-week-3 sns-units-shipped-week-4 oos-rate-week-4 sns-sale-price-week-4 sns-discount-week-4 +offer-state snapshot-date sku fnsku asin product-name country week-1-start-date sns-units-shipped-week-1 oos-rate-week-1 sns-sale-price-week-1 sns-discount-week-1 sns-units-shipped-week-2 oos-rate-week-2 sns-sale-price-week-2 sns-discount-week-2 sns-units-shipped-week-3 oos-rate-week-3 sns-sale-price-week-3 sns-discount-week-3 sns-units-shipped-week-4 oos-rate-week-4 sns-sale-price-week-4 sns-discount-week-4 +offer-state snapshot-date sku fnsku asin product-name country week-1-start-date sns-units-shipped-week-1 oos-rate-week-1 sns-sale-price-week-1 sns-discount-week-1 sns-units-shipped-week-2 oos-rate-week-2 sns-sale-price-week-2 sns-discount-week-2 sns-units-shipped-week-3 oos-rate-week-3 sns-sale-price-week-3 sns-discount-week-3 sns-units-shipped-week-4 oos-rate-week-4 sns-sale-price-week-4 sns-discount-week-4 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_STORAGE_FEE_CHARGES_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_STORAGE_FEE_CHARGES_DATA.csv new file mode 100644 index 000000000000..5e444ce2142c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FBA_STORAGE_FEE_CHARGES_DATA.csv @@ -0,0 +1,3 @@ +asin fnsku product_name fulfillment_center country_code longest_side median_side shortest_side measurement_units weight weight_units item_volume volume_units product_size_tier average_quantity_on_hand average_quantity_pending_removal estimated_total_item_volume month_of_charge storage_rate currency estimated_monthly_storage_fee dangerous_goods_storage_type eligible_for_inventory_discount qualifies_for_inventory_discount total_incentive_fee_amount breakdown_incentive_fee_amount average_quantity_customer_orders +asin fnsku product_name fulfillment_center country_code longest_side median_side shortest_side measurement_units weight weight_units item_volume volume_units product_size_tier average_quantity_on_hand average_quantity_pending_removal estimated_total_item_volume month_of_charge storage_rate currency estimated_monthly_storage_fee dangerous_goods_storage_type eligible_for_inventory_discount qualifies_for_inventory_discount total_incentive_fee_amount breakdown_incentive_fee_amount average_quantity_customer_orders +asin fnsku product_name fulfillment_center country_code longest_side median_side shortest_side measurement_units weight weight_units item_volume volume_units product_size_tier average_quantity_on_hand average_quantity_pending_removal estimated_total_item_volume month_of_charge storage_rate currency estimated_monthly_storage_fee dangerous_goods_storage_type eligible_for_inventory_discount qualifies_for_inventory_discount total_incentive_fee_amount breakdown_incentive_fee_amount average_quantity_customer_orders diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.csv new file mode 100644 index 000000000000..308f234839cd --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.csv @@ -0,0 +1,3 @@ +order-id order-item-id purchase-date payments-date reporting-date promise-date days-past-promise buyer-email buyer-phone-number sku product-name quantity-purchased quantity-shipped quantity-to-ship ship-service-level recipient-name ship-address-1 ship-address-2 ship-address-3 ship-city ship-state ship-postal-code ship-country is-business-order purchase-order-number price-designation is-prime +order-id order-item-id purchase-date payments-date reporting-date promise-date days-past-promise buyer-email buyer-phone-number sku product-name quantity-purchased quantity-shipped quantity-to-ship ship-service-level recipient-name ship-address-1 ship-address-2 ship-address-3 ship-city ship-state ship-postal-code ship-country is-business-order purchase-order-number price-designation is-prime +order-id order-item-id purchase-date payments-date reporting-date promise-date days-past-promise buyer-email buyer-phone-number sku product-name quantity-purchased quantity-shipped quantity-to-ship ship-service-level recipient-name ship-address-1 ship-address-2 ship-address-3 ship-city ship-state ship-postal-code ship-country is-business-order purchase-order-number price-designation is-prime diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.csv new file mode 100644 index 000000000000..80b4bcf8d400 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.csv @@ -0,0 +1,3 @@ +amazon-order-id merchant-order-id purchase-date last-updated-date order-status fulfillment-channel sales-channel order-channel url ship-service-level product-name sku asin item-status quantity currency item-price item-tax shipping-price shipping-tax gift-wrap-price gift-wrap-tax item-promotion-discount ship-promotion-discount ship-country ship-promotion-id promotion-ids is-business-order purchase-order-number price-designation customized-url customized-page is-heavy-or-bulky is-replacement-order +amazon-order-id merchant-order-id purchase-date last-updated-date order-status fulfillment-channel sales-channel order-channel url ship-service-level product-name sku asin item-status quantity currency item-price item-tax shipping-price shipping-tax gift-wrap-price gift-wrap-tax item-promotion-discount ship-promotion-discount ship-country ship-promotion-id promotion-ids is-business-order purchase-order-number price-designation customized-url customized-page is-heavy-or-bulky is-replacement-order +amazon-order-id merchant-order-id purchase-date last-updated-date order-status fulfillment-channel sales-channel order-channel url ship-service-level product-name sku asin item-status quantity currency item-price item-tax shipping-price shipping-tax gift-wrap-price gift-wrap-tax item-promotion-discount ship-promotion-discount ship-country ship-promotion-id promotion-ids is-business-order purchase-order-number price-designation customized-url customized-page is-heavy-or-bulky is-replacement-order diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.csv new file mode 100644 index 000000000000..4dd55103fdba --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.csv @@ -0,0 +1,3 @@ +Order ID Order date Return request date Return request status Amazon RMA ID Merchant RMA ID Label type Label cost Currency code Return carrier Tracking ID Label to be paid by A-to-Z Claim Is prime ASIN Merchant SKU Item Name Return quantity Return Reason In policy Return type Resolution Invoice number Return delivery date Order Amount Order quantity SafeT Action reason SafeT claim id SafeT claim state SafeT claim creation time SafeT claim reimbursement amount Refunded Amount +Order ID Order date Return request date Return request status Amazon RMA ID Merchant RMA ID Label type Label cost Currency code Return carrier Tracking ID Label to be paid by A-to-Z Claim Is prime ASIN Merchant SKU Item Name Return quantity Return Reason In policy Return type Resolution Invoice number Return delivery date Order Amount Order quantity SafeT Action reason SafeT claim id SafeT claim state SafeT claim creation time SafeT claim reimbursement amount Refunded Amount +Order ID Order date Return request date Return request status Amazon RMA ID Merchant RMA ID Label type Label cost Currency code Return carrier Tracking ID Label to be paid by A-to-Z Claim Is prime ASIN Merchant SKU Item Name Return quantity Return Reason In policy Return type Resolution Invoice number Return delivery date Order Amount Order quantity SafeT Action reason SafeT claim id SafeT claim state SafeT claim creation time SafeT claim reimbursement amount Refunded Amount diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_DETAIL_VIEW_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_DETAIL_VIEW_DATA.csv new file mode 100644 index 000000000000..1df2f6161f65 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_DETAIL_VIEW_DATA.csv @@ -0,0 +1,3 @@ +Date FNSKU ASIN MSKU Title EventType ReferenceID Quantity FulfillmentCenter Disposition Reason Country ReconciledQuantity UnreconciledQuantity +11/21/2021 FNSKU ASIN MSKU Title EventType ReferenceID Quantity FulfillmentCenter Disposition Reason Country ReconciledQuantity UnreconciledQuantity +11/21/2021 FNSKU ASIN MSKU Title EventType ReferenceID Quantity FulfillmentCenter Disposition Reason Country ReconciledQuantity UnreconciledQuantity diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_SUMMARY_VIEW_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_SUMMARY_VIEW_DATA.csv new file mode 100644 index 000000000000..e794b787c519 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_LEDGER_SUMMARY_VIEW_DATA.csv @@ -0,0 +1,3 @@ +Date FNSKU ASIN MSKU Title Disposition StartingWarehouseBalance InTransitBetweenWarehouses Receipts CustomerShipments CustomerReturns VendorReturns WarehouseTransferIn/Out Found Lost Damaged Disposed OtherEvents EndingWarehouseBalance UnknownEvents Location +12/22/2022 FNSKU ASIN MSKU Title Disposition StartingWarehouseBalance InTransitBetweenWarehouses Receipts CustomerShipments CustomerReturns VendorReturns WarehouseTransferIn/Out Found Lost Damaged Disposed OtherEvents EndingWarehouseBalance UnknownEvents Location +12/22/2022 FNSKU ASIN MSKU Title Disposition StartingWarehouseBalance InTransitBetweenWarehouses Receipts CustomerShipments CustomerReturns VendorReturns WarehouseTransferIn/Out Found Lost Damaged Disposed OtherEvents EndingWarehouseBalance UnknownEvents Location diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_MERCHANT_CANCELLED_LISTINGS_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_MERCHANT_CANCELLED_LISTINGS_DATA.csv new file mode 100644 index 000000000000..7b0df2b1bcc6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_MERCHANT_CANCELLED_LISTINGS_DATA.csv @@ -0,0 +1,3 @@ +item-name item-description seller-sku price quantity image-url item-is-marketplace product-id-type zshop-shipping-fee item-note item-condition zshop-category1 zshop-browse-path zshop-storefront-feature asin1 asin2 asin3 will-ship-internationally expedited-shipping zshop-boldface product-id add-delete merchant-shipping-group +item-name item-description seller-sku price quantity image-url item-is-marketplace product-id-type zshop-shipping-fee item-note item-condition zshop-category1 zshop-browse-path zshop-storefront-feature asin1 asin2 asin3 will-ship-internationally expedited-shipping zshop-boldface product-id add-delete merchant-shipping-group +item-name item-description seller-sku price quantity image-url item-is-marketplace product-id-type zshop-shipping-fee item-note item-condition zshop-category1 zshop-browse-path zshop-storefront-feature asin1 asin2 asin3 will-ship-internationally expedited-shipping zshop-boldface product-id add-delete merchant-shipping-group diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_ORDER_REPORT_DATA_SHIPPING.xml b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_ORDER_REPORT_DATA_SHIPPING.xml new file mode 100644 index 000000000000..b588785b665f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_ORDER_REPORT_DATA_SHIPPING.xml @@ -0,0 +1,101 @@ + + +
      + 1.01 + M_SELLER_354577 +
      + Product + true + + 1 + + AmazonOrderID 1 + AmazonSessionID 1 + 2022-07-05 + 2022-07-05 + + BuyerEmailAddress 1 + BuyerName 1 + BuyerPhoneNumber 1 + + + FulfillmentMethod 1 + FulfillmentServiceLevel 1 +
      + Name 1 + AddressFieldOne 1 + City 1 + PostalCode 1 + CountryCode 1 + PhoneNumber 1 +
      +       + false + true + false + + AmazonOrderItemCode 1 + SKU 1 + Title 1 + 11 + ProductTaxCode 1 + + Type 1 + 111.00 + USD + + + Type 1 + 0.00 + USD + + +       +       + + 2 + + AmazonOrderID 2 + AmazonSessionID 2 + 2022-07-06 + 2022-07-06 + + BuyerEmailAddress 2 + BuyerName 2 + BuyerPhoneNumber 2 + + + FulfillmentMethod 2 + FulfillmentServiceLevel 2 +
      + Name 2 + AddressFieldOne 2 + City 2 + PostalCode 2 + CountryCode 2 + PhoneNumber 2 +
      +       + false + true + false + + AmazonOrderItemCode 2 + SKU 2 + Title 2 + 2 + ProductTaxCode 2 + + Type 2 + 222.00 + USD + + + Type 2 + 0.00 + USD + + +       +       +       diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SALES_AND_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SALES_AND_TRAFFIC_REPORT.json new file mode 100644 index 000000000000..e5782a60a52b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SALES_AND_TRAFFIC_REPORT.json @@ -0,0 +1,123 @@ +{ + "reportSpecification": { + "reportType": "GET_SALES_AND_TRAFFIC_REPORT", + "reportOptions": { + "dateGranularity": "DAY", + "asinGranularity": "SKU" + }, + "dataStartTime": "2021-06-11", + "dataEndTime": "2021-06-14", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "salesAndTrafficByDate": [ + { + "date": "2021-06-11", + "salesByDate": { + "orderedProductSales": { + "amount": 238.44, + "currencyCode": "USD" + }, + "unitsOrdered": 23, + "totalOrderItems": 20, + "averageSalesPerOrderItem": { + "amount": 11.92, + "currencyCode": "USD" + }, + "averageUnitsPerOrderItem": 1.15, + "averageSellingPrice": { + "amount": 10.37, + "currencyCode": "USD" + }, + "unitsRefunded": 1, + "refundRate": 4.35, + "claimsGranted": 0, + "claimsAmount": { + "amount": 0.0, + "currencyCode": "USD" + }, + "shippedProductSales": { + "amount": 650.72, + "currencyCode": "USD" + }, + "unitsShipped": 59, + "ordersShipped": 54 + }, + "trafficByDate": { + "browserPageViews": 1158, + "mobileAppPageViews": 500, + "pageViews": 1658, + "browserSessions": 906, + "mobileAppSessions": 94, + "sessions": 1000, + "buyBoxPercentage": 10.54, + "orderItemSessionPercentage": 2.21, + "unitSessionPercentage": 2.54, + "averageOfferCount": 9686, + "averageParentItems": 9630, + "feedbackReceived": 10, + "negativeFeedbackReceived": 1, + "receivedNegativeFeedbackRate": 10.0 + } + } + ], + "salesAndTrafficByAsin": [ + { + "parentAsin": "B123456789", + "childAsin": "B123456789", + "sku": "AB-1C2D-EFGH", + "salesByAsin": { + "unitsOrdered": 1, + "orderedProductSales": { + "amount": 16.79, + "currencyCode": "USD" + }, + "totalOrderItems": 1 + }, + "trafficByAsin": { + "browserSessions": 13, + "mobileAppSessions": 5, + "sessions": 18, + "browserSessionPercentage": 0.33, + "mobileAppSessionPercentage": 0.2, + "sessionPercentage": 0.26, + "browserPageViews": 21, + "mobileAppPageViews": 22, + "pageViews": 43, + "browserPageViewsPercentage": 0.41, + "mobileAppPageViewsPercentage": 0.2, + "pageViewsPercentage": 0.3, + "buyBoxPercentage": 95.24, + "unitSessionPercentage": 7.69 + } + }, + { + "parentAsin": "B234567890", + "childAsin": "B234567890", + "sku": "CD-2E3F-GHIJ", + "salesByAsin": { + "unitsOrdered": 3, + "orderedProductSales": { + "amount": 26.25, + "currencyCode": "USD" + }, + "totalOrderItems": 2 + }, + "trafficByAsin": { + "browserSessions": 8, + "mobileAppSessions": 5, + "sessions": 13, + "browserSessionPercentage": 0.33, + "mobileAppSessionPercentage": 0.1, + "sessionPercentage": 0.2, + "browserPageViews": 21, + "mobileAppPageViews": 12, + "pageViews": 33, + "browserPageViewsPercentage": 0.41, + "mobileAppPageViewsPercentage": 0.25, + "pageViewsPercentage": 0.32, + "buyBoxPercentage": 0.0, + "unitSessionPercentage": 37.5 + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SELLER_FEEDBACK_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SELLER_FEEDBACK_DATA.csv new file mode 100644 index 000000000000..1d11467128aa --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_SELLER_FEEDBACK_DATA.csv @@ -0,0 +1,3 @@ +Date Rating Comments Your Response Order ID Rater Email +10/20/20 Rating Comments Your Response Order ID Rater Email +10/20/20 Rating Comments Your Response Order ID Rater Email diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_STRANDED_INVENTORY_UI_DATA.csv b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_STRANDED_INVENTORY_UI_DATA.csv new file mode 100644 index 000000000000..f49d85f0efd3 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_STRANDED_INVENTORY_UI_DATA.csv @@ -0,0 +1,3 @@ +primary-action date-stranded Date-to-take-auto-removal status-primary status-secondary error-message stranded-reason asin sku fnsku product-name condition fulfilled-by fulfillable-qty your-price unfulfillable-qty reserved-quantity inbound-shipped-qty +primary-action date-stranded Date-to-take-auto-removal status-primary status-secondary error-message stranded-reason asin sku fnsku product-name condition fulfilled-by fulfillable-qty your-price unfulfillable-qty reserved-quantity inbound-shipped-qty +primary-action date-stranded Date-to-take-auto-removal status-primary status-secondary error-message stranded-reason asin sku fnsku product-name condition fulfilled-by fulfillable-qty your-price unfulfillable-qty reserved-quantity inbound-shipped-qty diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_INVENTORY_REPORT.json new file mode 100644 index 000000000000..3ea8e2298d55 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_INVENTORY_REPORT.json @@ -0,0 +1,167 @@ +{ + "reportSpecification": { + "reportType": "GET_VENDOR_INVENTORY_REPORT", + "reportOptions": { + "reportPeriod": "WEEK", + "sellingProgram": "RETAIL", + "distributorView": "MANUFACTURING" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "inventoryAggregate": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "vendorConfirmationRate": 0.88, + "netReceivedInventoryCost": { + "amount": 2345.5, + "currencyCode": "USD" + }, + "netReceivedInventoryUnits": 278, + "openPurchaseOrderUnits": 123, + "averageVendorLeadTimeDays": 10.2, + "sellThroughRate": 0.88, + "unfilledCustomerOrderedUnits": 12, + "sellableOnHandInventoryCost": { + "amount": 43123.99, + "currencyCode": "USD" + }, + "sellableOnHandInventoryUnits": 5490, + "unsellableOnHandInventoryCost": { + "amount": 2345.5, + "currencyCode": "USD" + }, + "unsellableOnHandInventoryUnits": 881, + "aged90PlusDaysSellableInventoryCost": { + "amount": 123.5, + "currencyCode": "USD" + }, + "aged90PlusDaysSellableInventoryUnits": 2234, + "unhealthyInventoryCost": { + "amount": 123.45, + "currencyCode": "USD" + }, + "unhealthyInventoryUnits": 114, + "procurableProductOutOfStockRate": 0.72, + "uft": 0.19, + "receiveFillRate": 0.67 + }, + { + "startDate": "2021-06-13", + "endDate": "2021-06-19", + "vendorConfirmationRate": 0.98, + "netReceivedInventoryCost": { + "amount": 4335.5, + "currencyCode": "USD" + }, + "netReceivedInventoryUnits": 123, + "openPurchaseOrderUnits": 422, + "averageVendorLeadTimeDays": 5.2, + "sellThroughRate": 0.98, + "unfilledCustomerOrderedUnits": 3, + "sellableOnHandInventoryCost": { + "amount": 43123.99, + "currencyCode": "USD" + }, + "sellableOnHandInventoryUnits": 4490, + "unsellableOnHandInventoryCost": { + "amount": 3345.5, + "currencyCode": "USD" + }, + "unsellableOnHandInventoryUnits": 881, + "aged90PlusDaysSellableInventoryCost": { + "amount": 323.5, + "currencyCode": "USD" + }, + "aged90PlusDaysSellableInventoryUnits": 2234, + "unhealthyInventoryCost": { + "amount": 323.45, + "currencyCode": "USD" + }, + "unhealthyInventoryUnits": 314, + "procurableProductOutOfStockRate": 0.73, + "uft": 0.18, + "receiveFillRate": 0.77 + } + ], + "inventoryByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "vendorConfirmationRate": 0.88, + "netReceivedInventoryCost": { + "amount": 2345.5, + "currencyCode": "USD" + }, + "netReceivedInventoryUnits": 278, + "openPurchaseOrderUnits": 123, + "averageVendorLeadTimeDays": 10.2, + "sellThroughRate": 0.88, + "unfilledCustomerOrderedUnits": 12, + "sellableOnHandInventoryCost": { + "amount": 43123.99, + "currencyCode": "USD" + }, + "sellableOnHandInventoryUnits": 5490, + "unsellableOnHandInventoryCost": { + "amount": 2345.5, + "currencyCode": "USD" + }, + "unsellableOnHandInventoryUnits": 881, + "aged90PlusDaysSellableInventoryCost": { + "amount": 123.5, + "currencyCode": "USD" + }, + "aged90PlusDaysSellableInventoryUnits": 2234, + "unhealthyInventoryCost": { + "amount": 123.45, + "currencyCode": "USD" + }, + "unhealthyInventoryUnits": 114, + "procurableProductOutOfStockRate": 0.47, + "uft": 0.26, + "receiveFillRate": 0.98 + }, + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B987654321", + "vendorConfirmationRate": 0.22, + "netReceivedInventoryCost": { + "amount": 235.5, + "currencyCode": "USD" + }, + "netReceivedInventoryUnits": 78, + "openPurchaseOrderUnits": 23, + "averageVendorLeadTimeDays": 1.2, + "sellThroughRate": 0.28, + "unfilledCustomerOrderedUnits": 1, + "sellableOnHandInventoryCost": { + "amount": 123.99, + "currencyCode": "USD" + }, + "sellableOnHandInventoryUnits": 590, + "unsellableOnHandInventoryCost": { + "amount": 245.5, + "currencyCode": "USD" + }, + "unsellableOnHandInventoryUnits": 81, + "aged90PlusDaysSellableInventoryCost": { + "amount": 13.5, + "currencyCode": "USD" + }, + "aged90PlusDaysSellableInventoryUnits": 234, + "unhealthyInventoryCost": { + "amount": 23.45, + "currencyCode": "USD" + }, + "unhealthyInventoryUnits": 14, + "procurableProductOutOfStockRate": 0.25, + "uft": 0.49, + "receiveFillRate": 0.81 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json new file mode 100644 index 000000000000..41f291c7f0e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json @@ -0,0 +1,37 @@ +{ + "reportSpecification": { + "reportType": "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", + "reportOptions": { + "reportPeriod": "WEEK" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "netPureProductMarginAggregate": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "netPureProductMargin": 0.1234 + }, + { + "startDate": "2021-06-13", + "endDate": "2021-06-19", + "netPureProductMargin": 0.1234 + } + ], + "netPureProductMarginByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "netPureProductMargin": 0.1234 + }, + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B987654321", + "netPureProductMargin": 0.1234 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json new file mode 100644 index 000000000000..b289bb968e38 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json @@ -0,0 +1,22 @@ +{ + "reportSpecification": { + "reportType": "GET_VENDOR_REAL_TIME_INVENTORY_REPORT", + "dataStartTime": "2022-10-01T00:00:00Z", + "dataEndTime": "2022-10-01T02:00:00Z", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "reportData": [ + { + "startTime": "2022-10-01T00:00:00Z", + "endTime": "2022-10-01T01:00:00Z", + "asin": "B123456789", + "highlyAvailableInventory": 270 + }, + { + "startTime": "2022-10-01T00:00:00Z", + "endTime": "2022-10-01T01:00:00Z", + "asin": "B987654321", + "highlyAvailableInventory": 650 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_SALES_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_SALES_REPORT.json new file mode 100644 index 000000000000..43ab6d456d60 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_SALES_REPORT.json @@ -0,0 +1,95 @@ +{ + "reportSpecification": { + "reportType": "GET_VENDOR_SALES_REPORT", + "reportOptions": { + "distributorView": "MANUFACTURING", + "reportPeriod": "WEEK", + "sellingProgram": "RETAIL" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "salesAggregate": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "customerReturns": 0, + "orderedRevenue": { + "amount": 1500.0, + "currencyCode": "USD" + }, + "orderedUnits": 75, + "shippedCogs": { + "amount": 90.0, + "currencyCode": "USD" + }, + "shippedRevenue": { + "amount": 200.0, + "currencyCode": "USD" + }, + "shippedUnits": 10 + }, + { + "startDate": "2021-06-13", + "endDate": "2021-06-19", + "customerReturns": 0, + "orderedRevenue": { + "amount": 0.0, + "currencyCode": "USD" + }, + "orderedUnits": 0, + "shippedCogs": { + "amount": 0.0, + "currencyCode": "USD" + }, + "shippedRevenue": { + "amount": 0.0, + "currencyCode": "USD" + }, + "shippedUnits": 0 + } + ], + "salesByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "customerReturns": 0, + "orderedRevenue": { + "amount": 1000.0, + "currencyCode": "USD" + }, + "orderedUnits": 25, + "shippedCogs": { + "amount": 50.0, + "currencyCode": "USD" + }, + "shippedRevenue": { + "amount": 150.0, + "currencyCode": "USD" + }, + "shippedUnits": 5 + }, + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B987654321", + "customerReturns": 0, + "orderedRevenue": { + "amount": 500, + "currencyCode": "USD" + }, + "orderedUnits": 50, + "shippedCogs": { + "amount": 40.0, + "currencyCode": "USD" + }, + "shippedRevenue": { + "amount": 50.0, + "currencyCode": "USD" + }, + "shippedUnits": 5 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_TRAFFIC_REPORT.json new file mode 100644 index 000000000000..fcf1c3d30f3c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/GET_VENDOR_TRAFFIC_REPORT.json @@ -0,0 +1,37 @@ +{ + "reportSpecification": { + "reportType": "GET_VENDOR_TRAFFIC_REPORT", + "reportOptions": { + "reportPeriod": "WEEK" + }, + "dataStartTime": "2021-06-06", + "dataEndTime": "2021-06-19", + "marketplaceIds": ["ATVPDKIKX0DER"] + }, + "trafficAggregate": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "glanceViews": 100 + }, + { + "startDate": "2021-06-13", + "endDate": "2021-06-19", + "glanceViews": 250 + } + ], + "trafficByAsin": [ + { + "startDate": "2021-06-06", + "endDate": "2021-06-12", + "asin": "B123456789", + "glanceViews": 100 + }, + { + "startDate": "2021-06-13", + "endDate": "2021-06-19", + "asin": "B123456789", + "glanceViews": 250 + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorDirectFulfillmentShipping.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorDirectFulfillmentShipping.json new file mode 100644 index 000000000000..591d1b53f52a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorDirectFulfillmentShipping.json @@ -0,0 +1,44 @@ +{ + "payload": { + "shippingLabels": [ + { + "purchaseOrderNumber": "2JK3S9VCE", + "sellingParty": { + "partyId": "999US" + }, + "shipFromParty": { + "partyId": "ABCD" + }, + "labelFormat": "PNG", + "labelData": [ + { + "packageIdentifier": "PKG001", + "trackingNumber": "1Z6A34Y60369738804", + "shipMethod": "UPS_GR_RES", + "shipMethodName": "UPS Ground Residential", + "content": "Base 64 encoded string goes here" + } + ] + }, + { + "purchaseOrderNumber": "2JK3S9VD", + "sellingParty": { + "partyId": "999US" + }, + "shipFromParty": { + "partyId": "ABCD" + }, + "labelFormat": "PNG", + "labelData": [ + { + "packageIdentifier": "PKG002", + "trackingNumber": "1Z6A34Y60369738805", + "shipMethod": "UPS_GR_RES", + "shipMethodName": "UPS Ground Residential", + "content": "Base 64 encoded string goes here" + } + ] + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorOrders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorOrders.json new file mode 100644 index 000000000000..b3426ef45d04 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/resource/http/response/VendorOrders.json @@ -0,0 +1,90 @@ +{ + "payload": { + "orders": [ + { + "purchaseOrderNumber": "L8266355", + "purchaseOrderState": "New", + "orderDetails": { + "purchaseOrderDate": "2019-05-23T10:00:00Z", + "purchaseOrderChangedDate": "2019-05-24T16:05:00Z", + "purchaseOrderStateChangedDate": "2019-05-23T10:00:00Z", + "purchaseOrderType": "RegularOrder", + "importDetails": { + "methodOfPayment": "PaidByBuyer", + "internationalCommercialTerms": "ExWorks", + "portOfDelivery": "YANTIAN, CHINA", + "importContainers": "1-40'HC, 1-20'", + "shippingInstructions": "PREFERENCE IS PALLET-LOAD, BUT IF CONTAINERS ARE FLOOR-LOADED, THEN PLEASE DO CLAMP-LOAD OR STRAIGHT FLOOR-LOAD. DO NOT USE SLIP SHEET FOR THIS FC DESTINATION. PAYMENT TERMS ARE PER CONTAINER." + }, + "dealCode": "BTS", + "paymentMethod": "Invoice", + "buyingParty": { + "partyId": "ABCD", + "address": { + "name": "APPARIO RETAIL PVT.LTD.", + "addressLine1": "3APPARIO RETAIL PVT.LTD.- C/O. AMAZON SELLER SERVIC", + "city": "Siddhapudur", + "stateOrRegion": "Tamil Nadu", + "postalCode": "641044", + "countryCode": "IN", + "phone": "206-266-8000" + } + }, + "sellingParty": { + "partyId": "TEST1" + }, + "shipToParty": { + "partyId": "ABCD", + "address": { + "name": "APPARIO RETAIL PVT.LTD.", + "addressLine1": "3APPARIO RETAIL PVT.LTD.- C/O. AMAZON SELLER SERVIC", + "city": "Siddhapudur", + "stateOrRegion": "Tamil Nadu", + "postalCode": "641044", + "countryCode": "IN", + "phone": "206-266-8000" + } + }, + "billToParty": { + "partyId": "ABCD", + "address": { + "name": "APPARIO RETAIL PVT.LTD.", + "addressLine1": "3APPARIO RETAIL PVT.LTD.- C/O. AMAZON SELLER SERVIC", + "city": "Siddhapudur", + "stateOrRegion": "Tamil Nadu", + "postalCode": "641044", + "countryCode": "IN", + "phone": "206-266-8000" + } + }, + "taxInfo": { + "taxType": "GST", + "taxRegistrationNumber": "098522PCA6346DTEDD" + } + }, + "deliveryWindow": "2019-05-23T10:00:00Z--2019-05-30T10:00:00Z", + "items": [ + { + "itemSequenceNumber": "1", + "amazonProductIdentifier": "ABC123434", + "vendorProductIdentifier": "028877454078", + "orderedQuantity": { + "amount": 2, + "unitOfMeasure": "Cases", + "unitSize": 10 + }, + "isBackOrderAllowed": true, + "netCost": { + "amount": "1800", + "currencyCode": "INR" + }, + "listPrice": { + "amount": "2000", + "currencyCode": "INR" + } + } + ] + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py index b59a9240e7ea..ee69ee4ccd30 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict + from unittest.mock import patch import pendulum @@ -55,7 +55,7 @@ def test_augmented_data_incorrect_period(self, report_init_kwargs): report_options = {"reportPeriod": "DAYS123"} with pytest.raises(Exception) as e: stream._augmented_data(report_options) - assert e.value.args[0] == [{'message': 'This reportPeriod is not implemented.'}] + assert e.value.args[0] == [{"message": "This reportPeriod is not implemented."}] @pytest.mark.parametrize( ("report_options", "report_option_dates"), @@ -90,9 +90,10 @@ def test_report_data(self, report_init_kwargs, stream_slice): stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) expected_data = {"reportType": stream.name, "marketplaceIds": [report_init_kwargs["marketplace_id"]]} expected_data.update(stream_slice) - assert stream._report_data( - sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_slice=stream_slice - ) == expected_data + assert ( + stream._report_data(sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_slice=stream_slice) + == expected_data + ) @pytest.mark.parametrize( ("current_stream_state", "latest_record", "expected_date"), @@ -126,13 +127,13 @@ def test_get_updated_state(self, report_init_kwargs, current_stream_state, lates [{"dataStartTime": "2023-09-06T00:00:00Z", "dataEndTime": "2023-09-06T23:59:59Z"}], ), ( - "2023-05-01T00:00:00Z", - "2023-09-07T00:00:00Z", + "2022-05-01T00:00:00Z", + "2023-09-05T00:00:00Z", None, 0, [ - {"dataStartTime": "2023-05-01T00:00:00Z", "dataEndTime": "2023-07-29T23:59:59Z"}, - {"dataStartTime": "2023-07-30T00:00:00Z", "dataEndTime": "2023-09-07T00:00:00Z"}, + {"dataStartTime": "2022-05-01T00:00:00Z", "dataEndTime": "2023-04-30T23:59:59Z"}, + {"dataStartTime": "2023-05-01T00:00:00Z", "dataEndTime": "2023-09-05T00:00:00Z"}, ], ), ), @@ -143,6 +144,7 @@ def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_st stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) stream.fixed_period_in_days = fixed_period_in_days with patch("pendulum.now", return_value=pendulum.parse("2023-09-09T00:00:00Z")): - assert stream.stream_slices( - sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_state=stream_state - ) == expected_slices + assert ( + list(stream.stream_slices(sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_state=stream_state)) + == expected_slices + ) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index e84ee83dbd02..1d3ad8b7cb4f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from unittest import mock import pendulum diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index 7ff6c7fb1958..1421de0b1f3d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -51,7 +51,7 @@ class TestMigrateReportOptions: ("input_config", "expected_report_options_list"), ( ( - {"report_options": "{\"GET_REPORT\": {\"reportPeriod\": \"WEEK\"}}"}, + {"report_options": '{"GET_REPORT": {"reportPeriod": "WEEK"}}'}, [{"stream_name": "GET_REPORT", "options_list": [{"option_name": "reportPeriod", "option_value": "WEEK"}]}], ), ({"report_options": None}, []), diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py index 95549b861721..a57deb47b748 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from unittest import mock import pytest @@ -143,20 +144,6 @@ def test_request_params(self, order_items_stream, next_page_token, expected_para stream = order_items_stream() assert stream.request_params(stream_state={}, next_page_token=next_page_token) == expected_params - @pytest.mark.parametrize( - ("current_stream_state", "cached_state", "expected_date"), - ( - ({"LastUpdateDate": "2022-10-03T00:00:00Z"}, {"LastUpdateDate": "2022-10-04T00:00:00Z"}, "2022-10-04T00:00:00Z"), - ({"LastUpdateDate": "2022-10-04T00:00:00Z"}, {"LastUpdateDate": "2022-10-03T00:00:00Z"}, "2022-10-04T00:00:00Z"), - ({}, {"LastUpdateDate": "2022-10-03T00:00:00Z"}, "2022-10-03T00:00:00Z"), - ), - ) - def test_get_updated_state(self, order_items_stream, current_stream_state, cached_state, expected_date): - stream = order_items_stream() - stream.cached_state = cached_state - expected_state = {stream.cursor_field: expected_date} - assert stream.get_updated_state(current_stream_state, {}) == expected_state - @pytest.mark.parametrize( ("response_headers", "expected_backoff_time"), (({"x-amzn-RateLimit-Limit": "2"}, 0.5), ({}, 10)), diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index 8e33fafd47cf..0bdf83116fbc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest from airbyte_cdk.models import SyncMode from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index 35feab08be6b..c9f86dc90d29 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -2,13 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import logging from unittest.mock import patch import pytest from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.streams import VendorSalesReports +from source_amazon_seller_partner.streams import VendorOrders from source_amazon_seller_partner.utils import AmazonConfigException logger = logging.getLogger("airbyte") @@ -23,6 +24,30 @@ def connector_config_with_report_options(): "lwa_app_id": "amzn1.application-oa2-client.abc123", "lwa_client_secret": "abc123", "aws_environment": "SANDBOX", + "account_type": "Seller", + "region": "US", + "report_options_list": [ + { + "stream_name": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + "options_list": [ + {"option_name": "some_name_1", "option_value": "some_value_1"}, + {"option_name": "some_name_2", "option_value": "some_value_2"}, + ], + }, + ], + } + + +@pytest.fixture +def connector_vendor_config_with_report_options(): + return { + "replication_start_date": "2017-01-25T00:00:00Z", + "replication_end_date": "2017-02-25T00:00:00Z", + "refresh_token": "Atzr|IwEBIP-abc123", + "lwa_app_id": "amzn1.application-oa2-client.abc123", + "lwa_client_secret": "abc123", + "aws_environment": "SANDBOX", + "account_type": "Vendor", "region": "US", "report_options_list": [ { @@ -47,7 +72,7 @@ def connector_config_without_start_date(): } -def test_check_connection_with_vendor_report(mocker, requests_mock, connector_config_with_report_options): +def test_check_connection_with_vendor_report(mocker, requests_mock, connector_vendor_config_with_report_options): mocker.patch("time.sleep", lambda x: None) requests_mock.register_uri( "POST", @@ -55,15 +80,9 @@ def test_check_connection_with_vendor_report(mocker, requests_mock, connector_co status_code=200, json={"access_token": "access_token", "expires_in": "3600"}, ) - requests_mock.register_uri( - "GET", - "https://sandbox.sellingpartnerapi-na.amazon.com/orders/v0/orders", - status_code=403, - json={"error": "forbidden"}, - ) - with patch.object(VendorSalesReports, "read_records", return_value=iter([{"some_key": "some_value"}])): - assert SourceAmazonSellerPartner().check_connection(logger, connector_config_with_report_options) == (True, None) + with patch.object(VendorOrders, "read_records", return_value=iter([{"some_key": "some_value"}])): + assert SourceAmazonSellerPartner().check_connection(logger, connector_vendor_config_with_report_options) == (True, None) def test_check_connection_with_orders_stop_iteration(requests_mock, connector_config_with_report_options): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index 52c88d888eea..a6e3661c5846 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict + from unittest.mock import patch import pendulum @@ -10,7 +10,12 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.utils import AirbyteTracedException -from source_amazon_seller_partner.streams import IncrementalReportsAmazonSPStream, ReportsAmazonSPStream, VendorDirectFulfillmentShipping +from source_amazon_seller_partner.streams import ( + IncrementalReportsAmazonSPStream, + ReportProcessingStatus, + ReportsAmazonSPStream, + VendorDirectFulfillmentShipping, +) class SomeReportStream(ReportsAmazonSPStream): @@ -54,17 +59,20 @@ def test_report_data(self, report_init_kwargs): [{"dataStartTime": "2022-09-01T00:00:00Z", "dataEndTime": "2022-10-01T00:00:00Z"}], ), ( - "2022-09-01T00:00:00Z", - "2023-01-01T00:00:00Z", + "2021-05-01T00:00:00Z", + "2022-09-05T00:00:00Z", [ - {"dataStartTime": "2022-09-01T00:00:00Z", "dataEndTime": "2022-11-29T23:59:59Z"}, - {"dataStartTime": "2022-11-30T00:00:00Z", "dataEndTime": "2023-01-01T00:00:00Z"}, + {"dataStartTime": "2021-05-01T00:00:00Z", "dataEndTime": "2022-04-30T23:59:59Z"}, + {"dataStartTime": "2022-05-01T00:00:00Z", "dataEndTime": "2022-09-05T00:00:00Z"}, ], ), ( - "2022-10-01T00:00:00Z", + "2021-10-01T00:00:00Z", None, - [{"dataStartTime": "2022-10-03T00:00:00Z", "dataEndTime": "2022-12-31T23:59:59Z"}], + [ + {"dataStartTime": "2021-10-01T00:00:00Z", "dataEndTime": "2022-09-30T23:59:59Z"}, + {"dataStartTime": "2022-10-01T00:00:00Z", "dataEndTime": "2023-01-01T00:00:00Z"}, + ], ), ( "2022-11-01T00:00:00Z", @@ -110,17 +118,28 @@ def test_read_records_retrieve_fatal(self, report_init_kwargs, mocker, requests_ status_code=201, json={"reportId": report_id}, ) + document_id = "some_document_id" requests_mock.register_uri( "GET", f"https://test.url/reports/2021-06-30/reports/{report_id}", status_code=200, - json={"processingStatus": "FATAL", "dataEndTime": "2022-10-03T00:00:00Z"}, + json={"processingStatus": ReportProcessingStatus.FATAL, "dataEndTime": "2022-10-03T00:00:00Z", "reportDocumentId": document_id}, ) stream = SomeReportStream(**report_init_kwargs) + stream_start = "2022-09-03T00:00:00Z" + stream_end = "2022-10-03T00:00:00Z" with pytest.raises(AirbyteTracedException) as e: - list(stream.read_records(sync_mode=SyncMode.full_refresh)) - assert e.value.message == "The report for stream 'GET_TEST_REPORT' was not created - skip reading" + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_slice={"dataStartTime": stream_start, "dataEndTime": stream_end}, + ) + ) + assert e.value.internal_message == ( + f"Failed to retrieve the report 'GET_TEST_REPORT' for period {stream_start}-{stream_end}. " + "This will be read during the next sync. Error: Failed to retrieve the report result document." + ) def test_read_records_retrieve_cancelled(self, report_init_kwargs, mocker, requests_mock, caplog): mocker.patch("time.sleep", lambda x: None) @@ -142,12 +161,12 @@ def test_read_records_retrieve_cancelled(self, report_init_kwargs, mocker, reque "GET", f"https://test.url/reports/2021-06-30/reports/{report_id}", status_code=200, - json={"processingStatus": "CANCELLED", "dataEndTime": "2022-10-03T00:00:00Z"}, + json={"processingStatus": ReportProcessingStatus.CANCELLED, "dataEndTime": "2022-10-03T00:00:00Z"}, ) stream = SomeReportStream(**report_init_kwargs) list(stream.read_records(sync_mode=SyncMode.full_refresh)) - assert "The report for stream 'GET_TEST_REPORT' was cancelled or there is no data to return" in caplog.messages[-1] + assert "The report for stream 'GET_TEST_REPORT' was cancelled or there is no data" in caplog.messages[-1] def test_read_records_retrieve_done(self, report_init_kwargs, mocker, requests_mock): mocker.patch("time.sleep", lambda x: None) @@ -170,7 +189,11 @@ def test_read_records_retrieve_done(self, report_init_kwargs, mocker, requests_m "GET", f"https://test.url/reports/2021-06-30/reports/{report_id}", status_code=200, - json={"processingStatus": "DONE", "dataEndTime": "2022-10-03T00:00:00Z", "reportDocumentId": document_id}, + json={ + "processingStatus": ReportProcessingStatus.DONE, + "dataEndTime": "2022-10-03T00:00:00Z", + "reportDocumentId": document_id, + }, ) requests_mock.register_uri( "GET", @@ -184,24 +207,97 @@ def test_read_records_retrieve_done(self, report_init_kwargs, mocker, requests_m records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) assert records[0] == {"some_key": "some_value", "dataEndTime": "2022-10-03"} + def test_read_records_retrieve_forbidden(self, report_init_kwargs, mocker, requests_mock, caplog): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + + report_id = "some_report_id" + requests_mock.register_uri( + "POST", + "https://test.url/reports/2021-06-30/reports", + status_code=403, + json={"reportId": report_id}, + reason="Forbidden", + ) + + stream = SomeReportStream(**report_init_kwargs) + assert list(stream.read_records(sync_mode=SyncMode.full_refresh)) == [] + assert ( + "The endpoint https://test.url/reports/2021-06-30/reports returned 403: Forbidden. " + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate." + ) in caplog.messages[-1] + -class TestVendorDirectFulfillmentShipping: +class TestVendorFulfillment: @pytest.mark.parametrize( - ("start_date", "end_date", "expected_params"), + ("start_date", "end_date", "stream_state", "expected_slices"), ( - ("2022-09-01T00:00:00Z", None, {"createdAfter": "2022-09-01T00:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}), - ("2022-08-01T00:00:00Z", None, {"createdAfter": "2022-08-28T23:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}), ( "2022-09-01T00:00:00Z", - "2022-09-05T00:00:00Z", - {"createdAfter": "2022-09-01T00:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}, + None, + None, + [{"createdAfter": "2022-09-01T00:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}], + ), + ( + "2022-08-01T00:00:00Z", + "2022-08-16T00:00:00Z", + None, + [ + {"createdAfter": "2022-08-01T00:00:00Z", "createdBefore": "2022-08-08T00:00:00Z"}, + {"createdAfter": "2022-08-08T00:00:00Z", "createdBefore": "2022-08-15T00:00:00Z"}, + {"createdAfter": "2022-08-15T00:00:00Z", "createdBefore": "2022-08-16T00:00:00Z"}, + ], + ), + ( + "2022-08-01T00:00:00Z", + "2022-08-05T00:00:00Z", + None, + [{"createdAfter": "2022-08-01T00:00:00Z", "createdBefore": "2022-08-05T00:00:00Z"}], + ), + ( + "2022-08-01T00:00:00Z", + "2022-08-11T00:00:00Z", + {"createdBefore": "2022-08-05T00:00:00Z"}, + [{"createdAfter": "2022-08-05T00:00:00Z", "createdBefore": "2022-08-11T00:00:00Z"}], ), + ("2022-08-01T00:00:00Z", "2022-08-05T00:00:00Z", {"createdBefore": "2022-08-06T00:00:00Z"}, []), ), ) - def test_request_params(self, report_init_kwargs, start_date, end_date, expected_params): + def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_state, expected_slices): report_init_kwargs["replication_start_date"] = start_date report_init_kwargs["replication_end_date"] = end_date stream = VendorDirectFulfillmentShipping(**report_init_kwargs) with patch("pendulum.now", return_value=pendulum.parse("2022-09-05T00:00:00Z")): - assert stream.request_params(stream_state={}) == expected_params + assert list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state)) == expected_slices + + @pytest.mark.parametrize( + ("stream_slice", "next_page_token", "expected_params"), + ( + ( + {"createdAfter": "2022-08-05T00:00:00Z", "createdBefore": "2022-08-11T00:00:00Z"}, + None, + {"createdAfter": "2022-08-05T00:00:00Z", "createdBefore": "2022-08-11T00:00:00Z"}, + ), + ( + {"createdAfter": "2022-08-05T00:00:00Z", "createdBefore": "2022-08-11T00:00:00Z"}, + {"nextToken": "123123123"}, + { + "createdAfter": "2022-08-05T00:00:00Z", + "createdBefore": "2022-08-11T00:00:00Z", + "nextToken": "123123123", + }, + ), + (None, {"nextToken": "123123123"}, {"nextToken": "123123123"}), + (None, None, {}), + ), + ) + def test_request_params(self, report_init_kwargs, stream_slice, next_page_token, expected_params): + stream = VendorDirectFulfillmentShipping(**report_init_kwargs) + assert stream.request_params(stream_state={}, stream_slice=stream_slice, next_page_token=next_page_token) == expected_params diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py index 4fe7fd4727d8..0acc47ef9079 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py @@ -2,8 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest -from source_amazon_seller_partner.streams import SellerFeedbackReports +from source_amazon_seller_partner.streams import ( + FlatFileSettlementV2Reports, + LedgerDetailedViewReports, + MerchantListingsFypReport, + MerchantListingsReports, + SellerFeedbackReports, +) def reports_stream(marketplace_id): @@ -53,3 +60,81 @@ def test_transform_seller_feedback(marketplace_id, input_data, expected_data): transformer.transform(input_data, schema) assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"item-name": "GiftBox", "open-date": "2022-07-11 01:34:18 PDT", "dataEndTime": "2022-07-31"}, + {"item-name": "GiftBox", "open-date": "2022-07-11T01:34:18-07:00", "dataEndTime": "2022-07-31"}, + ), + ( + {"item-name": "GiftBox", "open-date": "", "dataEndTime": "2022-07-31"}, + {"item-name": "GiftBox", "open-date": "", "dataEndTime": "2022-07-31"}, + ), + ), +) +def test_transform_merchant_reports(report_init_kwargs, input_data, expected_data): + stream = MerchantListingsReports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "dataEndTime": "2022-07-31"}, + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-29", "dataEndTime": "2022-07-31"}, + ), + ( + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "", "dataEndTime": "2022-07-31"}, + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "", "dataEndTime": "2022-07-31"}, + ), + ), +) +def test_transform_merchant_fyp_reports(report_init_kwargs, input_data, expected_data): + stream = MerchantListingsFypReport(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ({"Date": "07/29/2022", "dataEndTime": "2022-07-31"}, {"Date": "2022-07-29", "dataEndTime": "2022-07-31"}), + ({"Date": "7/29/2022", "dataEndTime": "2022-07-31"}, {"Date": "2022-07-29", "dataEndTime": "2022-07-31"}), + ({"Date": "07/2022", "dataEndTime": "2022-07-31"}, {"Date": "2022-07-01", "dataEndTime": "2022-07-31"}), + ({"Date": "7/2022", "dataEndTime": "2022-07-31"}, {"Date": "2022-07-01", "dataEndTime": "2022-07-31"}), + ({"Date": "", "dataEndTime": "2022-07-31"}, {"Date": "", "dataEndTime": "2022-07-31"}), + ), +) +def test_transform_ledger_reports(report_init_kwargs, input_data, expected_data): + stream = LedgerDetailedViewReports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"posted-date": "2023-11-09T18:44:35+00:00", "dataEndTime": "2022-07-31"}, + {"posted-date": "2023-11-09T18:44:35+00:00", "dataEndTime": "2022-07-31"}, + ), + ({"posted-date": "", "dataEndTime": "2022-07-31"}, {"posted-date": None, "dataEndTime": "2022-07-31"}), + ), +) +def test_transform_settlement_reports(report_init_kwargs, input_data, expected_data): + stream = FlatFileSettlementV2Reports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data diff --git a/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile b/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile index f8020036fba8..ef097b4cbff5 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile @@ -34,5 +34,5 @@ COPY source_amazon_sqs ./source_amazon_sqs ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-amazon-sqs diff --git a/airbyte-integrations/connectors/source-amazon-sqs/main.py b/airbyte-integrations/connectors/source-amazon-sqs/main.py index bbf86753b1de..3e218a144f8f 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_sqs import SourceAmazonSqs +from source_amazon_sqs.run import run if __name__ == "__main__": - source = SourceAmazonSqs() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml index a62580bedfa1..6b6bd34f1cfb 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml @@ -5,13 +5,17 @@ data: connectorSubtype: api connectorType: source definitionId: 983fd355-6bf3-4709-91b5-37afa391eeb6 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-amazon-sqs documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-sqs githubIssueLabel: source-amazon-sqs icon: awssqs.svg license: MIT name: Amazon SQS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-sqs registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index 3ca84d0f1041..e39e0d894b21 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "moto[sqs, iam]"] setup( + entry_points={ + "console_scripts": [ + "source-amazon-sqs=source_amazon_sqs.run:run", + ], + }, name="source_amazon_sqs", description="Source implementation for Amazon Sqs.", author="Alasdair Brown", author_email="airbyte@alasdairb.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py new file mode 100644 index 000000000000..428858388b63 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_sqs import SourceAmazonSqs + + +def run(): + source = SourceAmazonSqs() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json index 0bb7d64eded0..4c71ef50c0b8 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json @@ -21,31 +21,39 @@ "description": "AWS Region of the SQS Queue", "type": "string", "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 1 }, diff --git a/airbyte-integrations/connectors/source-amplitude/README.md b/airbyte-integrations/connectors/source-amplitude/README.md index 6ace21da93d7..6d9f9f816a68 100644 --- a/airbyte-integrations/connectors/source-amplitude/README.md +++ b/airbyte-integrations/connectors/source-amplitude/README.md @@ -1,86 +1,55 @@ -# Amplitude Source +# Amplitude source connector -This is the repository for the Amplitude configuration based source connector. + +This is the repository for the Amplitude source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/amplitude). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/amplitude) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amplitude/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source amplitude test creds` -and place them into `secrets/config.json`. - -### Locally running the connector docker image - - +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Installing the connector +From this connector directory, run: ```bash -airbyte-ci connectors --name=source-amplitude build +poetry install --with dev ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-amplitude:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/amplitude) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amplitude/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +### Locally running the connector +``` +poetry run source-amplitude spec +poetry run source-amplitude check --config secrets/config.json +poetry run source-amplitude discover --config secrets/config.json +poetry run source-amplitude read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-amplitude:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests ``` -Please use this as an example. This is not optimized. -2. Build your image: +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-amplitude:dev . -# Running the spec command against your patched connector -docker run airbyte/source-amplitude:dev spec +airbyte-ci connectors --name=source-amplitude build ``` -#### Run + +An image will be available on your host with the tag `airbyte/source-amplitude:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-amplitude:dev spec @@ -89,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amplitude:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amplitude:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-amplitude test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amplitude test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amplitude.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/amplitude.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amplitude/main.py b/airbyte-integrations/connectors/source-amplitude/main.py index 7bf14f9904f8..14500e9c73e6 100644 --- a/airbyte-integrations/connectors/source-amplitude/main.py +++ b/airbyte-integrations/connectors/source-amplitude/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amplitude import SourceAmplitude +from source_amplitude.run import run if __name__ == "__main__": - source = SourceAmplitude() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml index fac740931651..f277c1ad7774 100644 --- a/airbyte-integrations/connectors/source-amplitude/metadata.yaml +++ b/airbyte-integrations/connectors/source-amplitude/metadata.yaml @@ -11,13 +11,17 @@ data: connectorSubtype: api connectorType: source definitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 - dockerImageTag: 0.3.6 + dockerImageTag: 0.3.7 dockerRepository: airbyte/source-amplitude documentationUrl: https://docs.airbyte.com/integrations/sources/amplitude githubIssueLabel: source-amplitude icon: amplitude.svg license: MIT name: Amplitude + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amplitude registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-amplitude/poetry.lock b/airbyte-integrations/connectors/source-amplitude/poetry.lock new file mode 100644 index 000000000000..647f6526bac0 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/poetry.lock @@ -0,0 +1,1083 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.52.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.52.0.tar.gz", hash = "sha256:760b5bb279e5b06455bc33c9744dd9facbc0b203ccc4ac48e1e2877807e3c845"}, + {file = "airbyte_cdk-0.52.0-py3-none-any.whl", hash = "sha256:bf7c82b2a7ec3cc4ddedd17cd6cd6e2385991af965729f23ffbdb0515388a8e2"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "71149e8c9b376cbd538e039f53fb1be4ceb6562766a6221a6a95d15a2dab08e3" diff --git a/airbyte-integrations/connectors/source-amplitude/pyproject.toml b/airbyte-integrations/connectors/source-amplitude/pyproject.toml new file mode 100644 index 000000000000..e610b3f4642f --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.7" +name = "source-amplitude" +description = "Source implementation for Amplitude." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/amplitude" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_amplitude" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.52.0" + +[tool.poetry.scripts] +source-amplitude = "source_amplitude.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.2" diff --git a/airbyte-integrations/connectors/source-amplitude/setup.py b/airbyte-integrations/connectors/source-amplitude/setup.py deleted file mode 100644 index 29cccb75e13f..000000000000 --- a/airbyte-integrations/connectors/source-amplitude/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.2", - "pytest-mock~=3.6.1", -] - -setup( - name="source_amplitude", - description="Source implementation for Amplitude.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py new file mode 100644 index 000000000000..3649e5d9b311 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amplitude import SourceAmplitude + + +def run(): + source = SourceAmplitude() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile index c51f4d752c98..6d3b4a5de1b7 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile +++ b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile @@ -34,5 +34,5 @@ COPY source_apify_dataset ./source_apify_dataset ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.1.0 +LABEL io.airbyte.version=2.1.1 LABEL io.airbyte.name=airbyte/source-apify-dataset diff --git a/airbyte-integrations/connectors/source-apify-dataset/main.py b/airbyte-integrations/connectors/source-apify-dataset/main.py index d4b0ed78bc71..4ef9d72f02b4 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/main.py +++ b/airbyte-integrations/connectors/source-apify-dataset/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_apify_dataset import SourceApifyDataset +from source_apify_dataset.run import run if __name__ == "__main__": - source = SourceApifyDataset() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index 6fc16080b973..f87bf805db81 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.apify.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-apify-dataset registries: oss: enabled: true @@ -10,7 +14,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 - dockerImageTag: 2.1.0 + dockerImageTag: 2.1.1 dockerRepository: airbyte/source-apify-dataset githubIssueLabel: source-apify-dataset icon: apify.svg @@ -29,5 +33,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-apify-dataset/setup.py b/airbyte-integrations/connectors/source-apify-dataset/setup.py index 51b088de172c..994bf4b0f951 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/setup.py +++ b/airbyte-integrations/connectors/source-apify-dataset/setup.py @@ -16,8 +16,25 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-apify-dataset=source_apify_dataset.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py new file mode 100644 index 000000000000..c7488d02985e --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceApifyDataset + + +def run(): + source = SourceApifyDataset() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appfollow/main.py b/airbyte-integrations/connectors/source-appfollow/main.py index a4cd4bd63690..79dea68512c5 100644 --- a/airbyte-integrations/connectors/source-appfollow/main.py +++ b/airbyte-integrations/connectors/source-appfollow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appfollow import SourceAppfollow +from source_appfollow.run import run if __name__ == "__main__": - source = SourceAppfollow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appfollow/metadata.yaml b/airbyte-integrations/connectors/source-appfollow/metadata.yaml index e65309bd6e97..ad5e49e07ca4 100644 --- a/airbyte-integrations/connectors/source-appfollow/metadata.yaml +++ b/airbyte-integrations/connectors/source-appfollow/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.appfollow.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-appfollow registries: oss: enabled: true @@ -20,7 +24,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/appfollow tags: - - language:lowcode + - language:low-code releases: breakingChanges: 1.0.0: diff --git a/airbyte-integrations/connectors/source-appfollow/setup.py b/airbyte-integrations/connectors/source-appfollow/setup.py index c808dd5682e9..73e20b4ae8b1 100644 --- a/airbyte-integrations/connectors/source-appfollow/setup.py +++ b/airbyte-integrations/connectors/source-appfollow/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "requests_mock~=1.9"] setup( + entry_points={ + "console_scripts": [ + "source-appfollow=source_appfollow.run:run", + ], + }, name="source_appfollow", description="Source implementation for Appfollow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py b/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py new file mode 100644 index 000000000000..019ce80f3ba5 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appfollow import SourceAppfollow + + +def run(): + source = SourceAppfollow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apple-search-ads/main.py b/airbyte-integrations/connectors/source-apple-search-ads/main.py index 8998f6bd4388..df8b0f70775c 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/main.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_apple_search_ads import SourceAppleSearchAds +from source_apple_search_ads.run import run if __name__ == "__main__": - source = SourceAppleSearchAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml index 1406f248390e..bbbaecfeef94 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml @@ -8,6 +8,10 @@ data: icon: apple.svg license: MIT name: Apple Search Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-apple-search-ads registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-apple-search-ads/setup.py b/airbyte-integrations/connectors/source-apple-search-ads/setup.py index a66de217e0fd..c70cfcb0168b 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/setup.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-apple-search-ads=source_apple_search_ads.run:run", + ], + }, name="source_apple_search_ads", description="Source implementation for Apple Search Ads.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py new file mode 100644 index 000000000000..dc2def6147ad --- /dev/null +++ b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_apple_search_ads import SourceAppleSearchAds + + +def run(): + source = SourceAppleSearchAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appsflyer/main.py b/airbyte-integrations/connectors/source-appsflyer/main.py index 1dee48f3f0ab..ebf2655b2ec4 100644 --- a/airbyte-integrations/connectors/source-appsflyer/main.py +++ b/airbyte-integrations/connectors/source-appsflyer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appsflyer import SourceAppsflyer +from source_appsflyer.run import run if __name__ == "__main__": - source = SourceAppsflyer() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml index 4f3c263c6cbb..21d47e8efcbd 100644 --- a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml @@ -8,6 +8,10 @@ data: icon: appsflyer.svg license: MIT name: AppsFlyer + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-appsflyer registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-appsflyer/setup.py b/airbyte-integrations/connectors/source-appsflyer/setup.py index ac74ebcabfc9..613efc02fdda 100644 --- a/airbyte-integrations/connectors/source-appsflyer/setup.py +++ b/airbyte-integrations/connectors/source-appsflyer/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-appsflyer=source_appsflyer.run:run", + ], + }, name="source_appsflyer", description="Source implementation for Appsflyer.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py new file mode 100644 index 000000000000..40c1e2feb038 --- /dev/null +++ b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appsflyer import SourceAppsflyer + + +def run(): + source = SourceAppsflyer() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appstore-singer/main.py b/airbyte-integrations/connectors/source-appstore-singer/main.py index 5e0f99d007be..34e585afeeed 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/main.py +++ b/airbyte-integrations/connectors/source-appstore-singer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appstore_singer import SourceAppstoreSinger +from source_appstore_singer.run import run if __name__ == "__main__": - source = SourceAppstoreSinger() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml index cb0d55be57b3..c0aad524053e 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml @@ -8,11 +8,16 @@ data: icon: appstore.svg license: MIT name: Appstore - registries: + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-appstore-singer + registries: # Removed from registries due to LEGACY STATE cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/appstore tags: diff --git a/airbyte-integrations/connectors/source-appstore-singer/setup.py b/airbyte-integrations/connectors/source-appstore-singer/setup.py index b6ecccc99a03..ecf51e5403a8 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/setup.py +++ b/airbyte-integrations/connectors/source-appstore-singer/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-appstore-singer=source_appstore_singer.run:run", + ], + }, name="source_appstore_singer", description="Source implementation for Appstore, built on the Singer tap implementation.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py b/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py new file mode 100644 index 000000000000..ef9f845e8d81 --- /dev/null +++ b/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appstore_singer import SourceAppstoreSinger + + +def run(): + source = SourceAppstoreSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-asana/main.py b/airbyte-integrations/connectors/source-asana/main.py index ac64981a1e3e..5fde4e3c72d6 100644 --- a/airbyte-integrations/connectors/source-asana/main.py +++ b/airbyte-integrations/connectors/source-asana/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_asana import SourceAsana +from source_asana.run import run if __name__ == "__main__": - source = SourceAsana() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index d3a4dd21bda3..ae99033996f8 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -15,6 +15,10 @@ data: icon: asana.svg license: MIT name: Asana + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-asana registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-asana/setup.py b/airbyte-integrations/connectors/source-asana/setup.py index dda6ee977db1..08e8edc8363f 100644 --- a/airbyte-integrations/connectors/source-asana/setup.py +++ b/airbyte-integrations/connectors/source-asana/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock~=1.9.3"] setup( + entry_points={ + "console_scripts": [ + "source-asana=source_asana.run:run", + ], + }, name="source_asana", description="Source implementation for Asana.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-asana/source_asana/run.py b/airbyte-integrations/connectors/source-asana/source_asana/run.py new file mode 100644 index 000000000000..cf9dbf1ffa48 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_asana import SourceAsana + + +def run(): + source = SourceAsana() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ashby/main.py b/airbyte-integrations/connectors/source-ashby/main.py index a19039ecb615..96b5bfe556e3 100644 --- a/airbyte-integrations/connectors/source-ashby/main.py +++ b/airbyte-integrations/connectors/source-ashby/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ashby import SourceAshby +from source_ashby.run import run if __name__ == "__main__": - source = SourceAshby() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ashby/metadata.yaml b/airbyte-integrations/connectors/source-ashby/metadata.yaml index 510c05376240..94a4355863a3 100644 --- a/airbyte-integrations/connectors/source-ashby/metadata.yaml +++ b/airbyte-integrations/connectors/source-ashby/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ashby.svg license: MIT name: Ashby + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ashby registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-ashby/setup.py b/airbyte-integrations/connectors/source-ashby/setup.py index d4fd781c5f1e..d506c08b0965 100644 --- a/airbyte-integrations/connectors/source-ashby/setup.py +++ b/airbyte-integrations/connectors/source-ashby/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ashby=source_ashby.run:run", + ], + }, name="source_ashby", description="Source implementation for Ashby.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/run.py b/airbyte-integrations/connectors/source-ashby/source_ashby/run.py new file mode 100644 index 000000000000..2330c6f910b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ashby import SourceAshby + + +def run(): + source = SourceAshby() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-auth0/main.py b/airbyte-integrations/connectors/source-auth0/main.py index 3904766c4ec2..f29790fbabd7 100644 --- a/airbyte-integrations/connectors/source-auth0/main.py +++ b/airbyte-integrations/connectors/source-auth0/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_auth0 import SourceAuth0 +from source_auth0.run import run if __name__ == "__main__": - source = SourceAuth0() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-auth0/metadata.yaml b/airbyte-integrations/connectors/source-auth0/metadata.yaml index 5eb0080a09cb..21016c9467fd 100644 --- a/airbyte-integrations/connectors/source-auth0/metadata.yaml +++ b/airbyte-integrations/connectors/source-auth0/metadata.yaml @@ -17,6 +17,10 @@ data: icon: auth0.svg license: MIT name: Auth0 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-auth0 registries: cloud: enabled: true @@ -26,5 +30,5 @@ data: releaseStage: alpha supportLevel: community tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-auth0/setup.py b/airbyte-integrations/connectors/source-auth0/setup.py index 29b448c8c0de..ed772442f1e0 100644 --- a/airbyte-integrations/connectors/source-auth0/setup.py +++ b/airbyte-integrations/connectors/source-auth0/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-auth0=source_auth0.run:run", + ], + }, name="source_auth0", description="Source implementation for Auth0.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/run.py b/airbyte-integrations/connectors/source-auth0/source_auth0/run.py new file mode 100644 index 000000000000..bae97c9f5cb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_auth0 import SourceAuth0 + + +def run(): + source = SourceAuth0() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/main.py b/airbyte-integrations/connectors/source-aws-cloudtrail/main.py index 7ae051b2be22..f2324dfe8812 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/main.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aws_cloudtrail import SourceAwsCloudtrail +from source_aws_cloudtrail.run import run if __name__ == "__main__": - source = SourceAwsCloudtrail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml index 3b3240c49425..f4d483e04c68 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml @@ -12,6 +12,10 @@ data: icon: awscloudtrail.svg license: MIT name: AWS CloudTrail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aws-cloudtrail registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py index bda35b60aa1e..3bf29110ccb4 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aws-cloudtrail=source_aws_cloudtrail.run:run", + ], + }, name="source_aws_cloudtrail", description="Source implementation for Aws Cloudtrail.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py b/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py new file mode 100644 index 000000000000..576aae749bc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aws_cloudtrail import SourceAwsCloudtrail + + +def run(): + source = SourceAwsCloudtrail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml index e15f5f60be54..71d40148b88f 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml @@ -86,6 +86,8 @@ acceptance_tests: status: succeed - config_path: secrets/jsonl_newlines_config.json status: succeed + - config_path: secrets/unstructured_config.json + status: succeed discovery: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl index 29dcaa565ccc..8ac3010fd350 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl @@ -1,2 +1,2 @@ -{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "Testdoc.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc.pdf"}, "emitted_at": 1698666216334} -{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "This is a test", "document_key": "Testdoc_OCR.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf"}, "emitted_at": 1698666218048} \ No newline at end of file +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "Testdoc.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698666216334} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "This is a test", "document_key": "Testdoc_OCR.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698666218048} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json index cf4c66d2d9ce..81b04111ee81 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json @@ -58,7 +58,7 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", "type": "string", "airbyte_hidden": true }, @@ -288,12 +288,46 @@ "const": "unstructured", "type": "string" }, - "skip_unprocessable_file_types": { + "skip_unprocessable_files": { "type": "boolean", "default": true, - "title": "Skip Unprocessable File Types", - "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] } }, "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/main.py b/airbyte-integrations/connectors/source-azure-blob-storage/main.py index b3361a6556d7..5e798013d9e4 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/main.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/main.py @@ -2,32 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys -import traceback -from datetime import datetime - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type -from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader +from source_azure_blob_storage.run import run if __name__ == "__main__": - args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(args) - try: - source = SourceAzureBlobStorage(SourceAzureBlobStorageStreamReader(), Config, catalog_path) - except Exception: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - else: - launch(source, args) + run() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 4cb4fc681f22..5652593c7331 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -7,13 +7,17 @@ data: connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.2.5 + dockerImageTag: 0.3.4 dockerRepository: airbyte/source-azure-blob-storage documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage icon: azureblobstorage.svg license: MIT name: Azure Blob Storage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-azure-blob-storage registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index 3af748ec694f..97e8173bf2b2 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.55.5", + "airbyte-cdk[file-based]>=0.61.0", "smart_open[azure]", "pytz", ] @@ -14,13 +14,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2"] setup( + entry_points={ + "console_scripts": [ + "source-azure-blob-storage=source_azure_blob_storage.run:run", + ], + }, name="source_azure_blob_storage", description="Source implementation for Azure Blob Storage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py index da222c774fbc..9955603ba74b 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py @@ -2,8 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Optional +from typing import Any, Dict, Optional +import dpath.util from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from pydantic import AnyUrl, Field @@ -44,3 +45,16 @@ def documentation_url(cls) -> AnyUrl: examples=["blob.core.windows.net"], order=11, ) + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py new file mode 100644 index 000000000000..a671d836526a --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys +import traceback +from datetime import datetime + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + try: + source = SourceAzureBlobStorage( + SourceAzureBlobStorageStreamReader(), + Config, + SourceAzureBlobStorage.read_catalog(catalog_path) if catalog_path else None, + SourceAzureBlobStorage.read_config(config_path) if catalog_path else None, + SourceAzureBlobStorage.read_state(state_path) if catalog_path else None, + ) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + else: + launch(source, args) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py index 419119bb3ef8..792dcdfe2221 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py @@ -11,14 +11,15 @@ class SourceAzureBlobStorage(FileBasedSource): - def read_config(self, config_path: str) -> Mapping[str, Any]: + @classmethod + def read_config(cls, config_path: str) -> Mapping[str, Any]: """ Used to override the default read_config so that when the new file-based Azure Blob Storage connector processes a config in the legacy format, it can be transformed into the new config. This happens in entrypoint before we validate the config against the new spec. """ - config = super().read_config(config_path) - if not self._is_v1_config(config): + config = FileBasedSource.read_config(config_path) + if not cls._is_v1_config(config): converted_config = LegacyConfigTransformer.convert(config) emit_configuration_as_airbyte_control_message(converted_config) return converted_config diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index 47a235c00f14..c751b72403bd 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -59,7 +59,6 @@ def get_matching_files( if not globs or self.file_matches_globs(remote_file, globs): yield remote_file - @contextmanager def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: try: result = open( @@ -73,8 +72,4 @@ def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str f"We don't have access to {file.uri}. The file appears to have become unreachable during sync." f"Check whether key {file.uri} exists in `{self.config.azure_blob_storage_container_name}` container and/or has proper ACL permissions" ) - # see https://docs.python.org/3/library/contextlib.html#contextlib.contextmanager for why we do this - try: - yield result - finally: - result.close() + return result diff --git a/airbyte-integrations/connectors/source-azure-table/Dockerfile b/airbyte-integrations/connectors/source-azure-table/Dockerfile index 6b59fb5dbbb7..ce1a741668d0 100644 --- a/airbyte-integrations/connectors/source-azure-table/Dockerfile +++ b/airbyte-integrations/connectors/source-azure-table/Dockerfile @@ -34,5 +34,5 @@ COPY source_azure_table ./source_azure_table ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-azure-table diff --git a/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml b/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml index 89f5b61d61a3..dfb70d970ab8 100644 --- a/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml @@ -1,27 +1,32 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-azure-table:dev -tests: +acceptance_tests: spec: - - spec_path: "source_azure_table/spec.json" + tests: + - spec_path: "source_azure_table/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - validate_schema: False + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + validate_schema: False incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - ignored_fields: - "AirbyteTest": ["record"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json index ac616e99229b..c0596a4d3d31 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json @@ -1,5 +1,9 @@ -{ - "Test": { - "PartitionKey": "abcd" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "PartitionKey": "999" }, + "stream_descriptor": { "name": "pokemon" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json deleted file mode 100644 index 40d0d8b72734..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "Test", - "json_schema": { - "properties": { - "PartitionKey": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "source_defined_cursor": true, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["PartitionKey"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json index 40d0d8b72734..bfa35916e08e 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json @@ -2,20 +2,24 @@ "streams": [ { "stream": { - "name": "Test", - "json_schema": { - "properties": { - "PartitionKey": { - "type": "string" - } - } - }, + "name": "pokemon", + "json_schema": {}, + "source_defined_cursor": true, "supported_sync_modes": ["full_refresh", "incremental"] }, - "source_defined_cursor": true, "sync_mode": "incremental", "destination_sync_mode": "append", "cursor_field": ["PartitionKey"] + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "source_defined_cursor": true, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json deleted file mode 100644 index 3859e6df625f..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "Test": { - "PartitionKey": "1" - } -} diff --git a/airbyte-integrations/connectors/source-azure-table/main.py b/airbyte-integrations/connectors/source-azure-table/main.py index ffdca7c26cef..0831f8065766 100644 --- a/airbyte-integrations/connectors/source-azure-table/main.py +++ b/airbyte-integrations/connectors/source-azure-table/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_azure_table import SourceAzureTable +from source_azure_table.run import run if __name__ == "__main__": - source = SourceAzureTable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-azure-table/metadata.yaml b/airbyte-integrations/connectors/source-azure-table/metadata.yaml index 06efb503b20c..f709d284dcee 100644 --- a/airbyte-integrations/connectors/source-azure-table/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-table/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: database connectorType: source definitionId: 798ae795-5189-42b6-b64e-3cb91db93338 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 dockerRepository: airbyte/source-azure-table githubIssueLabel: source-azure-table icon: azureblobstorage.svg license: MIT name: Azure Table Storage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-azure-table registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-azure-table/setup.py b/airbyte-integrations/connectors/source-azure-table/setup.py index a04c790cbb8f..2a7451ab9484 100644 --- a/airbyte-integrations/connectors/source-azure-table/setup.py +++ b/airbyte-integrations/connectors/source-azure-table/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-azure-table=source_azure_table.run:run", + ], + }, name="source_azure_table", description="Source implementation for Azure Table.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py new file mode 100644 index 000000000000..b39667cd684a --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_azure_table import SourceAzureTable + + +def run(): + source = SourceAzureTable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py index e5ef7b3f6967..9a2fcc8f567e 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py @@ -39,6 +39,7 @@ def get_typed_schema(self) -> object: return { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": True, "properties": {"PartitionKey": {"type": "string"}}, } @@ -50,7 +51,7 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn next(tables_iterator) return AirbyteConnectionStatus(status=Status.SUCCEEDED) except StopIteration: - logger.log("No tables found, but credentials are correct.") + logger.info("The credentials you provided are valid, but no tables were found in the Storage Account.") return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") @@ -70,8 +71,7 @@ def discover(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteC default_cursor_field=["PartitionKey"], ) streams.append(stream) - logger.info(f"Total {streams.count} streams found.") - + logger.info(f"Total {len(streams)} streams found.") return AirbyteCatalog(streams=streams) def streams(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py index 13f68760f540..d4f2669109a2 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py @@ -24,7 +24,7 @@ def name(self): return self.stream_name def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - return {self.cursor_field[0]: latest_record.record.data.get(self.cursor_field[0])} + return {self.cursor_field[0]: latest_record.data.get(self.cursor_field[0])} def _update_state(self, latest_cursor): self._state = latest_cursor diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py new file mode 100644 index 000000000000..4b78c28bb43d --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from unittest import mock + +import pytest +from source_azure_table.azure_table import AzureTableReader +from source_azure_table.source import SourceAzureTable + + +# Fixtures +@pytest.fixture +def config(): + return {"storage_account_name": "dummy-value", "storage_access_key": "dummy-value", "storage_endpoint_suffix": "dummy-value"} + + +@pytest.fixture +def tables(): + table1 = mock.Mock() + table1.name = "AzureTable1" + table2 = mock.Mock() + table2.name = "AzureTable2" + + tables = mock.MagicMock() + tables.__iter__.return_value = [table1, table2] + return tables + + +@pytest.fixture +def source(): + return SourceAzureTable() + + +@pytest.fixture +def logger(): + return logging.getLogger("airbyte") + + +@pytest.fixture +def reader(config, logger): + return AzureTableReader(logger, config) diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py new file mode 100644 index 000000000000..752e8472480c --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest + + +def test_get_table_service_client_return(mocker, reader): + """ + Test that the get_table_service_client method returns the expected Table Service Client. + """ + mock_client = "dummy-client" + mocker.patch( + "source_azure_table.azure_table.TableServiceClient.from_connection_string", + return_value=mock_client, + ) + + client = reader.get_table_service_client() + assert client == mock_client + + +def test_get_table_service_client_handles_exception(mocker, reader): + """ + Test that get_table_service_client method handles exceptions correctly. + """ + mocker.patch( + "source_azure_table.azure_table.TableServiceClient.from_connection_string", + side_effect=Exception("Connection error") + ) + + with pytest.raises(Exception) as exc_info: + reader.get_table_service_client() + + assert "Connection error" in str(exc_info.value) + + +def test_get_table_client_return(mocker, reader): + """ + Test that the get_table_client method returns the expected Table Client. + """ + mock_client = "dummy-client" + mocker.patch( + "source_azure_table.azure_table.TableClient.from_connection_string", + return_value=mock_client, + ) + + table = reader.get_table_client("dummy-table") + assert table == mock_client + + +def test_get_table_client_handles_exception(mocker, reader): + """ + Test that get_table_client method handles exceptions correctly. + """ + + # The method throws its own exception for empty table names + with pytest.raises(Exception) as exc_info: + reader.get_table_client("") + assert "table name is not valid." in str(exc_info.value) + + mocker.patch( + "source_azure_table.azure_table.TableClient.from_connection_string", + side_effect=Exception("Connection error") + ) + + with pytest.raises(Exception) as exc_info: + reader.get_table_client("valid_table_name") + assert "Connection error" in str(exc_info.value) + + +def test_get_tables_return(mocker, reader, tables): + """ + Test that the get_tables method returns the expected tables. + """ + mock_client = mocker.MagicMock() + mock_client.list_tables.return_value = tables.__iter__() + mocker.patch( + "azure.data.tables.TableServiceClient.from_connection_string", + return_value=mock_client + ) + + result = reader.get_tables() + result_table_names = [table.name for table in result] + + expected_table_names = ["AzureTable1", "AzureTable2"] + assert result_table_names == expected_table_names + + +def test_get_tables_handles_exception(mocker, reader): + """ + Test that get_tables method handles exceptions correctly. + """ + mock_client = mocker.MagicMock() + mock_client.list_tables.side_effect = Exception("Failed to list tables") + mocker.patch( + "azure.data.tables.TableServiceClient.from_connection_string", + return_value=mock_client + ) + + with pytest.raises(Exception) as exc_info: + reader.get_tables() + + assert "Failed to list tables" in str(exc_info.value) diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py index bc1ce50f12bc..956375acda0b 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py @@ -2,38 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from unittest import mock - -import pytest from airbyte_cdk.models import AirbyteCatalog, SyncMode -from source_azure_table.source import SourceAzureTable from source_azure_table.streams import AzureTableStream -source = SourceAzureTable() -logger = logging.getLogger() - - -# Fixtures -@pytest.fixture -def config(): - return {"storage_account_name": "dummy-value", "storage_access_key": "dummy-value", "storage_endpoint_suffix": "dummy-value"} - - -@pytest.fixture -def tables(): - table1 = mock.Mock() - table1.name = "AzureTable1" - table2 = mock.Mock() - table2.name = "AzureTable2" - - tables = mock.MagicMock() - tables.__iter__.return_value = [table1, table2] - return tables - # Tests -def test_discover(mocker, config, tables): +def test_discover(mocker, config, tables, source, logger): mocker.patch( "source_azure_table.azure_table.AzureTableReader.get_tables", return_value=tables, @@ -47,6 +21,7 @@ def test_discover(mocker, config, tables): assert stream.json_schema == { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": True, "properties": {"PartitionKey": {"type": "string"}}, } assert stream.supported_sync_modes == [SyncMode.full_refresh, SyncMode.incremental] @@ -54,7 +29,7 @@ def test_discover(mocker, config, tables): assert stream.default_cursor_field == ["PartitionKey"] -def test_streams(mocker, config, tables): +def test_streams(mocker, config, tables, source, logger): mocker.patch( "source_azure_table.azure_table.AzureTableReader.get_tables", return_value=tables, diff --git a/airbyte-integrations/connectors/source-babelforce/main.py b/airbyte-integrations/connectors/source-babelforce/main.py index 33b4c6d35a82..da6273a1dca9 100644 --- a/airbyte-integrations/connectors/source-babelforce/main.py +++ b/airbyte-integrations/connectors/source-babelforce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_babelforce import SourceBabelforce +from source_babelforce.run import run if __name__ == "__main__": - source = SourceBabelforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-babelforce/metadata.yaml b/airbyte-integrations/connectors/source-babelforce/metadata.yaml index 9425eae3c84b..5e2159e65e2b 100644 --- a/airbyte-integrations/connectors/source-babelforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-babelforce/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${region}.babelforce.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-babelforce registries: oss: enabled: true @@ -20,7 +24,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/babelforce tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-babelforce/setup.py b/airbyte-integrations/connectors/source-babelforce/setup.py index 36286f264643..5033ec7b34e2 100644 --- a/airbyte-integrations/connectors/source-babelforce/setup.py +++ b/airbyte-integrations/connectors/source-babelforce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-babelforce=source_babelforce.run:run", + ], + }, name="source_babelforce", description="Source implementation for Babelforce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py b/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py new file mode 100644 index 000000000000..056cf590f326 --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_babelforce import SourceBabelforce + + +def run(): + source = SourceBabelforce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bamboo-hr/main.py b/airbyte-integrations/connectors/source-bamboo-hr/main.py index 2ebc95cb1256..0118185a67fc 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/main.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bamboo_hr import SourceBambooHr +from source_bamboo_hr.run import run if __name__ == "__main__": - source = SourceBambooHr() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml index ea2b009e281f..f5ac3969826f 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml +++ b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml @@ -12,6 +12,10 @@ data: icon: bamboohr.svg license: MIT name: BambooHR + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-bamboo-hr registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-bamboo-hr/setup.py b/airbyte-integrations/connectors/source-bamboo-hr/setup.py index a81279a9f2dd..465c981987f3 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/setup.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-bamboo-hr=source_bamboo_hr.run:run", + ], + }, name="source_bamboo_hr", description="Source implementation for Bamboo Hr.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py new file mode 100644 index 000000000000..d9a04d56a964 --- /dev/null +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bamboo_hr import SourceBambooHr + + +def run(): + source = SourceBambooHr() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bigcommerce/main.py b/airbyte-integrations/connectors/source-bigcommerce/main.py index 2d28000deab0..7830bf519a65 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/main.py +++ b/airbyte-integrations/connectors/source-bigcommerce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bigcommerce import SourceBigcommerce +from source_bigcommerce.run import run if __name__ == "__main__": - source = SourceBigcommerce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml index fd54e9032d49..9a2a63d4a3e0 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.bigcommerce.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-bigcommerce registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-bigcommerce/setup.py b/airbyte-integrations/connectors/source-bigcommerce/setup.py index 7189d313888f..43cc76692ad7 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/setup.py +++ b/airbyte-integrations/connectors/source-bigcommerce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-bigcommerce=source_bigcommerce.run:run", + ], + }, name="source_bigcommerce", description="Source implementation for Bigcommerce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py new file mode 100644 index 000000000000..7e5234a1b2b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bigcommerce import SourceBigcommerce + + +def run(): + source = SourceBigcommerce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml deleted file mode 100644 index db0e62843784..000000000000 --- a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-bigquery:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" - connection: - tests: - - config_path: "secrets/sat-config.json" - status: "succeed" - discovery: - tests: - - config_path: "secrets/sat-config.json" - basic_read: - tests: - - config_path: "secrets/sat-config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - full_refresh: - tests: - - config_path: "secrets/sat-config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" -# DISABLED DUE TO ISSUE WITH DB STATES NOT MATCHING ACCEPTANCE TESTS EXPECTATIONS (wrong key-values) -# incremental: -# tests: -# - config_path: "secrets/sat-config.json" -# configured_catalog_path: "integration_tests/configured_catalog_inc.json" diff --git a/airbyte-integrations/connectors/source-bigquery/build.gradle b/airbyte-integrations/connectors/source-bigquery/build.gradle index ea56e5021420..9eb0f82ba47d 100644 --- a/airbyte-integrations/connectors/source-bigquery/build.gradle +++ b/airbyte-integrations/connectors/source-bigquery/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.13.2' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-bigquery/metadata.yaml b/airbyte-integrations/connectors/source-bigquery/metadata.yaml index f6e8623837bb..e203e3fae69f 100644 --- a/airbyte-integrations/connectors/source-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.2 dockerRepository: airbyte/source-bigquery documentationUrl: https://docs.airbyte.com/integrations/sources/bigquery githubIssueLabel: source-bigquery diff --git a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java index a30c4b4546f3..6c08f587900a 100644 --- a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java +++ b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java @@ -9,6 +9,7 @@ import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; import com.fasterxml.jackson.databind.JsonNode; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.Table; @@ -55,6 +56,10 @@ public class BigQuerySource extends AbstractDbSource>> discoverInternal(fin .name(table.getTableId().getTable()) .fields(Objects.requireNonNull(table.getDefinition().getSchema()).getFields().stream() .map(f -> { - final StandardSQLTypeName standardType = f.getType().getStandardType(); + final StandardSQLTypeName standardType; + if (f.getType().getStandardType() == StandardSQLTypeName.STRUCT && f.getMode() == Field.Mode.REPEATED) { + standardType = StandardSQLTypeName.ARRAY; + } else + standardType = f.getType().getStandardType(); + return new CommonField<>(f.getName(), standardType); }) .collect(Collectors.toList())) diff --git a/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceStructRepeatedTest.java b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceStructRepeatedTest.java new file mode 100644 index 000000000000..1d7b03292515 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceStructRepeatedTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.bigquery; + +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_DATASET_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class BigQuerySourceStructRepeatedTest extends AbstractBigQuerySourceTest { + + @Override + public void createTable(String datasetId) throws SQLException { + // create column name interval which should be escaped + database.execute("CREATE TABLE " + datasetId + ".struct_repeated(id int64, key_value_pairs ARRAY>);"); + database.execute("INSERT INTO " + datasetId + ".struct_repeated (id, key_value_pairs) VALUES (1, [('a', 0.7), ('b', 0.8), ('c', 1.2)]);"); + } + + @Test + public void testReadSuccess() throws Exception { + final List actualMessages = MoreIterators.toList(new BigQuerySource().read(config, getConfiguredCatalog(), null)); + + ObjectMapper mapper = new ObjectMapper(); + // JsonNode actualObj = mapper.readTree("{\"key_value_pairs\":[{ \"key\": \"a\",\"value\": \"0.7\"}, + // {\"key\": \"b\",\"value\": \"0.8\"}, {\"key\": \"c\",\"value\": \"1.2\"}]}"); + JsonNode actualObj = mapper.readTree("[{ \"key\": \"a\",\"value\": 0.7}, {\"key\": \"b\",\"value\": 0.8}, {\"key\": \"c\",\"value\": 1.2}]"); + + assertNotNull(actualMessages); + assertEquals(1, actualMessages.size()); + + assertNotNull(actualMessages.get(0).getRecord().getData().get("id")); + assertEquals(actualObj, actualMessages.get(0).getRecord().getData().get("key_value_pairs")); + } + + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return CatalogHelpers.createConfiguredAirbyteCatalog( + "struct_repeated", + config.get(CONFIG_DATASET_ID).asText(), + Field.of("id", JsonSchemaType.NUMBER), + Field.of("key_value_pairs", JsonSchemaType.ARRAY)); + } + +} diff --git a/airbyte-integrations/connectors/source-bing-ads/README.md b/airbyte-integrations/connectors/source-bing-ads/README.md index 7daaf3ea4797..d8e88f8da26f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/README.md +++ b/airbyte-integrations/connectors/source-bing-ads/README.md @@ -1,133 +1,55 @@ -# Bing Ads Source +# Bing-Ads source connector -This is the repository for the Bing Ads source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/bing-ads). + +This is the repository for the Bing-Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/bing-ads). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Debugging tips +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -Put this code to the top of `source.py` file to be able to debug API requests/responses - -```python -import logging -logging.basicConfig(level=logging.INFO) -logging.getLogger("suds.client").setLevel(logging.DEBUG) -logging.getLogger("suds.transport.http").setLevel(logging.DEBUG) -``` - -#### Caching responses -We need to cache `Accounts` to reuse in `Campaigns`, cache `Campaigns` to reuse in `AdGroups` and cache `AdGroups` to reuse in `Ads` - - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/bing-ads) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bing_ads/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/bing-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bing_ads/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source bing-ads test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-bing-ads spec +poetry run source-bing-ads check --config secrets/config.json +poetry run source-bing-ads discover --config secrets/config.json +poetry run source-bing-ads read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-bing-ads build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-bing-ads:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations +An image will be available on your host with the tag `airbyte/source-bing-ads:dev`. -from typing import TYPE_CHECKING -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-bing-ads:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-bing-ads:dev . -# Running the spec command against your patched connector -docker run airbyte/source-bing-ads:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-bing-ads:dev spec @@ -136,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bing-ads:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bing-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-bing-ads test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bing-ads test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bing-ads.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/bing-ads.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index bef3f6d63a2a..1dfe3da1c17e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -91,6 +91,11 @@ acceptance_tests: - name: campaign_labels bypass_reason: "This stream is tested without start date" ignored_fields: + ads: + - name: Descriptions/AssetLink/*/AssetPerformanceLabel + bypass_reason: "This field indicates the asset's performance and is dynamically updated by the API." + - name: Headlines/AssetLink/*/AssetPerformanceLabel + bypass_reason: "This field indicates the asset's performance and is dynamically updated by the API." account_impression_performance_report_weekly: - name: Ctr bypass_reason: "dynamic field" @@ -100,6 +105,24 @@ acceptance_tests: bypass_reason: "dynamic field" - name: LowQualityImpressionsPercent bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: LowQualityImpressions + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" age_gender_audience_report_daily: - name: Impressions bypass_reason: "dynamic field" @@ -111,6 +134,8 @@ acceptance_tests: bypass_reason: "dynamic field" - name: MainlineBid bypass_reason: "dynamic field" + - name: FirstPageBid + bypass_reason: "dynamic field" campaign_impression_performance_report_weekly: - name: Impressions bypass_reason: "dynamic field" @@ -118,6 +143,24 @@ acceptance_tests: bypass_reason: "dynamic field" - name: LowQualityImpressionsPercent bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" campaign_performance_report_weekly: - name: Impressions bypass_reason: "dynamic field" @@ -127,6 +170,24 @@ acceptance_tests: bypass_reason: "dynamic field" - name: LowQualityImpressions bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" ad_group_impression_performance_report_weekly: - name: Impressions bypass_reason: "dynamic field" @@ -140,6 +201,26 @@ acceptance_tests: bypass_reason: "dynamic field" - name: HistoricalLandingPageExperience bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" ad_group_performance_report_weekly: - name: Impressions bypass_reason: "dynamic field" @@ -149,6 +230,18 @@ acceptance_tests: bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" ad_performance_report_daily: - name: TimePeriod bypass_reason: "dynamic field" @@ -224,6 +317,16 @@ acceptance_tests: bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" budget_summary_report: - name: Date bypass_reason: "dynamic field" @@ -234,16 +337,12 @@ acceptance_tests: - name: MonthToDateSpend bypass_reason: "dynamic field" campaign_impression_performance_report_daily: - - name: TimePeriod - bypass_reason: "dynamic field" - name: AdDistribution bypass_reason: "dynamic field" - name: LowQualityImpressions bypass_reason: "dynamic field" - name: LowQualityImpressionsPercent bypass_reason: "dynamic field" - - name: DeviceType - bypass_reason: "dynamic field" - name: ImpressionSharePercent bypass_reason: "dynamic field" - name: ImpressionLostToBudgetPercent @@ -258,8 +357,6 @@ acceptance_tests: bypass_reason: "dynamic field" - name: HistoricalLandingPageExperience bypass_reason: "dynamic field" - - name: Network - bypass_reason: "dynamic field" - name: ExactMatchImpressionSharePercent bypass_reason: "dynamic field" - name: AbsoluteTopImpressionSharePercent @@ -284,6 +381,20 @@ acceptance_tests: bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: LowQualityClicks + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: LowQualityGeneralClicks + bypass_reason: "dynamic field" account_performance_report_daily: - name: Ctr bypass_reason: "dynamic field" @@ -292,12 +403,24 @@ acceptance_tests: - name: AverageCpm bypass_reason: "dynamic field" account_performance_report_weekly: - - name: Ctr + - name: Clicks bypass_reason: "dynamic field" - - name: Impressions + - name: Spend + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" account_impression_performance_report_daily: - name: Ctr bypass_reason: "dynamic field" @@ -307,6 +430,28 @@ acceptance_tests: bypass_reason: "dynamic field" - name: LowQualityImpressionsPercent bypass_reason: "dynamic field" + - name: ExactMatchImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionSharePercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: TopImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionRatePercent + bypass_reason: "dynamic field" + - name: ImpressionSharePercent + bypass_reason: "dynamic field" + - name: ImpressionLostToBudgetPercent + bypass_reason: "dynamic field" + - name: ImpressionLostToRankAggPercent + bypass_reason: "dynamic field" ad_group_impression_performance_report_daily: - name: Ctr bypass_reason: "dynamic field" @@ -314,6 +459,24 @@ acceptance_tests: bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: ImpressionSharePercent + bypass_reason: "dynamic field" + - name: ImpressionLostToBudgetPercent + bypass_reason: "dynamic field" + - name: ImpressionLostToRankAggPercent + bypass_reason: "dynamic field" + - name: ExactMatchImpressionSharePercent + bypass_reason: "dynamic field" campaign_performance_report_daily: - name: Ctr bypass_reason: "dynamic field" @@ -321,6 +484,16 @@ acceptance_tests: bypass_reason: "dynamic field" - name: AverageCpm bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" keyword_performance_report_daily: - name: Language bypass_reason: "dynamic field" @@ -354,6 +527,16 @@ acceptance_tests: bypass_reason: "dynamic field" - name: MainlineBid bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: FirstPageBid + bypass_reason: "dynamic field" timeout_seconds: 9000 - config_path: secrets/config_no_date.json expect_records: diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl index 5555639268c9..91a3227cb703 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -1,29 +1,29 @@ -{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":2.27},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1356799861840328,"Language":null,"Name":"keywords","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":7,"Month":11,"Year":2023},"Status":"Active","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null},"CampaignId":531016227,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913367220} -{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"Active","FinalAppUrls":null,"FinalMobileUrls":null,"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.com"]},"ForwardCompatibilityMap":null,"Id":84800390693061,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239363892977,"Name":null,"Type":"TextAsset","Text":"Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892976,"Name":null,"Type":"TextAsset","Text":"Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.com","Headlines":{"AssetLink":[{"Asset":{"Id":10239363892979,"Name":null,"Type":"TextAsset","Text":"Get synced with Airbyte"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893384,"Name":null,"Type":"TextAsset","Text":"Data management made easy"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892978,"Name":null,"Type":"TextAsset","Text":"Connectors for every need"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892980,"Name":null,"Type":"TextAsset","Text":"Industry-leading connectors"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893383,"Name":null,"Type":"TextAsset","Text":"Try Airbyte now for free"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null,"AdGroupId":1356799861840328,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1700075716309} -{"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":2.0,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":531016227,"MultimediaAdsBidAdjustment":40,"Name":"Airbyte test","Status":"Active","SubType":null,"TimeZone":"CentralTimeUSCanada","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913381852} -{"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180535609,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T08:24:26.603000","Name":"DEMO-ACCOUNT","Number":"F149W3B6","ParentCustomerId":251186883,"PaymentMethodId":null,"PaymentMethodType":null,"PrimaryUserId":138225488,"AccountLifeCycleStatus":"Pause","TimeStamp":"AAAAAH10c1A=","TimeZone":"Santiago","PauseReason":2,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149694999,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1699913384475} -{"stream": "account_performance_report_daily", "data": {"AccountId": 180519267, "TimePeriod": "2023-12-07", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Smartphone", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "AccountName": "Airbyte", "AccountNumber": "F149MJ18", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 1, "Ctr": 0.42, "Spend": 0.04, "Impressions": 236, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "AverageCpc": 0.04, "AveragePosition": 0.0, "AverageCpm": 0.17, "Conversions": 0.0, "ConversionsQualified": 0.0, "ConversionRate": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": 0.0, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979051100} -{"stream": "account_performance_report_weekly", "data": {"AccountId": 180519267, "TimePeriod": "2023-12-03", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Tablet", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "AccountName": "Airbyte", "AccountNumber": "F149MJ18", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 8, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionsQualified": 0.0, "ConversionRate": null, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979146771} -{"stream": "ad_group_performance_report_daily", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "TimePeriod": "2023-12-07", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Smartphone", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "Language": "English", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "AdGroupName": "keywords", "AdGroupType": "Standard", "Impressions": 239, "Clicks": 1, "Ctr": 0.42, "Spend": 0.04, "CostPerConversion": null, "QualityScore": 6.0, "ExpectedCtr": "2", "AdRelevance": 3.0, "LandingPageExperience": 1.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "CostPerAssist": null, "CustomParameters": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllConversions": 0, "AllConversionRate": 0.0, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.04, "AveragePosition": 0.0, "AverageCpm": 0.17, "Conversions": 0.0, "ConversionRate": 0.0, "ConversionsQualified": 0.0, "HistoricalQualityScore": null, "HistoricalExpectedCtr": null, "HistoricalAdRelevance": null, "HistoricalLandingPageExperience": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979416341} -{"stream": "ad_group_performance_report_weekly", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "TimePeriod": "2023-12-03", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Tablet", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "Language": "English", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "AdGroupName": "keywords", "AdGroupType": "Standard", "Impressions": 7, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 6.0, "ExpectedCtr": "2", "AdRelevance": 3.0, "LandingPageExperience": 1.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "CostPerAssist": null, "CustomParameters": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "HistoricalQualityScore": 6.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 1.0, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979256059} -{"stream": "ad_group_impression_performance_report_daily", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-07", "Status": "Active", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 295, "Clicks": 2, "Ctr": 0.68, "AverageCpc": 0.03, "Spend": 0.06, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": 0.0, "CostPerConversion": null, "DeviceType": "Smartphone", "Language": "English", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "QualityScore": 6, "ExpectedCtr": 2.0, "AdRelevance": 3, "LandingPageExperience": 1, "HistoricalQualityScore": null, "HistoricalExpectedCtr": null, "HistoricalAdRelevance": null, "HistoricalLandingPageExperience": null, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "TrackingTemplate": null, "CustomParameters": null, "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupLabels": null, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "FinalUrlSuffix": null, "CampaignType": "Search & content", "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": null, "TopImpressionRatePercent": null, "BaseCampaignId": 531016227, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": 0.0, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "RelativeCtr": null, "AdGroupType": "Standard", "AverageCpm": 0.2, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701979628655} -{"stream": "ad_group_impression_performance_report_weekly", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-03", "Status": "Active", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 14, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": null, "CostPerConversion": null, "DeviceType": "Tablet", "Language": "English", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "QualityScore": 6, "ExpectedCtr": 2.0, "AdRelevance": 3, "LandingPageExperience": 1, "HistoricalQualityScore": 6, "HistoricalExpectedCtr": 2, "HistoricalAdRelevance": 3, "HistoricalLandingPageExperience": 1, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "TrackingTemplate": null, "CustomParameters": null, "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupLabels": null, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "FinalUrlSuffix": null, "CampaignType": "Search & content", "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": null, "TopImpressionRatePercent": null, "BaseCampaignId": 531016227, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "RelativeCtr": null, "AdGroupType": "Standard", "AverageCpm": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701979716237} -{"stream": "ad_performance_report_daily", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "AdId": 84800390693061, "TimePeriod": "2023-12-07", "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Smartphone", "Language": "English", "Network": "Audience", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "DeliveredMatchType": "Exact", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "AdGroupName": "keywords", "Impressions": 239, "Clicks": 1, "Ctr": 0.42, "Spend": 0.04, "CostPerConversion": null, "DestinationUrl": null, "Assists": 0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "AdDescription": null, "AdDescription2": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "Conversions": 0.0, "ConversionRate": 0.0, "ConversionsQualified": 0.0, "AverageCpc": 0.04, "AveragePosition": 0.0, "AverageCpm": 0.17, "AllConversions": 0, "AllConversionRate": 0.0, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979816123} -{"stream": "ad_performance_report_weekly", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "AdId": 84800390693061, "TimePeriod": "2023-12-03", "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Tablet", "Language": "English", "Network": "Audience", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "DeliveredMatchType": "Exact", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "AdGroupName": "keywords", "Impressions": 8, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "DestinationUrl": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "AdDescription": null, "AdDescription2": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701979935551} -{"stream": "budget_summary_report", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "CampaignId": 531016227, "CampaignName": "Airbyte test", "Date": "2023-12-07", "MonthlyBudget": 60.8, "DailySpend": 0.52, "MonthToDateSpend": 12.75}, "emitted_at": 1701980000241} -{"stream": "campaign_performance_report_daily", "data": {"AccountId": 180519267, "CampaignId": 531016227, "TimePeriod": "2023-12-07", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Smartphone", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "CampaignStatus": "Active", "CampaignLabels": null, "Impressions": 239, "Clicks": 1, "Ctr": 0.42, "Spend": 0.04, "CostPerConversion": null, "QualityScore": 6.0, "AdRelevance": 3.0, "LandingPageExperience": 1.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "CustomParameters": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllConversions": 0, "ConversionsQualified": 0.0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.04, "AveragePosition": 0.0, "AverageCpm": 0.17, "Conversions": 0.0, "ConversionRate": null, "LowQualityClicks": 0, "LowQualityClicksPercent": 0.0, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "HistoricalQualityScore": null, "HistoricalExpectedCtr": null, "HistoricalAdRelevance": null, "HistoricalLandingPageExperience": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "BudgetName": null, "BudgetStatus": null, "BudgetAssociationStatus": "Current"}, "emitted_at": 1701980084453} -{"stream": "campaign_performance_report_weekly", "data": {"AccountId": 180519267, "CampaignId": 531016227, "TimePeriod": "2023-12-03", "CurrencyCode": "USD", "AdDistribution": "Audience", "DeviceType": "Tablet", "Network": "Audience", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Audience network", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "CampaignType": "Search & content", "CampaignStatus": "Active", "CampaignLabels": null, "Impressions": 7, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 6.0, "AdRelevance": 3.0, "LandingPageExperience": 1.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "ConversionsQualified": 0.0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "HistoricalQualityScore": 6.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 1.0, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "BudgetName": null, "BudgetStatus": null, "BudgetAssociationStatus": "Current"}, "emitted_at": 1701980157383} -{"stream": "campaign_impression_performance_report_daily", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-07", "CampaignStatus": "Active", "CampaignName": "Airbyte test", "CampaignId": 531016227, "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 297, "Clicks": 2, "Ctr": 0.67, "AverageCpc": 0.03, "Spend": 0.06, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": null, "CostPerConversion": null, "LowQualityClicks": 0, "LowQualityClicksPercent": 0.0, "LowQualityImpressions": 2, "LowQualityImpressionsPercent": 0.67, "LowQualityConversions": 0, "LowQualityConversionRate": null, "DeviceType": "Smartphone", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "QualityScore": 6.0, "ExpectedCtr": "2", "AdRelevance": 3.0, "LandingPageExperience": 1.0, "HistoricalQualityScore": null, "HistoricalExpectedCtr": null, "HistoricalAdRelevance": null, "HistoricalLandingPageExperience": null, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "TrackingTemplate": null, "CustomParameters": null, "AccountStatus": "Active", "LowQualityGeneralClicks": 0, "LowQualitySophisticatedClicks": 0, "CampaignLabels": null, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "FinalUrlSuffix": null, "CampaignType": "Search & content", "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "BaseCampaignId": 531016227, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "RelativeCtr": null, "AverageCpm": 0.2, "ConversionsQualified": 0.0, "LowQualityConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701980256348} -{"stream": "campaign_impression_performance_report_weekly", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-03", "CampaignStatus": "Active", "CampaignName": "Airbyte test", "CampaignId": 531016227, "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 14, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": null, "CostPerConversion": null, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualityImpressionsPercent": 0.0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "DeviceType": "Tablet", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "QualityScore": 6.0, "ExpectedCtr": "2", "AdRelevance": 3.0, "LandingPageExperience": 1.0, "HistoricalQualityScore": 6, "HistoricalExpectedCtr": 2, "HistoricalAdRelevance": 3, "HistoricalLandingPageExperience": 1, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "TrackingTemplate": null, "CustomParameters": null, "AccountStatus": "Active", "LowQualityGeneralClicks": 0, "LowQualitySophisticatedClicks": 0, "CampaignLabels": null, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "FinalUrlSuffix": null, "CampaignType": "Search & content", "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "BaseCampaignId": 531016227, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "RelativeCtr": null, "AverageCpm": 0.0, "ConversionsQualified": 0.0, "LowQualityConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701980347415} -{"stream": "keyword_performance_report_daily", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "KeywordId": 84801135055370, "Keyword": "Airbyte", "AdId": 84800390693061, "TimePeriod": "2023-12-07", "CurrencyCode": "USD", "DeliveredMatchType": "Phrase", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "Portuguese", "Network": "Syndicated search partners", "DeviceOS": "Unknown", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "AdGroupName": "keywords", "KeywordStatus": "Active", "HistoricalExpectedCtr": null, "HistoricalAdRelevance": null, "HistoricalLandingPageExperience": null, "HistoricalQualityScore": null, "Impressions": 3, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 2.27, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 10.0, "ExpectedCtr": "3", "AdRelevance": 3.0, "LandingPageExperience": 3.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": 4.77, "MainlineBid": 0.22, "FirstPageBid": 0.13, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701980440595} -{"stream": "keyword_performance_report_weekly", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "KeywordId": 84801135055370, "Keyword": "Airbyte", "AdId": 84800390693061, "TimePeriod": "2023-12-03", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "Portuguese", "Network": "Microsoft sites and select traffic", "DeviceOS": "Windows", "TopVsOther": "Microsoft sites and select traffic - other", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "AdGroupName": "keywords", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 2.27, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 10.0, "ExpectedCtr": "3", "AdRelevance": 3.0, "LandingPageExperience": 3.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": 4.77, "MainlineBid": 0.22, "FirstPageBid": 0.13, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701980530452} -{"stream": "geographic_performance_report_daily", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "TimePeriod": "2023-12-07", "AccountNumber": "F149MJ18", "Country": "South Africa", "State": "Western Cape", "MetroArea": "City of Cape Town", "City": "Cape Town", "ProximityTargetLocation": null, "Radius": "0", "LocationType": "Physical location", "MostSpecificLocation": "Cape Town", "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "137943", "BaseCampaignId": "531016227", "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "0.00", "AllConversionsQualified": "0.00", "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Microsoft sites and select traffic", "DeviceOS": "Windows", "TopVsOther": "Microsoft sites and select traffic - other", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "AdGroupName": "keywords", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701980771941} -{"stream": "geographic_performance_report_weekly", "data": {"AccountId": 180519267, "CampaignId": 531016227, "AdGroupId": 1356799861840328, "TimePeriod": "2023-12-03", "AccountNumber": "F149MJ18", "Country": "South Africa", "State": "Western Cape", "MetroArea": "City of Cape Town", "City": "Cape Town", "ProximityTargetLocation": null, "Radius": "0", "LocationType": "Physical location", "MostSpecificLocation": "Cape Town", "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "137943", "BaseCampaignId": "531016227", "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "0.00", "AllConversionsQualified": "0.00", "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Microsoft sites and select traffic", "DeviceOS": "Windows", "TopVsOther": "Microsoft sites and select traffic - other", "BidMatchType": "Broad", "AccountName": "Airbyte", "CampaignName": "Airbyte test", "AdGroupName": "keywords", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1701981094311} -{"stream": "age_gender_audience_report_daily", "data": {"AccountId": 180519267, "AgeGroup": "65+", "Gender": "Male", "TimePeriod": "2023-12-07", "AllConversions": 0, "AccountName": "Airbyte", "AccountNumber": "F149MJ18", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "AdDistribution": "Audience", "Impressions": 11, "Clicks": 0, "Conversions": 0.0, "Spend": 0.0, "Revenue": 0.0, "ExtendedCost": 0.0, "Assists": 0, "Language": "English", "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupStatus": "Active", "BaseCampaignId": "531016227", "AllRevenue": 0.0, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0}, "emitted_at": 1701981164857} -{"stream": "age_gender_audience_report_weekly", "data": {"AccountId": 180519267, "AgeGroup": "65+", "Gender": "Female", "TimePeriod": "2023-12-03", "AllConversions": 0, "AccountName": "Airbyte", "AccountNumber": "F149MJ18", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "AdDistribution": "Audience", "Impressions": 6, "Clicks": 0, "Conversions": 0.0, "Spend": 0.0, "Revenue": 0.0, "ExtendedCost": 0.0, "Assists": 0, "Language": "English", "AccountStatus": "Active", "CampaignStatus": "Active", "AdGroupStatus": "Active", "BaseCampaignId": "531016227", "AllRevenue": 0.0, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0}, "emitted_at": 1701981230932} -{"stream": "search_query_performance_report_daily", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-07", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "AdId": 84800390693061, "AdType": "Responsive search ad", "DestinationUrl": null, "BidMatchType": "Broad", "DeliveredMatchType": "Broad", "CampaignStatus": "Active", "AdStatus": "Active", "Impressions": 1, "Clicks": 1, "Ctr": 100.0, "AverageCpc": 0.05, "Spend": 0.05, "AveragePosition": 0.0, "SearchQuery": "datasource", "Keyword": "ELT infrastructure", "AdGroupCriterionId": null, "Conversions": 0, "ConversionRate": 0.0, "CostPerConversion": null, "Language": "English", "KeywordId": 84801135055369, "Network": "Microsoft sites and select traffic", "TopVsOther": "Microsoft sites and select traffic - other", "DeviceType": "Computer", "DeviceOS": "Windows", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "AccountStatus": "Active", "AdGroupStatus": "Active", "KeywordStatus": "Active", "CampaignType": "Search & content", "CustomerId": 251186883, "CustomerName": "Daxtarity Inc.", "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": 0.0, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllRevenuePerConversion": null, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "AverageCpm": 50.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0}, "emitted_at": 1701981312538} -{"stream": "search_query_performance_report_weekly", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-03", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "AdId": 84800390693061, "AdType": "Responsive search ad", "DestinationUrl": null, "BidMatchType": "Broad", "DeliveredMatchType": "Broad", "CampaignStatus": "Active", "AdStatus": "Active", "Impressions": 3, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "SearchQuery": "informatica data integration platform", "Keyword": "ELT infrastructure", "AdGroupCriterionId": null, "Conversions": 0, "ConversionRate": null, "CostPerConversion": null, "Language": "English", "KeywordId": 84801135055369, "Network": "Syndicated search partners", "TopVsOther": "Syndicated search partners - Other", "DeviceType": "Smartphone", "DeviceOS": "Android", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "AccountStatus": "Active", "AdGroupStatus": "Active", "KeywordStatus": "Active", "CampaignType": "Search & content", "CustomerId": 251186883, "CustomerName": "Daxtarity Inc.", "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "AverageCpm": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0}, "emitted_at": 1701981381431} -{"stream": "user_location_performance_report_daily", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-07", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "Country": "South Africa", "State": "Western Cape", "MetroArea": "City of Cape Town", "CurrencyCode": "USD", "AdDistribution": "Search", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "ProximityTargetLocation": null, "Radius": 0, "Language": "English", "City": "Cape Town", "QueryIntentCountry": "United States", "QueryIntentState": null, "QueryIntentCity": null, "QueryIntentDMA": null, "BidMatchType": "Broad", "DeliveredMatchType": "Broad", "Network": "Microsoft sites and select traffic", "TopVsOther": "Microsoft sites and select traffic - other", "DeviceType": "Computer", "DeviceOS": "Windows", "Assists": 0, "Conversions": 0, "ConversionRate": null, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerConversion": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "County": null, "PostalCode": null, "QueryIntentCounty": null, "QueryIntentPostalCode": null, "LocationId": 137943, "QueryIntentLocationId": 190, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "AverageCpm": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "Neighborhood": null, "QueryIntentNeighborhood": null, "ViewThroughRevenue": 0.0, "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null}, "emitted_at": 1701981614442} -{"stream": "user_location_performance_report_weekly", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-03", "CampaignName": "Airbyte test", "CampaignId": 531016227, "AdGroupName": "keywords", "AdGroupId": 1356799861840328, "Country": "South Africa", "State": "Western Cape", "MetroArea": "City of Cape Town", "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 7, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "ProximityTargetLocation": null, "Radius": 0, "Language": "English", "City": "Cape Town", "QueryIntentCountry": "South Africa", "QueryIntentState": null, "QueryIntentCity": null, "QueryIntentDMA": null, "BidMatchType": "Broad", "DeliveredMatchType": "Exact", "Network": "Audience", "TopVsOther": "Audience network", "DeviceType": "Computer", "DeviceOS": "Unknown", "Assists": 0, "Conversions": 0, "ConversionRate": null, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerConversion": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "County": null, "PostalCode": null, "QueryIntentCounty": null, "QueryIntentPostalCode": null, "LocationId": 137943, "QueryIntentLocationId": 168, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": 0.0, "AverageCpm": 0.0, "ConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "Neighborhood": null, "QueryIntentNeighborhood": null, "ViewThroughRevenue": 0.0, "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null}, "emitted_at": 1701981938898} -{"stream": "account_impression_performance_report_daily", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-07", "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 299, "Clicks": 2, "Ctr": 0.67, "AverageCpc": 0.03, "Spend": 0.06, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": 0.0, "CostPerConversion": null, "LowQualityClicks": 0, "LowQualityClicksPercent": 0.0, "LowQualityImpressions": 2, "LowQualityImpressionsPercent": 0.66, "LowQualityConversions": 0, "LowQualityConversionRate": null, "DeviceType": "Smartphone", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": 0.0, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "AccountStatus": "Active", "LowQualityGeneralClicks": 0, "LowQualitySophisticatedClicks": 0, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": null, "TopImpressionRatePercent": null, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": 0.0, "AllCostPerConversion": null, "AllReturnOnAdSpend": 0.0, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "AverageCpm": 0.2, "ConversionsQualified": 0.0, "LowQualityConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701982014926} -{"stream": "account_impression_performance_report_weekly", "data": {"AccountName": "Airbyte", "AccountNumber": "F149MJ18", "AccountId": 180519267, "TimePeriod": "2023-12-03", "CurrencyCode": "USD", "AdDistribution": "Audience", "Impressions": 20, "Clicks": 0, "Ctr": 0.0, "AverageCpc": 0.0, "Spend": 0.0, "AveragePosition": 0.0, "Conversions": 0, "ConversionRate": null, "CostPerConversion": null, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 2, "LowQualityImpressionsPercent": 9.09, "LowQualityConversions": 0, "LowQualityConversionRate": null, "DeviceType": "Tablet", "ImpressionSharePercent": null, "ImpressionLostToBudgetPercent": null, "ImpressionLostToRankAggPercent": null, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Network": "Audience", "Assists": 0, "Revenue": 0.0, "ReturnOnAdSpend": null, "CostPerAssist": null, "RevenuePerConversion": null, "RevenuePerAssist": null, "AccountStatus": "Active", "LowQualityGeneralClicks": 0, "LowQualitySophisticatedClicks": 0, "ExactMatchImpressionSharePercent": null, "ClickSharePercent": null, "AbsoluteTopImpressionSharePercent": null, "TopImpressionShareLostToRankPercent": null, "TopImpressionShareLostToBudgetPercent": null, "AbsoluteTopImpressionShareLostToRankPercent": null, "AbsoluteTopImpressionShareLostToBudgetPercent": null, "TopImpressionSharePercent": null, "AbsoluteTopImpressionRatePercent": null, "TopImpressionRatePercent": null, "AllConversions": 0, "AllRevenue": 0.0, "AllConversionRate": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllRevenuePerConversion": null, "ViewThroughConversions": 0, "AudienceImpressionSharePercent": null, "AudienceImpressionLostToRankPercent": null, "AudienceImpressionLostToBudgetPercent": null, "AverageCpm": 0.0, "ConversionsQualified": 0.0, "LowQualityConversionsQualified": 0.0, "AllConversionsQualified": 0.0, "ViewThroughConversionsQualified": null, "ViewThroughRevenue": 0.0, "VideoViews": 0, "ViewThroughRate": 0.0, "AverageCPV": null, "VideoViewsAt25Percent": 0, "VideoViewsAt50Percent": 0, "VideoViewsAt75Percent": 0, "CompletedVideoViews": 0, "VideoCompletionRate": null, "TotalWatchTimeInMS": 0, "AverageWatchTimePerVideoView": null, "AverageWatchTimePerImpression": 0.0, "Sales": 0, "CostPerSale": null, "RevenuePerSale": null, "Installs": 0, "CostPerInstall": null, "RevenuePerInstall": null}, "emitted_at": 1701982180735} +{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":2.27},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1356799861840328,"Language":null,"Name":"keywords","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":7,"Month":11,"Year":2023},"Status":"Active","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null},"CampaignId":531016227,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1704833256596} +{"stream": "ads", "data": {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "Active", "FinalAppUrls": null, "FinalMobileUrls": null, "FinalUrlSuffix": null, "FinalUrls": {"string": ["https://airbyte.com"]}, "ForwardCompatibilityMap": null, "Id": 84800390693061, "Status": "Active", "TrackingUrlTemplate": null, "Type": "ResponsiveSearch", "UrlCustomParameters": null, "Descriptions": {"AssetLink": [{"Asset": {"Id": 10239363892977, "Name": null, "Type": "TextAsset", "Text": "Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239363892976, "Name": null, "Type": "TextAsset", "Text": "Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Domain": "airbyte.com", "Headlines": {"AssetLink": [{"Asset": {"Id": 10239363892979, "Name": null, "Type": "TextAsset", "Text": "Get synced with Airbyte"}, "AssetPerformanceLabel": "Good", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239363893384, "Name": null, "Type": "TextAsset", "Text": "Data management made easy"}, "AssetPerformanceLabel": "Best", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239363892978, "Name": null, "Type": "TextAsset", "Text": "Connectors for every need"}, "AssetPerformanceLabel": "Good", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239363892980, "Name": null, "Type": "TextAsset", "Text": "Industry-leading connectors"}, "AssetPerformanceLabel": "Good", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239363893383, "Name": null, "Type": "TextAsset", "Text": "Try Airbyte now for free"}, "AssetPerformanceLabel": "Low", "EditorialStatus": "Active", "PinnedField": null}]}, "Path1": null, "Path2": null, "AdGroupId": 1356799861840328, "AccountId": 180519267, "CustomerId": 251186883}, "emitted_at": 1706721454075} +{"stream": "campaigns", "data": {"AudienceAdsBidAdjustment": 0, "BiddingScheme": {"Type": "EnhancedCpc"}, "BudgetType": "DailyBudgetStandard", "DailyBudget": 2.0, "ExperimentId": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 531016227, "MultimediaAdsBidAdjustment": 40, "Name": "Airbyte test", "Status": "Active", "SubType": null, "TimeZone": "CentralTimeUSCanada", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "CampaignType": "Search", "Settings": {"Setting": [{"Type": "TargetSetting", "Details": {"TargetSettingDetail": [{"CriterionTypeGroup": "Audience", "TargetAndBid": false}]}}]}, "BudgetId": null, "Languages": {"string": ["English"]}, "AdScheduleUseSearcherTimeZone": false, "AccountId": 180519267, "CustomerId": 251186883}, "emitted_at": 1702903287209} +{"stream": "accounts", "data": {"BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", "Id": 180535609, "Language": "English", "LastModifiedByUserId": 0, "LastModifiedTime": "2023-08-11T08:24:26.603000", "Name": "DEMO-ACCOUNT", "Number": "F149W3B6", "ParentCustomerId": 251186883, "PaymentMethodId": null, "PaymentMethodType": null, "PrimaryUserId": 138225488, "AccountLifeCycleStatus": "Pause", "TimeStamp": "AAAAAH10c1A=", "TimeZone": "Santiago", "PauseReason": 2, "ForwardCompatibilityMap": null, "LinkedAgencies": null, "SalesHouseCustomerId": null, "TaxInformation": null, "BackUpPaymentInstrumentId": null, "BillingThresholdAmount": null, "BusinessAddress": {"City": "San Francisco", "CountryCode": "US", "Id": 149694999, "Line1": "350 29th avenue", "Line2": null, "Line3": null, "Line4": null, "PostalCode": "94121", "StateOrProvince": "CA", "TimeStamp": null, "BusinessName": "Daxtarity Inc."}, "AutoTagType": "Inactive", "SoldToPaymentInstrumentId": null, "AccountMode": "Expert"}, "emitted_at": 1702903290287} +{"stream":"account_performance_report_daily","data":{"AccountId":180519267,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionsQualified":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833285214} +{"stream":"account_performance_report_weekly","data":{"AccountId":180519267,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":5,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionsQualified":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":4,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833307364} +{"stream":"ad_group_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Microsoft sites and select traffic","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","Language":"Portuguese","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":2,"Clicks":1,"Ctr":50.0,"Spend":0.01,"CostPerConversion":null,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.01,"AveragePosition":0.0,"AverageCpm":5.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704884363801} +{"stream":"ad_group_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","Language":"German","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833349472} +{"stream":"ad_group_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"Czech","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":7,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":2,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833929228} +{"stream":"ad_group_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"Bulgarian","ImpressionSharePercent":13.64,"ImpressionLostToBudgetPercent":9.09,"ImpressionLostToRankAggPercent":77.27,"QualityScore":7,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":2,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833951765} +{"stream":"ad_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-12-18","AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Czech","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833373752} +{"stream":"ad_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-12-17","AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Bulgarian","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833394112} +{"stream":"budget_summary_report","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"CampaignId":531016227,"CampaignName":"Airbyte test","Date":"2023-12-18","MonthlyBudget":60.8,"DailySpend":2.06,"MonthToDateSpend":36.58},"emitted_at":1704833526694} +{"stream":"campaign_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"ConversionsQualified":0.0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current"},"emitted_at":1704833545467} +{"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":5,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"ConversionsQualified":0.0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":4,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current"},"emitted_at":1704833565296} +{"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":22,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":6,"LowQualityImpressionsPercent":21.43,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":34.92,"ImpressionLostToBudgetPercent":1.59,"ImpressionLostToRankAggPercent":63.49,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":5.26,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":10.2,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":68.0,"TopImpressionShareLostToBudgetPercent":0.0,"AbsoluteTopImpressionShareLostToRankPercent":89.8,"AbsoluteTopImpressionShareLostToBudgetPercent":0.0,"TopImpressionSharePercent":32.0,"AbsoluteTopImpressionRatePercent":22.73,"TopImpressionRatePercent":72.73,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":0.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833589146} +{"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":639,"Clicks":14,"Ctr":2.19,"AverageCpc":0.12,"Spend":1.74,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":30.0,"LowQualityImpressions":53,"LowQualityImpressionsPercent":7.66,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":13.57,"ImpressionLostToBudgetPercent":17.96,"ImpressionLostToRankAggPercent":68.47,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":6,"CampaignLabels":null,"ExactMatchImpressionSharePercent":17.65,"ClickSharePercent":1.28,"AbsoluteTopImpressionSharePercent":3.2,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":74.15,"TopImpressionShareLostToBudgetPercent":18.25,"AbsoluteTopImpressionShareLostToRankPercent":78.51,"AbsoluteTopImpressionShareLostToBudgetPercent":18.29,"TopImpressionSharePercent":7.6,"AbsoluteTopImpressionRatePercent":22.69,"TopImpressionRatePercent":53.99,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":2.72,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833610948} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-12-18","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Audience","DeviceType":"Computer","Language":"English","Network":"Audience","DeviceOS":"Unknown","TopVsOther":"Audience network","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0,"HistoricalQualityScore":5.0,"Impressions":6,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":0.66,"FirstPageBid":0.3,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833634746} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-12-17","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":0.66,"FirstPageBid":0.3,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833656374} +{"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-18","AccountNumber":"F149MJ18","Country":"Argentina","State":null,"MetroArea":null,"City":null,"ProximityTargetLocation":null,"Radius":"0","LocationType":"Physical location","MostSpecificLocation":"Argentina","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"8","BaseCampaignId":"531016227","Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833416620} +{"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-17","AccountNumber":"F149MJ18","Country":"United Arab Emirates","State":"Dubai","MetroArea":null,"City":"Dubai","ProximityTargetLocation":null,"Radius":"0","LocationType":"Physical location","MostSpecificLocation":"Dubai","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"154645","BaseCampaignId":"531016227","Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"0.00","AllConversionsQualified":"0.00","Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Audience","DeviceType":"Smartphone","Language":"English","Network":"Audience","DeviceOS":"Android","TopVsOther":"Audience network","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833479492} +{"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-12-18","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"Czech","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1704833673872} +{"stream":"age_gender_audience_report_weekly","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-12-17","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"Bulgarian","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1704833693674} +{"stream":"search_query_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","CampaignStatus":"Active","AdStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"airbyte","Keyword":"Airbyte","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84801135055370,"Network":"Microsoft sites and select traffic","TopVsOther":"Microsoft sites and select traffic - top","DeviceType":"Computer","DeviceOS":"Windows","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1704833715419} +{"stream":"search_query_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","CampaignStatus":"Active","AdStatus":"Active","Impressions":1,"Clicks":1,"Ctr":100.0,"AverageCpc":0.04,"Spend":0.04,"AveragePosition":0.0,"SearchQuery":"airbyte","Keyword":"Airbyte","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"Language":"Czech","KeywordId":84801135055370,"Network":"Microsoft sites and select traffic","TopVsOther":"Microsoft sites and select traffic - top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":40.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1704833737157} +{"stream":"user_location_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"Argentina","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"Spanish","City":null,"QueryIntentCountry":"Argentina","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":8,"QueryIntentLocationId":8,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1704833762092} +{"stream":"user_location_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"United Arab Emirates","State":"Dubai","MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Audience","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"English","City":"Dubai","QueryIntentCountry":"United Arab Emirates","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","Network":"Audience","TopVsOther":"Audience network","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":154645,"QueryIntentLocationId":218,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":0.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1704833830043} +{"stream":"account_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","Impressions":22,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":6,"LowQualityImpressionsPercent":21.43,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":34.92,"ImpressionLostToBudgetPercent":1.59,"ImpressionLostToRankAggPercent":63.49,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":5.26,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":10.2,"TopImpressionShareLostToRankPercent":68.0,"TopImpressionShareLostToBudgetPercent":0.0,"AbsoluteTopImpressionShareLostToRankPercent":89.8,"AbsoluteTopImpressionShareLostToBudgetPercent":0.0,"TopImpressionSharePercent":32.0,"AbsoluteTopImpressionRatePercent":22.73,"TopImpressionRatePercent":72.73,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":0.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833886551} +{"stream":"account_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","Impressions":639,"Clicks":14,"Ctr":2.19,"AverageCpc":0.12,"Spend":1.74,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":30.0,"LowQualityImpressions":53,"LowQualityImpressionsPercent":7.66,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":13.57,"ImpressionLostToBudgetPercent":17.96,"ImpressionLostToRankAggPercent":68.47,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":6,"ExactMatchImpressionSharePercent":17.65,"ClickSharePercent":1.28,"AbsoluteTopImpressionSharePercent":3.2,"TopImpressionShareLostToRankPercent":74.15,"TopImpressionShareLostToBudgetPercent":18.25,"AbsoluteTopImpressionShareLostToRankPercent":78.51,"AbsoluteTopImpressionShareLostToBudgetPercent":18.29,"TopImpressionSharePercent":7.6,"AbsoluteTopImpressionRatePercent":22.69,"TopImpressionRatePercent":53.99,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":2.72,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833908003} diff --git a/airbyte-integrations/connectors/source-bing-ads/main.py b/airbyte-integrations/connectors/source-bing-ads/main.py index 11548ad408a8..c05297b01ad8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/main.py +++ b/airbyte-integrations/connectors/source-bing-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bing_ads import SourceBingAds +from source_bing_ads.run import run if __name__ == "__main__": - source = SourceBingAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index d2ea910aac5b..d0bf4b329ece 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,13 +16,17 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 2.0.1 + dockerImageTag: 2.1.4 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads icon: bingads.svg license: MIT name: Bing Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-bing-ads registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-bing-ads/poetry.lock b/airbyte-integrations/connectors/source-bing-ads/poetry.lock new file mode 100644 index 000000000000..0200423f5f0f --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/poetry.lock @@ -0,0 +1,1221 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.60.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.60.1.tar.gz", hash = "sha256:fc5212b2962c1dc6aca9cc6f1c2000d7636b7509915846c126420c2b0c814317"}, + {file = "airbyte_cdk-0.60.1-py3-none-any.whl", hash = "sha256:94b33c0f6851d1e2546eac3cec54c67489239595d9e0a496ef57c3fc808e89e3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bingads" +version = "13.0.18.1" +description = "A library to make working with the Bing Ads APIs and bulk services easy" +optional = false +python-versions = "*" +files = [ + {file = "bingads-13.0.18.1.tar.gz", hash = "sha256:7e861ac0d959d374519043c32d04db1fc8dd97f3a0566ea6bdca76bd60784f86"}, +] + +[package.dependencies] +requests = "*" +suds-community = ">=1.1.0" + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cached-property" +version = "1.5.2" +description = "A decorator for caching properties in classes." +optional = false +python-versions = "*" +files = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.2.0" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "suds-community" +version = "1.1.2" +description = "Lightweight SOAP client (community fork)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "suds-community-1.1.2.tar.gz", hash = "sha256:883b4173ad23e7b20e9779ac7238b06140c50d7852afd5dc49dad1ea5f5a3d08"}, + {file = "suds_community-1.1.2-py3-none-any.whl", hash = "sha256:18a0176bf4f5945e133024faa57c35c3d7320e02f6b84bfe95baa6ddf5e05cec"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "7ffb684bece88c37d64d09e4ce2e340727c0ad7811c8d9ffe988779870a9204c" diff --git a/airbyte-integrations/connectors/source-bing-ads/pyproject.toml b/airbyte-integrations/connectors/source-bing-ads/pyproject.toml new file mode 100644 index 000000000000..17bb4f851c66 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.1.4" +name = "source-bing-ads" +description = "Source implementation for Bing Ads." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/bing-ads" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_bing_ads" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +bingads = "==13.0.18.1" +pandas = "==2.2.0" +urllib3 = "==1.26.18" +airbyte-cdk = "==0.60.1" +cached-property = "==1.5.2" + +[tool.poetry.scripts] +source-bing-ads = "source_bing_ads.run:run" + +[tool.poetry.group.dev.dependencies] +freezegun = "^1.4.0" +pytest-mock = "^3.6.1" +pytest = "^6.1" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py deleted file mode 100644 index 5c326b2d9aad..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.17", "urllib3<2.0", "pandas"] - -TEST_REQUIREMENTS = [ - "freezegun", - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", -] - -setup( - name="source_bing_ads", - description="Source implementation for Bing Ads.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py index 39f1eec1957e..7294f453deb1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -162,6 +162,11 @@ class Accounts(BingAdsStream): # maximum page size page_size_limit: int = 1000 + def __init__(self, client: Client, config: Mapping[str, Any]) -> None: + super().__init__(client, config) + self._account_names = config.get("account_names", []) + self._unique_account_ids = set() + def next_page_token(self, response: sudsobject.Object, current_page_token: Optional[int]) -> Optional[Mapping[str, Any]]: current_page_token = current_page_token or 0 if response is not None and hasattr(response, self.data_field): @@ -169,30 +174,46 @@ def next_page_token(self, response: sudsobject.Object, current_page_token: Optio else: return None + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + user_id_predicate = { + "Field": "UserId", + "Operator": "Equals", + "Value": self._user_id, + } + if self._account_names: + for account_config in self._account_names: + account_name_predicate = {"Field": "AccountName", "Operator": account_config["operator"], "Value": account_config["name"]} + + yield {"predicates": {"Predicate": [user_id_predicate, account_name_predicate]}} + else: + yield {"predicates": {"Predicate": [user_id_predicate]}} + def request_params( self, next_page_token: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, **kwargs: Mapping[str, Any], ) -> MutableMapping[str, Any]: - predicates = { - "Predicate": [ - { - "Field": "UserId", - "Operator": "Equals", - "Value": self._user_id, - } - ] - } - paging = self._service.factory.create("ns5:Paging") paging.Index = next_page_token or 0 paging.Size = self.page_size_limit return { "PageInfo": paging, - "Predicates": predicates, + "Predicates": stream_slice["predicates"], "ReturnAdditionalFields": self.additional_fields, } + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + records = self.client.asdict(response)[self.data_field] + for record in records: + if record["Id"] not in self._unique_account_ids: + self._unique_account_ids.add(record["Id"]) + yield record + class Campaigns(BingAdsCampaignManagementStream): """ @@ -241,8 +262,10 @@ def stream_slices( self, **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} class AdGroups(BingAdsCampaignManagementStream): @@ -274,11 +297,13 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: campaigns = Campaigns(self.client, self.config) - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records( - sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - ): - yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + for campaign in campaigns.read_records( + sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + ): + yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} class Ads(BingAdsCampaignManagementStream): diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py index 34d13b2627ca..440b0a3607a0 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py @@ -49,8 +49,10 @@ def stream_slices( self, **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} @property def state(self) -> Mapping[str, Any]: @@ -58,11 +60,17 @@ def state(self) -> Mapping[str, Any]: @state.setter def state(self, value: Mapping[str, Any]): - current_state_value = self._state.get(str(value["Account Id"]), {}).get(self.cursor_field, "") - if value[self.cursor_field]: + # if key 'Account Id' exists, so we receive a record that should be parsed to state + # otherwise state object from connection state was received + account_id = value.get("Account Id") + + if account_id and value[self.cursor_field]: + current_state_value = self._state.get(str(value["Account Id"]), {}).get(self.cursor_field, "") record_state_value = transform_bulk_datetime_format_to_rfc_3339(value[self.cursor_field]) new_state_value = max(current_state_value, record_state_value) self._state.update({str(value["Account Id"]): {self.cursor_field: new_state_value}}) + else: + self._state.update(value) def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None) -> Optional[pendulum.DateTime]: """ diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py index 5a0d1c5818a3..e1b98a4ec17c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union from urllib.parse import urlparse import _csv @@ -19,6 +19,7 @@ from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord from bingads.v13.reporting import ReportingDownloadParameters +from cached_property import cached_property from source_bing_ads.base_streams import Accounts, BingAdsStream from source_bing_ads.utils import transform_date_format_to_rfc_3339, transform_report_hourly_datetime_format_to_rfc_3339 from suds import WebFault, sudsobject @@ -31,7 +32,6 @@ class HourlyReportTransformerMixin: @transformer.registerCustomTransform def custom_transform_datetime_rfc3339(original_value, field_schema): if original_value and "format" in field_schema and field_schema["format"] == "date-time": - print(original_value) transformed_value = transform_report_hourly_datetime_format_to_rfc_3339(original_value) return transformed_value return original_value @@ -104,10 +104,14 @@ def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, Non return None if "%" in value: value = value.replace("%", "") - if value and set(self.get_json_schema()["properties"].get(column, {}).get("type")) & {"integer", "number"}: + if value and column in self._get_schema_numeric_properties: value = value.replace(",", "") return value + @cached_property + def _get_schema_numeric_properties(self) -> Set[str]: + return set(k for k, v in self.get_json_schema()["properties"].items() if set(v.get("type")) & {"integer", "number"}) + def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> sudsobject.Object: """ Creates XML Date object based on datetime. @@ -227,12 +231,16 @@ def get_report_record_timestamp(self, datestring: str) -> str: ) def stream_slices( - self, - **kwargs: Mapping[str, Any], + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for period in self.default_time_periods: - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + if self.get_start_date(stream_state, account["Id"]): # if start date is not provided default time periods will be used + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + else: + for period in self.default_time_periods: + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} class BingAdsReportingServicePerformanceStream(BingAdsReportingServiceStream, ABC): diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py new file mode 100644 index 000000000000..3dd73c2fec5e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bing_ads import SourceBingAds + + +def run(): + source = SourceBingAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index c18fcf5a913f..37c2b9bc5d2b 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -75,7 +75,10 @@ class SourceBingAds(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: client = Client(**config) - account_ids = {str(account["Id"]) for account in Accounts(client, config).read_records(SyncMode.full_refresh)} + accounts = Accounts(client, config) + account_ids = set() + for _slice in accounts.stream_slices(): + account_ids.update({str(account["Id"]) for account in accounts.read_records(SyncMode.full_refresh, _slice)}) self.validate_custom_reposts(config, client) if account_ids: return True, None diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 4461a72fd7dd..f7f8e586223e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -48,12 +48,36 @@ "airbyte_secret": true, "order": 4 }, + "account_names": { + "title": "Account Names Predicates", + "description": "Predicates that will be used to sync data by specific accounts.", + "type": "array", + "order": 5, + "items": { + "description": "Account Names Predicates Config.", + "type": "object", + "properties": { + "operator": { + "title": "Operator", + "description": "An Operator that will be used to filter accounts. The Contains predicate has features for matching words, matching inflectional forms of words, searching using wildcard characters, and searching using proximity. The Equals is used to return all rows where account name is equal(=) to the string that you provided", + "type": "string", + "enum": ["Contains", "Equals"] + }, + "name": { + "title": "Account Name", + "description": "Account Name is a string value for comparing with the specified predicate.", + "type": "string" + } + }, + "required": ["operator", "name"] + } + }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format. If not set, data from previous and current calendar year will be replicated.", - "order": 5 + "order": 6 }, "lookback_window": { "title": "Lookback window", @@ -62,12 +86,12 @@ "default": 0, "minimum": 0, "maximum": 90, - "order": 6 + "order": 7 }, "custom_reports": { "title": "Custom Reports", "description": "You can add your Custom Bing Ads report by creating one.", - "order": 7, + "order": 8, "type": "array", "items": { "title": "Custom Report Config", diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py index fec097978fff..4eb197298b2d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py @@ -19,6 +19,32 @@ def config_fixture(): } +@pytest.fixture(name="config_without_start_date") +def config_without_start_date_fixture(): + """Generates streams settings from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "lookback_window": 0, + } + + +@pytest.fixture(name="config_with_account_names") +def config_with_account_names_fixture(): + """Generates streams settings from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "account_names": [{"operator": "Equals", "name": "airbyte"}, {"operator": "Contains", "name": "demo"}], + "lookback_window": 0, + } + + @pytest.fixture(name="config_with_custom_reports") def config_with_custom_reports_fixture(): """Generates streams settings with custom reports from a config file""" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py index 7ded33ab23fe..c69f77e9bea2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -130,6 +130,28 @@ def test_bulk_stream_stream_state(mocked_client, config): assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} stream.state = {"Account Id": "some_account_id", "Modified Time": "05/25/2023 18:00:14.970"} assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} + # stream state saved to connection state + stream.state = { + "120342748234": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "27364572345": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "732645723": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "837563864": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + } + } + assert stream.state == { + "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "27364572345": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "732645723": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "837563864": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}, + } @patch.object(source_bing_ads.source, "Client") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index e00a6b67b6f7..9c89f47cdca3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -397,11 +397,12 @@ def test_custom_report_get_report_record_timestamp(mocked_client, config_with_cu @patch.object(source_bing_ads.source, "Client") -def test_account_performance_report_monthly_stream_slices(mocked_client, config): - account_performance_report_monthly = AccountPerformanceReportMonthly(mocked_client, config) +def test_account_performance_report_monthly_stream_slices(mocked_client, config_without_start_date): + mocked_client.reports_start_date = None + account_performance_report_monthly = AccountPerformanceReportMonthly(mocked_client, config_without_start_date) accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): - stream_slice = list(account_performance_report_monthly.stream_slices()) + stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) assert stream_slice == [ {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, @@ -410,6 +411,18 @@ def test_account_performance_report_monthly_stream_slices(mocked_client, config) ] +@patch.object(source_bing_ads.source, "Client") +def test_account_performance_report_monthly_stream_slices_no_time_period(mocked_client, config): + account_performance_report_monthly = AccountPerformanceReportMonthly(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) + assert stream_slice == [ + {'account_id': 180519267, 'customer_id': 100}, + {'account_id': 180278106, 'customer_id': 200} + ] + + @pytest.mark.parametrize( "aggregation", [ @@ -419,11 +432,12 @@ def test_account_performance_report_monthly_stream_slices(mocked_client, config) ) @patch.object(source_bing_ads.source, "Client") def test_custom_performance_report_no_last_year_stream_slices(mocked_client, config_with_custom_reports, aggregation): + mocked_client.reports_start_date = None # in case of start date time period won't be used in request params custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] custom_report.report_aggregation = aggregation accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): - stream_slice = list(custom_report.stream_slices()) + stream_slice = list(custom_report.stream_slices(sync_mode=SyncMode.full_refresh)) assert stream_slice == [ {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index 7d89ff112b1a..210ccf1031be 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -176,7 +176,7 @@ def test_ads_stream_slices(mocked_client, config): @pytest.mark.parametrize( ("stream", "stream_slice"), ( - (Accounts, {}), + (Accounts, {"predicates": {"Predicate": [{"Field": "UserId", "Operator": "Equals", "Value": "131313131"},]}}), (AdGroups, {"campaign_id": "campaign_id"}), (Ads, {"ad_group_id": "ad_group_id"}), (Campaigns, {"account_id": "account_id"}), @@ -204,3 +204,9 @@ def test_transform(mocked_client, config): "EditorialStatus": "ActiveLimited", "FinalAppUrls": None, } + + +@patch.object(source_bing_ads.source, "Client") +def test_check_connection_with_accounts_names_config(mocked_client, config_with_account_names, logger_mock): + with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): + assert SourceBingAds().check_connection(logger_mock, config=config_with_account_names) == (True, None) diff --git a/airbyte-integrations/connectors/source-braintree/main.py b/airbyte-integrations/connectors/source-braintree/main.py index 3b3c6039f9fe..d4ae7bec5223 100644 --- a/airbyte-integrations/connectors/source-braintree/main.py +++ b/airbyte-integrations/connectors/source-braintree/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_braintree import SourceBraintree +from source_braintree.run import run if __name__ == "__main__": - source = SourceBraintree() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-braintree/metadata.yaml b/airbyte-integrations/connectors/source-braintree/metadata.yaml index d2f0ca52f2bb..5d948eee41b6 100644 --- a/airbyte-integrations/connectors/source-braintree/metadata.yaml +++ b/airbyte-integrations/connectors/source-braintree/metadata.yaml @@ -12,6 +12,10 @@ data: icon: braintree.svg license: MIT name: Braintree + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-braintree registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-braintree/setup.py b/airbyte-integrations/connectors/source-braintree/setup.py index 0a6f5d53752e..cd584b74e699 100644 --- a/airbyte-integrations/connectors/source-braintree/setup.py +++ b/airbyte-integrations/connectors/source-braintree/setup.py @@ -13,13 +13,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-braintree=source_braintree.run:run", + ], + }, name="source_braintree_no_code", description="Source implementation for Braintree No Code.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/run.py b/airbyte-integrations/connectors/source-braintree/source_braintree/run.py new file mode 100644 index 000000000000..df776d3952c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_braintree import SourceBraintree + + +def run(): + source = SourceBraintree() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-braze/main.py b/airbyte-integrations/connectors/source-braze/main.py index 15453641a7f0..723116b28098 100644 --- a/airbyte-integrations/connectors/source-braze/main.py +++ b/airbyte-integrations/connectors/source-braze/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_braze import SourceBraze +from source_braze.run import run if __name__ == "__main__": - source = SourceBraze() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-braze/metadata.yaml b/airbyte-integrations/connectors/source-braze/metadata.yaml index bde782b0dc7f..e1fdcf539fd3 100644 --- a/airbyte-integrations/connectors/source-braze/metadata.yaml +++ b/airbyte-integrations/connectors/source-braze/metadata.yaml @@ -12,6 +12,10 @@ data: icon: braze.svg license: MIT name: Braze + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-braze registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index aade45358b41..43f778382f75 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-braze=source_braze.run:run", + ], + }, name="source_braze", description="Source implementation for Braze.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-braze/source_braze/run.py b/airbyte-integrations/connectors/source-braze/source_braze/run.py new file mode 100644 index 000000000000..645b7a31df24 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_braze import SourceBraze + + +def run(): + source = SourceBraze() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-breezometer/main.py b/airbyte-integrations/connectors/source-breezometer/main.py index e14371ac83bb..3fc185eec93c 100644 --- a/airbyte-integrations/connectors/source-breezometer/main.py +++ b/airbyte-integrations/connectors/source-breezometer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_breezometer import SourceBreezometer +from source_breezometer.run import run if __name__ == "__main__": - source = SourceBreezometer() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-breezometer/metadata.yaml b/airbyte-integrations/connectors/source-breezometer/metadata.yaml index b325179a45ca..16796af3c94f 100644 --- a/airbyte-integrations/connectors/source-breezometer/metadata.yaml +++ b/airbyte-integrations/connectors/source-breezometer/metadata.yaml @@ -8,6 +8,10 @@ data: icon: breezometer.svg license: MIT name: Breezometer + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-breezometer registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-breezometer/setup.py b/airbyte-integrations/connectors/source-breezometer/setup.py index 61bd2c525610..fc9d1e85cd20 100644 --- a/airbyte-integrations/connectors/source-breezometer/setup.py +++ b/airbyte-integrations/connectors/source-breezometer/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-breezometer=source_breezometer.run:run", + ], + }, name="source_breezometer", description="Source implementation for Breezometer.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py b/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py new file mode 100644 index 000000000000..6855613e5586 --- /dev/null +++ b/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_breezometer import SourceBreezometer + + +def run(): + source = SourceBreezometer() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-callrail/main.py b/airbyte-integrations/connectors/source-callrail/main.py index cb3607679473..d5651af4f615 100644 --- a/airbyte-integrations/connectors/source-callrail/main.py +++ b/airbyte-integrations/connectors/source-callrail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_callrail import SourceCallrail +from source_callrail.run import run if __name__ == "__main__": - source = SourceCallrail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-callrail/metadata.yaml b/airbyte-integrations/connectors/source-callrail/metadata.yaml index d876058aa9c0..f0e46fadc35a 100644 --- a/airbyte-integrations/connectors/source-callrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-callrail/metadata.yaml @@ -8,6 +8,10 @@ data: icon: callrail.svg license: MIT name: CallRail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-callrail registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-callrail/setup.py b/airbyte-integrations/connectors/source-callrail/setup.py index c744bbc957ff..e6d0d00d5d46 100644 --- a/airbyte-integrations/connectors/source-callrail/setup.py +++ b/airbyte-integrations/connectors/source-callrail/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-callrail=source_callrail.run:run", + ], + }, name="source_callrail", description="Source implementation for Callrail.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-callrail/source_callrail/run.py b/airbyte-integrations/connectors/source-callrail/source_callrail/run.py new file mode 100644 index 000000000000..d271cf7a502b --- /dev/null +++ b/airbyte-integrations/connectors/source-callrail/source_callrail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_callrail import SourceCallrail + + +def run(): + source = SourceCallrail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-captain-data/main.py b/airbyte-integrations/connectors/source-captain-data/main.py index 765d967fad15..50a0dfd7944d 100644 --- a/airbyte-integrations/connectors/source-captain-data/main.py +++ b/airbyte-integrations/connectors/source-captain-data/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_captain_data import SourceCaptainData +from source_captain_data.run import run if __name__ == "__main__": - source = SourceCaptainData() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-captain-data/metadata.yaml b/airbyte-integrations/connectors/source-captain-data/metadata.yaml index 715cb3f3c9c1..e5f9108699ad 100644 --- a/airbyte-integrations/connectors/source-captain-data/metadata.yaml +++ b/airbyte-integrations/connectors/source-captain-data/metadata.yaml @@ -8,6 +8,10 @@ data: icon: captain-data.svg license: MIT name: Captain Data + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-captain-data registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-captain-data/setup.py b/airbyte-integrations/connectors/source-captain-data/setup.py index f6121791ddd8..cf4b2f7f5fc7 100644 --- a/airbyte-integrations/connectors/source-captain-data/setup.py +++ b/airbyte-integrations/connectors/source-captain-data/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-captain-data=source_captain_data.run:run", + ], + }, name="source_captain_data", description="Source implementation for Captain Data.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py new file mode 100644 index 000000000000..5b25fd1be5a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_captain_data import SourceCaptainData + + +def run(): + source = SourceCaptainData() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-cart/main.py b/airbyte-integrations/connectors/source-cart/main.py index b294ae4e2c15..c7f69c914848 100644 --- a/airbyte-integrations/connectors/source-cart/main.py +++ b/airbyte-integrations/connectors/source-cart/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_cart import SourceCart +from source_cart.run import run if __name__ == "__main__": - source = SourceCart() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-cart/metadata.yaml b/airbyte-integrations/connectors/source-cart/metadata.yaml index d73ebd080df9..113417c673b0 100644 --- a/airbyte-integrations/connectors/source-cart/metadata.yaml +++ b/airbyte-integrations/connectors/source-cart/metadata.yaml @@ -8,6 +8,10 @@ data: icon: cart.svg license: MIT name: Cart.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-cart registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-cart/setup.py b/airbyte-integrations/connectors/source-cart/setup.py index 69df33c757d3..c0ee59c3d047 100644 --- a/airbyte-integrations/connectors/source-cart/setup.py +++ b/airbyte-integrations/connectors/source-cart/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-cart=source_cart.run:run", + ], + }, name="source_cart", description="Source implementation for Cart.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-cart/source_cart/run.py b/airbyte-integrations/connectors/source-cart/source_cart/run.py new file mode 100644 index 000000000000..7f639ab6c694 --- /dev/null +++ b/airbyte-integrations/connectors/source-cart/source_cart/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_cart import SourceCart + + +def run(): + source = SourceCart() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chargebee/README.md b/airbyte-integrations/connectors/source-chargebee/README.md index 4819ef2b1506..5169efb56baa 100644 --- a/airbyte-integrations/connectors/source-chargebee/README.md +++ b/airbyte-integrations/connectors/source-chargebee/README.md @@ -1,86 +1,55 @@ -# Chargebee Source +# Chargebee source connector -This is the repository for the Chargebee configuration based source connector. + +This is the repository for the Chargebee source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/chargebee). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/chargebee) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargebee/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source chargebee test creds` -and place them into `secrets/config.json`. - -### Locally running the connector docker image - - +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Installing the connector +From this connector directory, run: ```bash -airbyte-ci connectors --name=source-chargebee build +poetry install --with dev ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-chargebee:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/chargebee) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargebee/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +### Locally running the connector +``` +poetry run source-chargebee spec +poetry run source-chargebee check --config secrets/config.json +poetry run source-chargebee discover --config secrets/config.json +poetry run source-chargebee read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-chargebee:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests ``` -Please use this as an example. This is not optimized. -2. Build your image: +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-chargebee:dev . -# Running the spec command against your patched connector -docker run airbyte/source-chargebee:dev spec +airbyte-ci connectors --name=source-chargebee build ``` -#### Run + +An image will be available on your host with the tag `airbyte/source-chargebee:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-chargebee:dev spec @@ -89,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargebee:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chargebee:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-chargebee test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-chargebee test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/chargebee.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/chargebee.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml index 095e361ba8ed..e7ce8a221fb0 100644 --- a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml @@ -23,48 +23,41 @@ acceptance_tests: timeout_seconds: 1200 empty_streams: - name: "addon" - bypass_reason: "Not permitted for this site" + bypass_reason: "Not available for Product Catalog 2.0 sites." - name: "plan" - bypass_reason: "Not permitted for this site" + bypass_reason: "Not available for Product Catalog 2.0 sites." - name: "virtual_bank_account" bypass_reason: "Cannot populate with test data" - - name: "subscription" - bypass_reason: "Unstable data. Field current_term_start updated daily" - - name: "customer" - bypass_reason: "Unstable data. Depends on subscription" - - name: "invoice" - bypass_reason: "Unstable data. Depends on subscription" - - name: "credit_note" - bypass_reason: "Unstable data. Depends on subscription" - name: "event" - bypass_reason: "Unstable data. Depends on subscription" - - name: "unbilled_charge" - bypass_reason: "Empty stream. Unstable data" - - name: "hosted_page" - bypass_reason: "Empty stream. Unstable data" + bypass_reason: "Unstable data. Test data is not persistent." + - name: "site_migration_detail" + bypass_reason: "Cannnot populate with test data." + - name: "customer" + bypass_reason: "To be Tested with integration tests." + - name: "subscription" + bypass_reason: "To be Tested with integration tests." + - name: "coupon" + bypass_reason: "To be Tested with integration tests." expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no exact_order: no extra_records: yes - fail_on_extra_columns: false + fail_on_extra_columns: true incremental: - # tests: - # - config_path: "secrets/config.json" - # timeout_seconds: 2400 - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state: - # future_state_path: "integration_tests/future_state.json" - # missing_streams: - # - name: attached_item - # bypass_reason: "This stream is Full-Refresh only" - # - name: contact - # bypass_reason: "This stream is Full-Refresh only" - # - name: quote_line_group - # bypass_reason: "This stream is Full-Refresh only" - bypass_reason: > - "Incrremental tests are disabled until CAT works with cursor data-types directly, - relatated slack thread: https://airbyte-globallogic.slack.com/archives/C02U9R3AF37/p1690810513681859" + tests: + - config_path: "secrets/config.json" + timeout_seconds: 2400 + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/future_state.json" + missing_streams: + - name: contact + bypass_reason: "This stream is Full-Refresh only" + - name: quote_line_group + bypass_reason: "This stream is Full-Refresh only" + - name: attached_item + bypass_reason: "This stream is Full-Refresh only" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json index 950c314639f2..8adc9742557e 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json @@ -13,44 +13,6 @@ "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, - { - "stream": { - "name": "event", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["occurred_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["occurred_at"] - }, - { - "stream": { - "name": "customer", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "contact", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, { "stream": { "name": "invoice", @@ -107,13 +69,11 @@ "stream": { "name": "attached_item", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite" }, { "stream": { @@ -147,7 +107,7 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], + "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -160,6 +120,19 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, + "source_defined_primary_key": [["id"]], + "default_cursor_field": ["created_at"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "transaction", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, @@ -169,7 +142,7 @@ }, { "stream": { - "name": "coupon", + "name": "credit_note", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -182,12 +155,12 @@ }, { "stream": { - "name": "transaction", + "name": "unbilled_charge", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "default_cursor_field": ["updated_at"] }, "sync_mode": "incremental", "destination_sync_mode": "append", @@ -195,7 +168,7 @@ }, { "stream": { - "name": "credit_note", + "name": "quote", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -208,20 +181,30 @@ }, { "stream": { - "name": "unbilled_charge", + "name": "quote_line_group", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "comment", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], + "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["updated_at"] + "cursor_field": ["created_at"] }, { "stream": { - "name": "quote", + "name": "item_family", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -234,15 +217,16 @@ }, { "stream": { - "name": "quote_line_group", + "name": "differential_price", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl index fd515d457d6c..1ef91d437747 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl @@ -1,33 +1,51 @@ -{"stream":"contact","data":{"id":"0000002","first_name":"User2","last_name":"Sample","email":"user2.sample.airbyte@gmail.com","phone":"+13335556789","label":"Tag2","enabled":true,"send_account_email":true,"send_billing_email":true,"object":"contact","custom_fields":[]},"emitted_at":1676569185767} -{"stream":"contact","data":{"id":"Test 1","first_name":"Sample Name 1","last_name":"Sample Lastname 1","email":"name1@example.com","enabled":true,"send_account_email":false,"send_billing_email":false,"object":"contact","custom_fields":[]},"emitted_at":1676569186037} -{"stream":"contact","data":{"id":"Test Contact 2","first_name":"Sample Name Two","last_name":"Sample Lastname 2","email":"name2@example.com","phone":"+13888433888","enabled":true,"send_account_email":false,"send_billing_email":false,"object":"contact","custom_fields":[]},"emitted_at":1676569186337} -{"stream":"order","data":{"id":"1","document_number":"lol1","invoice_id":"24","subscription_id":"6olOsTTHieWUY9","customer_id":"cbdemo_tyler","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674036524,"shipping_date":1674036524,"created_by":"Auto generated by system","tax":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"rounding_adjustement":0,"paid_on":1674036524,"exchange_rate":1,"created_at":1674036525,"updated_at":1674036525,"is_resent":false,"resource_version":1674036525755,"deleted":false,"object":"order","discount":0,"sub_total":1000,"order_line_items":[{"id":"o_li169lB6TTHiez02Fb4","invoice_id":"24","invoice_line_item_id":"li_6olOsTTHieX6YB","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":1000,"tax_amount":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":0,"item_level_discount_amount":0,"description":"Test Plan 1","entity_type":"plan_item_price"}],"total":1000,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"billing_address":{"first_name":"Tyler","last_name":"Durden","company":"Iselectrics","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847508} -{"stream":"order","data":{"id":"2","document_number":"lol2","invoice_id":"25","subscription_id":"AzZTZgTTHixMHV3","customer_id":"cbdemo_richard","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674036596,"shipping_date":1674036596,"created_by":"Auto generated by system","tax":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"rounding_adjustement":0,"paid_on":1674036596,"exchange_rate":1,"created_at":1674036599,"updated_at":1674036684,"is_resent":false,"resource_version":1674036684213,"deleted":false,"object":"order","discount":0,"sub_total":1000,"order_line_items":[{"id":"o_li16CQyCTTHiy9912Tu","invoice_id":"25","invoice_line_item_id":"li_AzZTZgTTHixMhV5","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":1000,"tax_amount":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":0,"item_level_discount_amount":0,"description":"Test Plan 1","entity_type":"plan_item_price"}],"total":1000,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"shipping_address":{"first_name":"Sample Name 1","last_name":"Sample Lastname 1","email":"name1@example.com","company":"Semiconductors","phone":"+1 382 846 3883","line1":"Ms Ninette Franck","line2":"4381","city":"San Francisco","state_code":"CA","state":"California","country":"US","zip":"94114","validation_status":"not_validated","object":"shipping_address"},"billing_address":{"first_name":"Richard","last_name":"Hendricks","company":"Zencorporation","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847512} -{"stream":"order","data":{"id":"3","document_number":"lol3","invoice_id":"26","subscription_id":"AzZTZgTTHmX8Gc1","customer_id":"cbdemo_simon","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674037448,"shipping_date":1674037448,"created_by":"Auto generated by system","tax":0,"amount_paid":700,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":700,"rounding_adjustement":0,"paid_on":1674037448,"exchange_rate":1,"created_at":1674037452,"updated_at":1674037452,"is_resent":false,"resource_version":1674037452271,"deleted":false,"object":"order","discount":300,"sub_total":700,"order_line_items":[{"id":"o_liAzZZMnTTHmY0s1O7g","invoice_id":"26","invoice_line_item_id":"li_AzZTZgTTHmX93c3","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":700,"tax_amount":0,"amount_paid":700,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":700,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":300,"item_level_discount_amount":300,"description":"Test Plan 1","entity_type":"plan_item_price"}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"li_AzZTZgTTHmX93c3","discount_type":"item_level_coupon","discount_amount":300,"coupon_id":"cbdemo_launchoffer","entity_id":"cbdemo_launchoffer"}],"total":700,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"billing_address":{"first_name":"Simon","last_name":"Masrani","company":"Openlane Ltd","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847517} +{"stream": "contact", "data": {"id": "0000002", "first_name": "User2", "last_name": "Sample", "email": "user2.sample.airbyte@gmail.com", "phone": "+13335556789", "label": "Tag2", "enabled": true, "send_account_email": true, "send_billing_email": true, "object": "contact", "customer_id": "Azz5jBTTJ96UqlvE", "custom_fields": []}, "emitted_at": 1706028645460} +{"stream": "contact", "data": {"id": "Test 1", "first_name": "Sample Name 1", "last_name": "Sample Lastname 1", "email": "name1@example.com", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact", "customer_id": "cbdemo_richard", "custom_fields": []}, "emitted_at": 1706028645946} +{"stream": "contact", "data": {"id": "Test Contact 2", "first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "name2@example.com", "phone": "+13888433888", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact", "customer_id": "Test-Custome-1", "custom_fields": []}, "emitted_at": 1706028645251} +{"stream": "order", "data": {"id": "1", "document_number": "lol1", "invoice_id": "24", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036524, "shipping_date": 1674036524, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036524, "exchange_rate": 1.0, "created_at": 1674036525, "updated_at": 1674036525, "is_resent": false, "resource_version": 1674036525755, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li169lB6TTHiez02Fb4", "invoice_id": "24", "invoice_line_item_id": "li_6olOsTTHieX6YB", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216053} +{"stream": "order", "data": {"id": "2", "document_number": "lol2", "invoice_id": "25", "subscription_id": "AzZTZgTTHixMHV3", "customer_id": "cbdemo_richard", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036596, "shipping_date": 1674036596, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036596, "exchange_rate": 1.0, "created_at": 1674036599, "updated_at": 1674036684, "is_resent": false, "resource_version": 1674036684213, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li16CQyCTTHiy9912Tu", "invoice_id": "25", "invoice_line_item_id": "li_AzZTZgTTHixMhV5", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "shipping_address": {"first_name": "Sample Name 1", "last_name": "Sample Lastname 1", "email": "name1@example.com", "company": "Semiconductors", "phone": "+1 382 846 3883", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "shipping_address"}, "billing_address": {"first_name": "Richard", "last_name": "Hendricks", "company": "Zencorporation", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216060} +{"stream": "order", "data": {"id": "3", "document_number": "lol3", "invoice_id": "26", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674037448, "shipping_date": 1674037448, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "rounding_adjustement": 0, "paid_on": 1674037448, "exchange_rate": 1.0, "created_at": 1674037452, "updated_at": 1674037452, "is_resent": false, "resource_version": 1674037452271, "deleted": false, "object": "order", "discount": 300, "sub_total": 700, "order_line_items": [{"id": "o_liAzZZMnTTHmY0s1O7g", "invoice_id": "26", "invoice_line_item_id": "li_AzZTZgTTHmX93c3", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 700, "tax_amount": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 300, "item_level_discount_amount": 300, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "li_AzZTZgTTHmX93c3", "discount_type": "item_level_coupon", "discount_amount": 300, "coupon_id": "cbdemo_launchoffer", "entity_id": "cbdemo_launchoffer"}], "total": 700, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216066} {"stream": "item", "data": {"id": "cbdemo_advanced", "name": "Advanced", "external_name": "Advanced", "description": "Uncover hidden insights and carry out deeper analytics for your enterprise with this advanced plan.", "status": "active", "resource_version": 1674035640445, "updated_at": 1674035640, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136879} {"stream": "item", "data": {"id": "cbdemo_basic", "name": "Basic", "external_name": "Basic", "description": "Starter plan for all your basic reporting requirements.", "status": "active", "resource_version": 1674035673162, "updated_at": 1674035673, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136891} {"stream": "item", "data": {"id": "cbdemo_intermediary", "name": "Intermediary", "external_name": "Intermediary", "description": "Smart plan with the right mix of basic and slightly advanced reporting tools.", "status": "active", "resource_version": 1674035686971, "updated_at": 1674035686, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136900} -{"stream":"attached_item","data":{"id":"e49c6ed7-9f1b-4c79-9235-549ce8ae9a1f","parent_item_id":"cbdemo_advanced","item_id":"cbdemo_setup_charge","status":"active","charge_on_event":"subscription_trial_start","charge_once":false,"created_at":1674032839,"resource_version":1674032839573,"updated_at":1674032839,"object":"attached_item","custom_fields":[]},"emitted_at":1676569205846} -{"stream":"attached_item","data":{"id":"25976ccf-8e44-4fce-8eab-2a1658eb0a2b","parent_item_id":"cbdemo_advanced","item_id":"cbdemo_analytics_additionalusers","type":"mandatory","status":"active","quantity":1,"created_at":1674032827,"resource_version":1674032827801,"updated_at":1674032827,"object":"attached_item","custom_fields":[]},"emitted_at":1676569205849} -{"stream":"attached_item","data":{"id":"69b451b1-e00a-4522-ab6f-027586d24b85","parent_item_id":"cbdemo_basic","item_id":"cbdemo_setup_charge","status":"active","charge_on_event":"subscription_creation","charge_once":false,"created_at":1674032880,"resource_version":1674032880261,"updated_at":1674032880,"object":"attached_item","custom_fields":[]},"emitted_at":1676569206020} +{"stream": "attached_item", "data": {"id": "e49c6ed7-9f1b-4c79-9235-549ce8ae9a1f", "parent_item_id": "cbdemo_advanced", "item_id": "cbdemo_setup_charge", "status": "active", "charge_on_event": "subscription_trial_start", "charge_once": false, "created_at": 1674032839, "resource_version": 1674032839573, "updated_at": 1674032839, "object": "attached_item", "custom_fields": []}, "emitted_at": 1676569205846} +{"stream": "attached_item", "data": {"id": "25976ccf-8e44-4fce-8eab-2a1658eb0a2b", "parent_item_id": "cbdemo_advanced", "item_id": "cbdemo_analytics_additionalusers", "type": "mandatory", "status": "active", "quantity": 1, "created_at": 1674032827, "resource_version": 1674032827801, "updated_at": 1674032827, "object": "attached_item", "custom_fields": []}, "emitted_at": 1676569205849} +{"stream": "attached_item", "data": {"id": "69b451b1-e00a-4522-ab6f-027586d24b85", "parent_item_id": "cbdemo_basic", "item_id": "cbdemo_setup_charge", "status": "active", "charge_on_event": "subscription_creation", "charge_once": false, "created_at": 1674032880, "resource_version": 1674032880261, "updated_at": 1674032880, "object": "attached_item", "custom_fields": []}, "emitted_at": 1676569206020} {"stream": "item_price", "data": {"id": "Test-Plan-1-USD-Daily", "name": "Test Plan 1 USD Daily", "item_family_id": "cbdemo_pf_analytics", "item_id": "Test-Plan-1", "description": "Test", "status": "active", "external_name": "Test Plan 1", "pricing_model": "flat_fee", "price": 1000, "period": 1, "currency_code": "USD", "period_unit": "day", "shipping_period": 1, "shipping_period_unit": "day", "free_quantity": 0, "channel": "web", "resource_version": 1674036400224, "updated_at": 1674036400, "created_at": 1674036400, "invoice_notes": "Test", "is_taxable": true, "item_type": "plan", "show_description_in_invoices": true, "show_description_in_quotes": true, "object": "item_price", "custom_fields": []}, "emitted_at": 1678971392306} {"stream": "item_price", "data": {"id": "Test-Gift-Plan-1-USD-Daily", "name": "Test Gift Plan 1 USD Daily", "item_family_id": "cbdemo_pf_crm", "item_id": "Test-Gift-Plan-1", "description": "Test gift", "status": "active", "external_name": "Test Gift Plan 1", "pricing_model": "flat_fee", "price": 1500, "period": 1, "currency_code": "USD", "period_unit": "day", "shipping_period": 1, "shipping_period_unit": "day", "billing_cycles": 1, "free_quantity": 0, "channel": "web", "resource_version": 1674055340456, "updated_at": 1674055340, "created_at": 1674055340, "invoice_notes": "Test gift", "is_taxable": true, "item_type": "plan", "show_description_in_invoices": true, "show_description_in_quotes": true, "object": "item_price", "custom_fields": []}, "emitted_at": 1678971392312} {"stream": "item_price", "data": {"id": "Test-Gift-Plan-1-USD-Weekly", "name": "Test Gift Plan 1 USD Weekly", "item_family_id": "cbdemo_pf_crm", "item_id": "Test-Gift-Plan-1", "description": "Test", "status": "active", "external_name": "Test Gift Plan 1", "pricing_model": "flat_fee", "price": 20000, "period": 1, "currency_code": "USD", "period_unit": "week", "shipping_period": 1, "shipping_period_unit": "week", "billing_cycles": 1, "free_quantity": 0, "channel": "web", "resource_version": 1674056134136, "updated_at": 1674056134, "created_at": 1674056134, "is_taxable": true, "item_type": "plan", "show_description_in_invoices": true, "show_description_in_quotes": true, "object": "item_price", "custom_fields": []}, "emitted_at": 1678971392319} {"stream": "payment_source", "data": {"id": "pm_Azz5jBTTJ96QflvC", "updated_at": 1674057604, "resource_version": 1674057604123, "deleted": false, "object": "payment_source", "customer_id": "Azz5jBTTJ96Mjlv5", "type": "card", "reference_id": "tok_Azz5jBTTJ96QSlvA", "status": "valid", "gateway": "chargebee", "gateway_account_id": "gw_16CKmRSb2oGddH4", "ip_address": "85.209.47.207", "created_at": 1674057604, "card": {"iin": "411111", "last4": "1111", "funding_type": "credit", "expiry_month": 12, "expiry_year": 2029, "masked_number": "************1111", "object": "card", "brand": "visa"}, "custom_fields": []}, "emitted_at": 1678971627515} {"stream": "payment_source", "data": {"id": "pm_6olPmTVuo8BJuz", "updated_at": 1676446372, "resource_version": 1676446372397, "deleted": false, "object": "payment_source", "customer_id": "Azz5jBTTJ96UqlvE", "type": "card", "reference_id": "tok_6olPmTVuo8B8uy", "status": "valid", "gateway": "chargebee", "gateway_account_id": "gw_16CKmRSb2oGddH4", "created_at": 1676446372, "card": {"first_name": "User2", "last_name": "Sample", "iin": "411111", "last4": "1111", "funding_type": "credit", "expiry_month": 1, "expiry_year": 2025, "billing_addr1": "Test adderess 2", "billing_city": "San Francisco", "billing_state_code": "CA", "billing_state": "California", "billing_country": "US", "billing_zip": "94114", "masked_number": "************1111", "object": "card", "brand": "visa"}, "custom_fields": []}, "emitted_at": 1678971627741} -{"stream":"promotional_credit","data":{"id":"pc_16CR0QTTZPx6wEbXR","customer_id":"Test-Custome-1","type":"decrement","amount":1000,"description":"Applied to the invoice # 39","credit_type":"general","closing_balance":26000,"created_at":1674298041,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220310} -{"stream":"promotional_credit","data":{"id":"pc_16CZbuTTZGXtAFrxD","customer_id":"cbdemo_richard","type":"decrement","amount":1000,"description":"Applied to the invoice # 37","credit_type":"general","closing_balance":47000,"created_at":1674295799,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220318} -{"stream":"promotional_credit","data":{"id":"pc_AzqD80TTZGFG7TiT6","customer_id":"cbdemo_tyler","type":"decrement","amount":1000,"description":"Applied to the invoice # 36","credit_type":"general","closing_balance":107000,"created_at":1674295727,"object":"promotional_credit","currency_code":"USD","custom_fields":[]},"emitted_at":1676569220325} -{"stream":"gift","data":{"id":"Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950","status":"unclaimed","scheduled_at":1674057609,"auto_claim":false,"updated_at":1674057613,"resource_version":1674057613941,"object":"gift","no_expiry":true,"gifter":{"customer_id":"Azz5jBTTJ96Mjlv5","invoice_id":"27","signature":"Airbyte","note":"Test gift","object":"gifter"},"gift_receiver":{"customer_id":"Azz5jBTTJ96UqlvE","subscription_id":"Azz5jBTTJ96Y2lvK","first_name":"Test","last_name":"2","email":"integration-tgest@airbyte.io","object":"gift_receiver"},"gift_timelines":[{"status":"unclaimed","occurred_at":1674057613,"object":"gift_timeline"},{"status":"scheduled","occurred_at":1674057604,"object":"gift_timeline"}],"custom_fields":[]},"emitted_at":1676569222624} -{"stream": "coupon", "data": {"id": "TESTCOUPON4", "name": "Test Coupon 4", "invoice_name": "", "discount_type": "percentage", "discount_percentage": 12.0, "duration_type": "one_time", "valid_till": 1680332399, "status": "expired", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371290, "updated_at": 1676371290, "resource_version": 1676371290028, "object": "coupon", "redemptions": 1, "custom_fields": []}, "emitted_at": 1678971894181} -{"stream": "coupon", "data": {"id": "TESTCOUPON5", "name": "Test Coupon 5", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 300, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371313, "updated_at": 1676371313, "resource_version": 1676371313572, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894193} -{"stream": "coupon", "data": {"id": "TESTCOUPON6", "name": "Test Coupon 6", "invoice_name": "", "discount_type": "fixed_amount", "discount_amount": 500, "duration_type": "forever", "status": "active", "apply_discount_on": "not_applicable", "apply_on": "invoice_amount", "created_at": 1676371327, "updated_at": 1676371327, "resource_version": 1676371327854, "object": "coupon", "redemptions": 1, "currency_code": "USD", "custom_fields": []}, "emitted_at": 1678971894207} +{"stream": "promotional_credit", "data": {"id": "pc_16CR0QTTZPx6wEbXR", "customer_id": "Test-Custome-1", "type": "decrement", "amount": 1000, "description": "Applied to the invoice # 39", "credit_type": "general", "closing_balance": 26000, "created_at": 1674298041, "object": "promotional_credit", "currency_code": "USD", "custom_fields": []}, "emitted_at": 1676569220310} +{"stream": "promotional_credit", "data": {"id": "pc_16CZbuTTZGXtAFrxD", "customer_id": "cbdemo_richard", "type": "decrement", "amount": 1000, "description": "Applied to the invoice # 37", "credit_type": "general", "closing_balance": 47000, "created_at": 1674295799, "object": "promotional_credit", "currency_code": "USD", "custom_fields": []}, "emitted_at": 1676569220318} +{"stream": "promotional_credit", "data": {"id": "pc_AzqD80TTZGFG7TiT6", "customer_id": "cbdemo_tyler", "type": "decrement", "amount": 1000, "description": "Applied to the invoice # 36", "credit_type": "general", "closing_balance": 107000, "created_at": 1674295727, "object": "promotional_credit", "currency_code": "USD", "custom_fields": []}, "emitted_at": 1676569220325} {"stream": "transaction", "data": {"id": "txn_AzZTZgTTHbgFlKJ", "customer_id": "cbdemo_tyler", "subscription_id": "cbdemo_non_renewing_sub", "payment_method": "cash", "reference_number": "205000001", "gateway": "not_applicable", "type": "payment", "date": 1674070800, "exchange_rate": 1.0, "amount": 27000, "status": "success", "updated_at": 1674034862, "resource_version": 1674034862389, "deleted": false, "object": "transaction", "currency_code": "USD", "base_currency_code": "USD", "amount_unused": 0, "linked_invoices": [{"invoice_id": "cbdemo_inv_003", "applied_amount": 27000, "applied_at": 1674034862, "invoice_date": 1626053281, "invoice_total": 80000, "invoice_status": "paid"}], "linked_refunds": [], "custom_fields": []}, "emitted_at": 1678972145147} {"stream": "transaction", "data": {"id": "txn_16CQyCTTHdwAAwZq", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "gateway_account_id": "gw_16CKmRSb2oGddH4", "payment_source_id": "pm_AzZlweSefvdgrUy4", "payment_method": "card", "gateway": "chargebee", "type": "payment", "date": 1674035400, "exchange_rate": 1.0, "amount": 50000, "id_at_gateway": "cb_16CQyCTTHdwAKwZr", "status": "failure", "error_code": "3003", "error_text": "Activity limit exceeded", "updated_at": 1674035400, "resource_version": 1674035400229, "deleted": false, "object": "transaction", "masked_card_number": "***********8431", "currency_code": "USD", "base_currency_code": "USD", "amount_unused": 0, "linked_invoices": [{"invoice_id": "23", "applied_amount": 50000, "applied_at": 1674035400, "invoice_date": 1674035390, "invoice_total": 50000, "invoice_status": "paid"}], "linked_refunds": [], "payment_method_details": "{\"card\":{\"first_name\":\"Douglas\",\"last_name\":\"Quaid\",\"iin\":\"371449\",\"last4\":\"8431\",\"funding_type\":\"not_known\",\"expiry_month\":5,\"expiry_year\":2028,\"masked_number\":\"***********8431\",\"object\":\"card\",\"brand\":\"american_express\"}}", "custom_fields": []}, "emitted_at": 1678972145155} {"stream": "transaction", "data": {"id": "txn_16CLzOTTHe3QSDF", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "payment_method": "cash", "reference_number": "113234235", "gateway": "not_applicable", "type": "payment", "date": 1674071372, "exchange_rate": 1.0, "amount": 50000, "status": "success", "updated_at": 1674035428, "resource_version": 1674035428131, "deleted": false, "object": "transaction", "currency_code": "USD", "base_currency_code": "USD", "amount_unused": 0, "linked_invoices": [{"invoice_id": "23", "applied_amount": 50000, "applied_at": 1674035428, "invoice_date": 1674035390, "invoice_total": 50000, "invoice_status": "paid"}], "linked_refunds": [], "custom_fields": []}, "emitted_at": 1678972145163} -{"stream":"quote","data":{"id":"Q20230001","name":"Q1","customer_id":"Test-Custome-1","status":"closed","operation_type":"create_subscription_for_customer","price_type":"tax_exclusive","valid_till":1674979199,"date":1674054723,"total_payable":0,"charge_on_acceptance":0,"sub_total":1000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"version":1,"updated_at":1674979203,"resource_version":1674979203400,"object":"quote","line_items":[{"id":"AzZTZgTTIx14r2aG","date_from":1674054723,"date_to":1674141123,"unit_amount":1000,"quantity":1,"amount":1000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"Test-Custome-1","description":"Test Plan 1","entity_type":"plan_item_price","entity_id":"Test-Plan-1-USD-Daily","entity_description":"Test","metered": false,"discount_amount":1000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":1000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIx14r2aG","discount_type":"promotional_credits","discount_amount":1000}],"taxes":[],"line_item_taxes":[],"currency_code":"USD","billing_address":{"first_name":"Sample Name Two","last_name":"Sample Lastname 2","email":"name2@example.com","company":"Test Company Org 2","phone":"+1 388 846 3888","line1":"Ms Ninette Franck","line2":"4381","city":"San Francisco","state_code":"CA","state":"California","country":"US","zip":"94114","validation_status":"not_validated","object":"billing_address"},"shipping_address":{"first_name":"Sample Name Two","last_name":"Sample Lastname 2","email":"name2@example.com","company":"Test Company Org 2","phone":"+1 388 846 3888","line1":"Ms Ninette Franck","line2":"4381","city":"San Francisco","state_code":"CA","state":"California","country":"US","zip":"94114","validation_status":"not_validated","object":"shipping_address"},"custom_fields":[]},"emitted_at":1676569246807} -{"stream":"quote","data":{"id":"Q20230002","name":"Q2","customer_id":"cbdemo_richard","status":"closed","operation_type":"onetime_invoice","price_type":"tax_exclusive","valid_till":1674979199,"date":1674054772,"total_payable":0,"charge_on_acceptance":0,"sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"version":1,"updated_at":1674979203,"resource_version":1674979203437,"object":"quote","line_items":[{"id":"AzZTZgTTIxDoN2ar","date_from":1674054772,"date_to":1674227572,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_richard","description":"Implementation Charge","entity_type":"charge_item_price","entity_id":"cbdemo_implementation-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxDoN2ar","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"currency_code":"USD","billing_address":{"first_name":"Richard","last_name":"Hendricks","company":"Zencorporation","validation_status":"not_validated","object":"billing_address"},"shipping_address":{"validation_status":"not_validated","object":"shipping_address"},"custom_fields":[]},"emitted_at":1676569246817} -{"stream":"quote","data":{"id":"Q20230003","name":"Q3","customer_id":"cbdemo_tyler","status":"closed","operation_type":"onetime_invoice","price_type":"tax_exclusive","valid_till":1674979199,"date":1674054823,"total_payable":0,"charge_on_acceptance":0,"sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"version":1,"updated_at":1674979203,"resource_version":1674979203581,"object":"quote","line_items":[{"id":"AzZTZgTTIxQzS2bH","date_from":1674054823,"date_to":1674227623,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_tyler","description":"Setup Charge","entity_type":"charge_item_price","entity_id":"cbdemo_setup-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxQzS2bH","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"currency_code":"USD","billing_address":{"first_name":"Tyler","last_name":"Durden","company":"Iselectrics","validation_status":"not_validated","object":"billing_address"},"shipping_address":{"validation_status":"not_validated","object":"shipping_address"},"custom_fields":[]},"emitted_at":1676569246824} -{"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIx14p2aF","sub_total":1000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"subscription_renewal","billing_cycle_number":1,"object":"quote_line_group","line_items":[{"id":"AzZTZgTTIx14r2aG","date_from":1674054723,"date_to":1674141123,"unit_amount":1000,"quantity":1,"amount":1000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"Test-Custome-1","description":"Test Plan 1","entity_type":"plan_item_price","entity_id":"Test-Plan-1-USD-Daily","entity_description":"Test","metered": false,"discount_amount":1000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":1000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIx14r2aG","discount_type":"promotional_credits","discount_amount":1000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251063} -{"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIxDoL2aq","sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"immediate","object":"quote_line_group","line_items":[{"id":"AzZTZgTTIxDoN2ar","date_from":1674054772,"date_to":1674227572,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_richard","description":"Implementation Charge","entity_type":"charge_item_price","entity_id":"cbdemo_implementation-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxDoN2ar","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251257} -{"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIxQzR2bG","sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"immediate","object":"quote_line_group","line_items":[{"id":"AzZTZgTTIxQzS2bH","date_from":1674054823,"date_to":1674227623,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_tyler","description":"Setup Charge","entity_type":"charge_item_price","entity_id":"cbdemo_setup-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxQzS2bH","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251413} +{"stream": "quote", "data": {"id": "Q20230001", "name": "Q1", "customer_id": "Test-Custome-1", "status": "closed", "operation_type": "create_subscription_for_customer", "price_type": "tax_exclusive", "valid_till": 1674979199, "date": 1674054723, "total_payable": 0, "charge_on_acceptance": 0, "sub_total": 1000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "version": 1, "updated_at": 1674979203, "resource_version": 1674979203400, "object": "quote", "line_items": [{"id": "AzZTZgTTIx14r2aG", "date_from": 1674054723, "date_to": 1674141123, "unit_amount": 1000, "quantity": 1, "amount": 1000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "Test-Custome-1", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "metered": false, "discount_amount": 1000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 1000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIx14r2aG", "discount_type": "promotional_credits", "discount_amount": 1000}], "taxes": [], "line_item_taxes": [], "currency_code": "USD", "billing_address": {"first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "name2@example.com", "company": "Test Company Org 2", "phone": "+1 388 846 3888", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "billing_address"}, "shipping_address": {"first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "name2@example.com", "company": "Test Company Org 2", "phone": "+1 388 846 3888", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "shipping_address"}, "custom_fields": []}, "emitted_at": 1676569246807} +{"stream": "quote", "data": {"id": "Q20230002", "name": "Q2", "customer_id": "cbdemo_richard", "status": "closed", "operation_type": "onetime_invoice", "price_type": "tax_exclusive", "valid_till": 1674979199, "date": 1674054772, "total_payable": 0, "charge_on_acceptance": 0, "sub_total": 50000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "version": 1, "updated_at": 1674979203, "resource_version": 1674979203437, "object": "quote", "line_items": [{"id": "AzZTZgTTIxDoN2ar", "date_from": 1674054772, "date_to": 1674227572, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "cbdemo_richard", "description": "Implementation Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_implementation-charge-USD", "metered": false, "discount_amount": 50000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 50000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIxDoN2ar", "discount_type": "promotional_credits", "discount_amount": 50000}], "taxes": [], "line_item_taxes": [], "currency_code": "USD", "billing_address": {"first_name": "Richard", "last_name": "Hendricks", "company": "Zencorporation", "validation_status": "not_validated", "object": "billing_address"}, "shipping_address": {"validation_status": "not_validated", "object": "shipping_address"}, "custom_fields": []}, "emitted_at": 1676569246817} +{"stream": "quote", "data": {"id": "Q20230003", "name": "Q3", "customer_id": "cbdemo_tyler", "status": "closed", "operation_type": "onetime_invoice", "price_type": "tax_exclusive", "valid_till": 1674979199, "date": 1674054823, "total_payable": 0, "charge_on_acceptance": 0, "sub_total": 50000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "version": 1, "updated_at": 1674979203, "resource_version": 1674979203581, "object": "quote", "line_items": [{"id": "AzZTZgTTIxQzS2bH", "date_from": 1674054823, "date_to": 1674227623, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "cbdemo_tyler", "description": "Setup Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_setup-charge-USD", "metered": false, "discount_amount": 50000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 50000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIxQzS2bH", "discount_type": "promotional_credits", "discount_amount": 50000}], "taxes": [], "line_item_taxes": [], "currency_code": "USD", "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "shipping_address": {"validation_status": "not_validated", "object": "shipping_address"}, "custom_fields": []}, "emitted_at": 1676569246824} +{"stream": "quote_line_group", "data": {"version": 1, "id": "qlg_AzZTZgTTIx14p2aF", "sub_total": 1000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "charge_event": "subscription_renewal", "billing_cycle_number": 1, "object": "quote_line_group", "line_items": [{"id": "AzZTZgTTIx14r2aG", "date_from": 1674054723, "date_to": 1674141123, "unit_amount": 1000, "quantity": 1, "amount": 1000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "Test-Custome-1", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "metered": false, "discount_amount": 1000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 1000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIx14r2aG", "discount_type": "promotional_credits", "discount_amount": 1000}], "taxes": [], "line_item_taxes": [], "quote_id": "Q20230001", "custom_fields": []}, "emitted_at": 1706028674052} +{"stream": "quote_line_group", "data": {"version": 1, "id": "qlg_AzZTZgTTIxDoL2aq", "sub_total": 50000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "charge_event": "immediate", "object": "quote_line_group", "line_items": [{"id": "AzZTZgTTIxDoN2ar", "date_from": 1674054772, "date_to": 1674227572, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "cbdemo_richard", "description": "Implementation Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_implementation-charge-USD", "metered": false, "discount_amount": 50000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 50000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIxDoN2ar", "discount_type": "promotional_credits", "discount_amount": 50000}], "taxes": [], "line_item_taxes": [], "quote_id": "Q20230002", "custom_fields": []}, "emitted_at": 1706028674176} +{"stream": "quote_line_group", "data": {"version": 1, "id": "qlg_AzZTZgTTIxQzR2bG", "sub_total": 50000, "total": 0, "credits_applied": 0, "amount_paid": 0, "amount_due": 0, "charge_event": "immediate", "object": "quote_line_group", "line_items": [{"id": "AzZTZgTTIxQzS2bH", "date_from": 1674054823, "date_to": 1674227623, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "customer_id": "cbdemo_tyler", "description": "Setup Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_setup-charge-USD", "metered": false, "discount_amount": 50000, "item_level_discount_amount": 0}], "discounts": [{"object": "discount", "entity_type": "promotional_credits", "description": "Promotional Credits", "amount": 50000}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "AzZTZgTTIxQzS2bH", "discount_type": "promotional_credits", "discount_amount": 50000}], "taxes": [], "line_item_taxes": [], "quote_id": "Q20230003", "custom_fields": []}, "emitted_at": 1706028674305} +{"stream": "invoice", "data": {"id": "cbdemo_inv_003", "customer_id": "cbdemo_tyler", "subscription_id": "cbdemo_non_renewing_sub", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1626053281, "due_date": 1626053281, "net_term_days": 0, "exchange_rate": 1.0, "total": 80000, "amount_paid": 77000, "amount_adjusted": 3000, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674070800, "updated_at": 1674034862, "resource_version": 1674034862390, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 80000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1626053281, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_AzZlweSefvesUUyx", "date_from": 1626053281, "date_to": 1628731680, "unit_amount": 60000, "quantity": 1, "amount": 60000, "pricing_model": "per_unit", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_non_renewing_sub", "customer_id": "cbdemo_tyler", "description": "Intermediary - Monthly Plan", "entity_type": "plan_item_price", "entity_id": "cbdemo_intermediary-USD-monthly", "entity_description": "Intermediary Monthly Plan USD", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_AzZlweSefvesZUyy", "date_from": 1626053281, "date_to": 1628731680, "unit_amount": 20000, "quantity": 1, "amount": 20000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_non_renewing_sub", "customer_id": "cbdemo_tyler", "description": "Additional Analytics - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_additional-analytics-USD-monthly", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "sub_total": 80000, "linked_payments": [{"txn_id": "txn_AzZlweSefvetOUyz", "applied_amount": 50000, "applied_at": 1626053281, "txn_status": "success", "txn_date": 1626053281, "txn_amount": 50000}, {"txn_id": "txn_AzZTZgTTHbgFlKJ", "applied_amount": 27000, "applied_at": 1674034862, "txn_status": "success", "txn_date": 1674070800, "txn_amount": 27000}], "applied_credits": [], "adjustment_credit_notes": [{"cn_id": "TEST-CN-4", "cn_reason_code": "waiver", "cn_create_reason_code": "Waiver", "cn_date": 1626917282, "cn_total": 3000, "cn_status": "adjusted"}], "issued_credit_notes": [], "linked_orders": [], "dunning_attempts": [], "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "custom_fields": []}, "emitted_at": 1703114360732} +{"stream": "invoice", "data": {"id": "23", "po_number": "0000002", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1674035390, "due_date": 1674035390, "net_term_days": 0, "exchange_rate": 1.0, "total": 50000, "amount_paid": 50000, "amount_adjusted": 0, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674071372, "dunning_status": "stopped", "updated_at": 1674035428, "resource_version": 1674035428134, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 50000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1674035390, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_AzZTZgTTHdIUcNR", "date_from": 1674035247, "date_to": 1676713647, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "tiered", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHdIU1NP", "customer_id": "cbdemo_douglas", "description": "Additional Users - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_additional-users-USD-monthly", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "line_item_tiers": [{"starting_unit": 1, "ending_unit": 10, "quantity_used": 1, "unit_amount": 50000, "object": "line_item_tier", "line_item_id": "li_AzZTZgTTHdIUcNR"}], "sub_total": 50000, "linked_payments": [{"txn_id": "txn_16CQyCTTHdwAAwZq", "applied_amount": 50000, "applied_at": 1674035400, "txn_status": "failure", "txn_date": 1674035400, "txn_amount": 50000}, {"txn_id": "txn_16CLzOTTHe3QSDF", "applied_amount": 50000, "applied_at": 1674035428, "txn_status": "success", "txn_date": 1674071372, "txn_amount": 50000}], "applied_credits": [], "adjustment_credit_notes": [], "issued_credit_notes": [], "linked_orders": [], "dunning_attempts": [{"created_at": 1674035400, "attempt": 0, "dunning_type": "auto_collect", "transaction_id": "txn_16CQyCTTHdwAAwZq", "txn_status": "failure", "txn_amount": 50000}], "billing_address": {"first_name": "Douglas", "last_name": "Quaid", "company": "Greenplus Enterprises", "validation_status": "not_validated", "object": "billing_address"}, "custom_fields": []}, "emitted_at": 1703114360741} +{"stream": "invoice", "data": {"id": "24", "po_number": "0000003", "customer_id": "cbdemo_tyler", "subscription_id": "6olOsTTHieWUY9", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1674036523, "due_date": 1674036523, "net_term_days": 0, "exchange_rate": 1.0, "total": 51000, "amount_paid": 51000, "amount_adjusted": 0, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674036524, "updated_at": 1674036524, "resource_version": 1674036524204, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 51000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1674036523, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_6olOsTTHieX6YB", "date_from": 1674036523, "date_to": 1674122923, "unit_amount": 1000, "quantity": 1, "amount": 1000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_6olOsTTHieXBYC", "date_from": 1674036523, "date_to": 1674900523, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "description": "Setup Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_setup-charge-USD", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "sub_total": 51000, "linked_payments": [{"txn_id": "txn_6olOsTTHieYGYD", "applied_amount": 51000, "applied_at": 1674036524, "txn_status": "success", "txn_date": 1674036524, "txn_amount": 51000}], "applied_credits": [], "adjustment_credit_notes": [], "issued_credit_notes": [], "linked_orders": [{"id": "1", "status": "queued", "created_at": 1674036525}], "dunning_attempts": [], "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "notes": [{"note": "Test", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily"}], "custom_fields": []}, "emitted_at": 1703114360749} +{"stream": "credit_note", "data": {"id": "TEST-CN-5", "customer_id": "cbdemo_simon", "subscription_id": "cbdemo_future_sub", "reference_invoice_id": "19", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1674033113, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 80000, "amount_allocated": 80000, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1674872880, "generated_at": 1674033113, "updated_at": 1674872880, "channel": "web", "resource_version": 1674872880610, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 80000, "line_items": [{"id": "li_16CM7mTTHULEa1hh", "date_from": 1674069063, "date_to": 1674069063, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "per_unit", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_future_sub", "customer_id": "cbdemo_simon", "description": "Lite - Monthly Plan", "entity_type": "plan_item_price", "entity_id": "cbdemo_lite-USD-monthly", "entity_description": "Lite Monthly Plan USD", "reference_line_item_id": "li_16CZgbTIgigQV7CGu", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_16CM7mTTHULEe1hi", "date_from": 1674069063, "date_to": 1674069063, "unit_amount": 30000, "quantity": 1, "amount": 30000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_future_sub", "customer_id": "cbdemo_simon", "description": "Concierge Support - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_concierge-support-USD-monthly", "reference_line_item_id": "li_16CZgbTIgigQj7CGv", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 20300, "allocated_at": 1674872880, "invoice_id": "64", "invoice_date": 1674872877, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674815049, "invoice_id": "62", "invoice_date": 1674815048, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674728652, "invoice_id": "58", "invoice_date": 1674728648, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674642252, "invoice_id": "54", "invoice_date": 1674642248, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674555851, "invoice_id": "50", "invoice_date": 1674555848, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674469451, "invoice_id": "46", "invoice_date": 1674469448, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674383053, "invoice_id": "42", "invoice_date": 1674383048, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674296651, "invoice_id": "38", "invoice_date": 1674296648, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674210252, "invoice_id": "34", "invoice_date": 1674210248, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674123850, "invoice_id": "30", "invoice_date": 1674123848, "invoice_status": "paid"}, {"allocated_amount": 50700, "allocated_at": 1674037448, "invoice_id": "26", "invoice_date": 1674037448, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727121} +{"stream": "credit_note", "data": {"id": "TEST-CN-6", "customer_id": "cbdemo_simon", "subscription_id": "AzZTZgTTHmX8Gc1", "reference_invoice_id": "128", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1676371612, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 200, "amount_allocated": 200, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1676456650, "generated_at": 1676371612, "updated_at": 1676456650, "channel": "web", "resource_version": 1676456650511, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 200, "line_items": [{"id": "li_16CM0pTVpkRhZ29v", "date_from": 1676371612, "date_to": 1676371612, "unit_amount": 200, "quantity": 1, "amount": 200, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "reference_line_item_id": "li_16CR6XTVdxgpUH1gi", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 200, "allocated_at": 1676456650, "invoice_id": "140", "invoice_date": 1676456648, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727285} +{"stream": "credit_note", "data": {"id": "TEST-CN-7", "customer_id": "cbdemo_simon", "subscription_id": "AzZTZgTTHmX8Gc1", "reference_invoice_id": "128", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1676371655, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 100, "amount_allocated": 100, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1676456650, "generated_at": 1676371655, "updated_at": 1676456650, "channel": "web", "resource_version": 1676456650518, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 100, "line_items": [{"id": "li_AzZTODTVpkcu827U", "date_from": 1676371655, "date_to": 1676371655, "unit_amount": 100, "quantity": 1, "amount": 100, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "reference_line_item_id": "li_16CR6XTVdxgpUH1gi", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 100, "allocated_at": 1676456650, "invoice_id": "140", "invoice_date": 1676456648, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727293} +{"stream": "comment", "data": {"id": "cmt_16CM7mTTHULGY1hj", "entity_type": "credit_note", "entity_id": "TEST-CN-5", "notes": "Test", "added_by": "integration-test@airbyte.io", "created_at": 1674033113, "type": "user", "object": "comment", "custom_fields": []}, "emitted_at": 1703800968922} +{"stream": "comment", "data": {"id": "cmt_AzZTZgTTHaxTnJV", "entity_type": "subscription", "entity_id": "cbdemo_trial_sub", "notes": "Test PO 0000001", "added_by": "integration-test@airbyte.io", "created_at": 1674034690, "type": "user", "object": "comment", "custom_fields": []}, "emitted_at": 1703800968925} +{"stream": "comment", "data": {"id": "cmt_AzZTZgTTHbgGdKK", "entity_type": "transaction", "entity_id": "txn_AzZTZgTTHbgFlKJ", "notes": "Test cash payment", "added_by": "integration-test@airbyte.io", "created_at": 1674034862, "type": "user", "object": "comment", "custom_fields": []}, "emitted_at": 1703800968928} +{"stream": "differential_price", "data": {"id": "7748afa6-fdbe-4304-ac23-d18a17f27715", "item_price_id": "cbdemo_additional-analytics-USD-yearly", "parent_item_id": "cbdemo_advanced", "price": 200000, "status": "active", "resource_version": 1674032542218, "updated_at": 1674032542, "created_at": 1674032542, "currency_code": "USD", "object": "differential_price", "custom_fields": []}, "emitted_at": 1704223399378} +{"stream": "differential_price", "data": {"id": "7748afa6-fdbe-4304-ac23-d18a17f27715", "item_price_id": "cbdemo_additional-analytics-USD-yearly", "parent_item_id": "cbdemo_advanced", "price": 200000, "status": "active", "resource_version": 1674032542218, "updated_at": 1674032542, "created_at": 1674032542, "currency_code": "USD", "object": "differential_price", "custom_fields": []}, "emitted_at": 1704223399498} +{"stream": "differential_price", "data": {"id": "7748afa6-fdbe-4304-ac23-d18a17f27715", "item_price_id": "cbdemo_additional-analytics-USD-yearly", "parent_item_id": "cbdemo_advanced", "price": 200000, "status": "active", "resource_version": 1674032542218, "updated_at": 1674032542, "created_at": 1674032542, "currency_code": "USD", "object": "differential_price", "custom_fields": []}, "emitted_at": 1704223399634} +{"stream": "gift", "data": {"id": "Azz5jBTTJ96eclvRDvCs2SkyRM3cdsflXE5ClcIpcdbOPaa950", "status": "unclaimed", "scheduled_at": 1674057609, "auto_claim": false, "updated_at": 1674057613, "resource_version": 1674057613941, "object": "gift", "no_expiry": true, "gifter": {"customer_id": "Azz5jBTTJ96Mjlv5", "invoice_id": "27", "signature": "Airbyte", "note": "Test gift", "object": "gifter"}, "gift_receiver": {"customer_id": "Azz5jBTTJ96UqlvE", "subscription_id": "Azz5jBTTJ96Y2lvK", "first_name": "Test", "last_name": "2", "email": "integration-tgest@airbyte.io", "object": "gift_receiver"}, "gift_timelines": [{"status": "unclaimed", "occurred_at": 1674057613, "object": "gift_timeline"}, {"status": "scheduled", "occurred_at": 1674057604, "object": "gift_timeline"}], "custom_fields": []}, "emitted_at": 1705083808513} +{"stream": "unbilled_charge", "data": {"id": "li_AzyhFLU1ehAb9vUH", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "date_from": 1705478400, "date_to": 1705564799, "unit_amount": 123, "pricing_model": "flat_fee", "quantity": 1, "amount": 123, "discount_amount": 0, "description": "Test charge #2", "is_voided": false, "updated_at": 1705524542, "deleted": false, "object": "unbilled_charge", "entity_type": "adhoc", "currency_code": "USD", "custom_fields": []}, "emitted_at": 1705535324699} +{"stream": "unbilled_charge", "data": {"id": "li_6oap6U1egpE4vAs", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "date_from": 1705478400, "date_to": 1705564799, "unit_amount": 100, "pricing_model": "flat_fee", "quantity": 1, "amount": 100, "discount_amount": 0, "description": "Implementation charge", "is_voided": false, "updated_at": 1705524460, "deleted": false, "object": "unbilled_charge", "entity_type": "adhoc", "currency_code": "USD", "custom_fields": []}, "emitted_at": 1705535324701} +{"stream": "hosted_page", "data": {"id": "lxlrZIGiyRcuJCr2Uk3lfsqfOBhk2qwdA", "type": "checkout_gift", "url": "https://airbyte-test.chargebee.com/pages/v3/lxlrZIGiyRcuJCr2Uk3lfsqfOBhk2qwdA/", "state": "requested", "embed": false, "created_at": 1705600066, "object": "hosted_page", "updated_at": 1705600066, "resource_version": 1705600066437, "custom_fields": []}, "emitted_at": 1705600142998} +{"stream": "hosted_page", "data": {"id": "JgmYiyyrUG00JUQmjYcuVhcdOwnqoaCBw8", "type": "collect_now", "url": "https://airbyte-test.chargebee.com/pages/v3/JgmYiyyrUG00JUQmjYcuVhcdOwnqoaCBw8/collect_now", "state": "requested", "embed": false, "created_at": 1705600053, "expires_at": 1706032053, "object": "hosted_page", "updated_at": 1705600053, "resource_version": 1705600053347, "custom_fields": []}, "emitted_at": 1705600143003} +{"stream": "hosted_page", "data": {"id": "DEj7ybCXRlg2QBdtsPk80h0cuzyWfFcdHn", "type": "checkout_gift", "url": "https://airbyte-test.chargebee.com/pages/v3/DEj7ybCXRlg2QBdtsPk80h0cuzyWfFcdHn/", "state": "requested", "embed": false, "created_at": 1705599992, "object": "hosted_page", "updated_at": 1705599992, "resource_version": 1705599992794, "custom_fields": []}, "emitted_at": 1705600143008} +{"stream": "item_family", "data": {"id": "test-4", "name": "test item family 4", "status": "active", "resource_version": 1705960880668, "updated_at": 1705960880, "object": "item_family", "custom_fields": []}, "emitted_at": 1705960929497} +{"stream": "item_family", "data": {"id": "test-3", "name": "test item family 3", "status": "active", "resource_version": 1705956309899, "updated_at": 1705956309, "object": "item_family", "custom_fields": []}, "emitted_at": 1705960929501} +{"stream": "item_family", "data": {"id": "test-2", "name": "test item family 2", "status": "active", "resource_version": 1705956286577, "updated_at": 1705956286, "object": "item_family", "custom_fields": []}, "emitted_at": 1705960929506} +{"stream": "item_family", "data": {"id": "test-1", "name": "test item family 1", "status": "active", "resource_version": 1705956260965, "updated_at": 1705956260, "object": "item_family", "custom_fields": []}, "emitted_at": 1705960929509} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json index b7226a840311..24299a54ae84 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json @@ -100,15 +100,29 @@ { "type": "STREAM", "stream": { - "stream_state": { "occurred_at": 2147483647 }, - "stream_descriptor": { "name": "event" } + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "transaction" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": 2147483647 }, + "stream_descriptor": { "name": "comment" } } }, { "type": "STREAM", "stream": { "stream_state": { "updated_at": 2147483647 }, - "stream_descriptor": { "name": "transaction" } + "stream_descriptor": { "name": "item_family" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "differential_price" } } } ] diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json index 468a1e167331..f8a43e7e1d9a 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json @@ -110,5 +110,33 @@ "stream_state": { "updated_at": 1625596058 }, "stream_descriptor": { "name": "transaction" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "migrated_at": 1625596058 }, + "stream_descriptor": { "name": "site_migration_detail" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": 1625596058 }, + "stream_descriptor": { "name": "comment" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "item_family" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "differential_price" } + } } ] diff --git a/airbyte-integrations/connectors/source-chargebee/main.py b/airbyte-integrations/connectors/source-chargebee/main.py index 946cf215df2d..351ea1590b35 100644 --- a/airbyte-integrations/connectors/source-chargebee/main.py +++ b/airbyte-integrations/connectors/source-chargebee/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chargebee import SourceChargebee +from source_chargebee.run import run if __name__ == "__main__": - source = SourceChargebee() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml index 854035925e39..85a965bb5898 100644 --- a/airbyte-integrations/connectors/source-chargebee/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargebee/metadata.yaml @@ -10,19 +10,36 @@ data: connectorSubtype: api connectorType: source definitionId: 686473f1-76d9-4994-9cc7-9b13da46147c - dockerImageTag: 0.2.5 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-chargebee documentationUrl: https://docs.airbyte.com/integrations/sources/chargebee githubIssueLabel: source-chargebee icon: chargebee.svg license: MIT name: Chargebee + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chargebee registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - subscription + - customer + - invoice + - credit_note + - coupon + - transaction + - event + - order + - plan + - payment_source + - addon supportLevel: certified tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-chargebee/poetry.lock b/airbyte-integrations/connectors/source-chargebee/poetry.lock new file mode 100644 index 000000000000..4f28624f1368 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.1.tar.gz", hash = "sha256:0725c63184c37c2caf89faa2c9972e759d73877d03715b9e3eb56a132a6764a8"}, + {file = "airbyte_cdk-0.58.1-py3-none-any.whl", hash = "sha256:605299228e8838cbe6ea39c6d89c38c9674f3997e7b9b77f1dfb7577d84e0874"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "71e2453d758f0222900531815381ff9c55f1d1a6a68f4c64b00c16de6727c8da" diff --git a/airbyte-integrations/connectors/source-chargebee/pyproject.toml b/airbyte-integrations/connectors/source-chargebee/pyproject.toml new file mode 100644 index 000000000000..62f247cc3c9b --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.1" +name = "source-chargebee" +description = "Source implementation for Chargebee." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/chargebee" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_chargebee" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.1" + +[tool.poetry.scripts] +source-chargebee = "source_chargebee.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.2" diff --git a/airbyte-integrations/connectors/source-chargebee/setup.py b/airbyte-integrations/connectors/source-chargebee/setup.py deleted file mode 100644 index 5c3b625e0b4c..000000000000 --- a/airbyte-integrations/connectors/source-chargebee/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.2", - "pytest-mock~=3.6.1", -] - -setup( - name="source_chargebee", - description="Source implementation for Chargebee.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py index 4ded6f8abc65..9e71e8c97066 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py @@ -2,9 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import dataclass -from typing import Optional +from dataclasses import InitVar, dataclass +from typing import Any, Iterable, Mapping, Optional, Union +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.transformations.transformation import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @@ -46,3 +49,97 @@ def transform( """ record["custom_fields"] = [{"name": k, "value": v} for k, v in record.items() if k.startswith("cf_")] return record + + +@dataclass +class IncrementalSingleSliceCursor(Cursor): + cursor_field: Union[InterpolatedString, str] + config: Config + parameters: InitVar[Mapping[str, Any]] + + def __post_init__(self, parameters: Mapping[str, Any]): + self._state = {} + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) + + def get_request_params( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Current implementation does not provide any options to update request params. + # Returns empty dict + return self._get_request_option(RequestOptionType.request_parameter, stream_slice) + + def get_request_headers( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Current implementation does not provide any options to update request headers. + # Returns empty dict + return self._get_request_option(RequestOptionType.header, stream_slice) + + def get_request_body_data( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Current implementation does not provide any options to update body data. + # Returns empty dict + return self._get_request_option(RequestOptionType.body_data, stream_slice) + + def get_request_body_json( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Optional[Mapping]: + # Current implementation does not provide any options to update body json. + # Returns empty dict + return self._get_request_option(RequestOptionType.body_json, stream_slice) + + def _get_request_option(self, option_type: RequestOptionType, stream_slice: StreamSlice): + return {} + + def get_stream_state(self) -> StreamState: + return self._state + + def set_initial_state(self, stream_state: StreamState): + cursor_field = self.cursor_field.eval(self.config) + cursor_value = stream_state.get(cursor_field) + if cursor_value: + self._state[cursor_field] = cursor_value + self._state["prior_state"] = self._state.copy() + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + latest_record = self._state if self.is_greater_than_or_equal(self._state, most_recent_record) else most_recent_record + if latest_record: + cursor_field = self.cursor_field.eval(self.config) + self._state[cursor_field] = latest_record[cursor_field] + + def stream_slices(self) -> Iterable[Mapping[str, Any]]: + yield {} + + def should_be_synced(self, record: Record) -> bool: + """ + Evaluating if a record should be synced allows for filtering and stop condition on pagination + """ + record_cursor_value = record.get(self.cursor_field.eval(self.config)) + return bool(record_cursor_value) + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ + cursor_field = self.cursor_field.eval(self.config) + first_cursor_value = first.get(cursor_field) if first else None + second_cursor_value = second.get(cursor_field) if second else None + if first_cursor_value and second_cursor_value: + return first_cursor_value > second_cursor_value + elif first_cursor_value: + return True + else: + return False diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml index 5bfaee7de614..9b4fd4f7f279 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml @@ -63,7 +63,6 @@ definitions: field_name: offset requester: $ref: "#/definitions/requester" - base_stream: retriever: $ref: "#/definitions/retriever" @@ -81,6 +80,18 @@ definitions: updated_at[between]: "'[{{stream_slice['start_time']}}, {{stream_slice['end_time']}}]'" incremental_sync: $ref: "#/definitions/date_stream_slicer" + semi_incremental_stream: + $ref: "#/definitions/base_stream" + incremental_sync: + type: CustomIncrementalSync + class_name: source_chargebee.components.IncrementalSingleSliceCursor + cursor_field: "updated_at" + retriever: + $ref: "#/definitions/base_stream/retriever" + record_selector: + $ref: "#/definitions/nested_selector" + record_filter: + condition: "{{ record['updated_at'] >= ( stream_state.get('prior_state', {}).get('updated_at', 0) if stream_state else stream_slice.get('prior_state', {}).get('updated_at', 0) ) }}" addon_stream: $ref: "#/definitions/base_incremental_stream" $parameters: @@ -89,6 +100,7 @@ definitions: path: "/addons" stream_cursor_field: "updated_at" attached_item_stream: + $comment: "As of 2/8/24 this stream should be full refresh only as there is an issue with state being appended rather than replaced, causing duplicates. See GH Issue: https://github.com/airbytehq/airbyte/issues/33854" $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever" @@ -97,12 +109,12 @@ definitions: parent_stream_configs: - type: ParentStreamConfig stream: "#/definitions/item_stream" - parent_key: id - partition_field: id + parent_key: "id" + partition_field: "item_id" $parameters: name: "attached_item" primary_key: "id" - path: "/items/{{ stream_slice.id }}/attached_items" + path: "/items/{{ stream_slice.item_id }}/attached_items" customer_stream: $ref: "#/definitions/base_incremental_stream" $parameters: @@ -130,7 +142,7 @@ definitions: response_filters: - http_codes: [404] action: IGNORE - error_message: "Entity unavailable. Customer must have been deleted." + error_message: "Entity unavailable. Customer may have been deleted." - error_message_contains: "This API operation is not enabled for this site" action: IGNORE error_message: "Stream is available only for Product Catalog 1.0" @@ -138,6 +150,13 @@ definitions: backoff_strategies: - type: WaitTimeFromHeader header: "Retry-After" + transformations: + - type: AddFields + fields: + - path: ["customer_id"] + value: "{{ stream_slice.id }}" + - type: CustomTransformation + class_name: source_chargebee.components.CustomFieldTransformation $parameters: name: "contact" primary_key: "id" @@ -188,7 +207,7 @@ definitions: path: "/events" stream_cursor_field: "occurred_at" gift_stream: - $ref: "#/definitions/base_stream" + $ref: "#/definitions/semi_incremental_stream" $parameters: name: "gift" primary_key: "id" @@ -273,11 +292,12 @@ definitions: path: "/transactions" stream_cursor_field: "updated_at" unbilled_charge_stream: - $ref: "#/definitions/base_stream" + $ref: "#/definitions/semi_incremental_stream" $parameters: name: "unbilled_charge" primary_key: "id" path: "/unbilled_charges" + stream_cursor_field: "updated_at" virtual_bank_account_stream: $ref: "#/definitions/base_incremental_stream" $parameters: @@ -311,10 +331,75 @@ definitions: stream: "#/definitions/quote_stream" parent_key: id partition_field: id + requester: + $ref: "#/definitions/requester" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + error_message: "Entity unavailable. Quote may have been deleted." + transformations: + - type: AddFields + fields: + - path: ["quote_id"] + value: "{{ stream_slice.id }}" + - type: CustomTransformation + class_name: source_chargebee.components.CustomFieldTransformation $parameters: name: "quote_line_group" primary_key: "id" path: "/quotes/{{ stream_slice.id }}/quote_line_groups" + site_migration_detail_stream: + $ref: "#/definitions/base_stream" + incremental_sync: + type: CustomIncrementalSync + class_name: source_chargebee.components.IncrementalSingleSliceCursor + cursor_field: "migrated_at" + retriever: + $ref: "#/definitions/base_stream/retriever" + record_selector: + $ref: "#/definitions/nested_selector" + record_filter: + condition: "{{ record['updated_at'] >= ( stream_state.get('prior_state', {}).get('updated_at', 0) if stream_state else stream_slice.get('prior_state', {}).get('updated_at', 0) ) }}" + $parameters: + name: "site_migration_detail" + primary_key: "entity_id" + path: "/site_migration_details" + cursor_field: "migrated_at" + comment_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/retriever/requester" + request_parameters: + sort_by[asc]: created_at + include_deleted: "true" + created_at[between]: "'[{{stream_slice['start_time']}}, {{stream_slice['end_time']}}]'" + incremental_sync: + $ref: "#/definitions/date_stream_slicer" + $parameters: + name: "comment" + primary_key: "id" + path: "/comments" + stream_cursor_field: "created_at" + item_family_stream: + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "item_family" + primary_key: "id" + path: "/item_families" + stream_cursor_field: "updated_at" + differential_price_stream: + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "differential_price" + primary_key: "id" + path: "/differential_prices" + stream_cursor_field: "updated_at" streams: - "#/definitions/addon_stream" @@ -339,6 +424,10 @@ streams: - "#/definitions/virtual_bank_account_stream" - "#/definitions/quote_stream" - "#/definitions/quote_line_group_stream" + - "#/definitions/site_migration_detail_stream" + - "#/definitions/comment_stream" + - "#/definitions/item_family_stream" + - "#/definitions/differential_price_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py new file mode 100644 index 000000000000..5c0b427da197 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chargebee import SourceChargebee + + +def run(): + source = SourceChargebee() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json index b92355f36862..25e97d7b4a41 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json @@ -4,20 +4,16 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "name": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "invoice_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "pricing_model": { "type": ["string", "null"] @@ -26,23 +22,19 @@ "type": ["string", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "period_unit": { "type": ["string", "null"] }, "unit": { - "type": ["string", "null"], - "maxLength": 30 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -54,12 +46,10 @@ "type": ["boolean", "null"] }, "tax_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "taxjar_product_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "avalara_sale_type": { "type": ["string", "null"] @@ -71,35 +61,28 @@ "type": ["integer", "null"] }, "sku": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category1": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category2": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category3": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category4": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "is_shippable": { "type": ["boolean", "null"] }, "shipping_frequency_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "shipping_frequency_period_unit": { "type": ["string", "null"] @@ -111,22 +94,19 @@ "type": ["integer", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "included_in_mrr": { "type": ["boolean", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] }, "taxable": { "type": ["boolean", "null"] }, "tax_profile_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "meta_data": { "type": ["object", "null"], @@ -138,50 +118,43 @@ "show_description_in_quotes": { "type": ["boolean", "null"] }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, "tiers": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "starting_unit": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "starting_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "ending_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json index 73207b7e2c57..a52c86315350 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/attached_item.json @@ -4,16 +4,13 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "parent_item_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "item_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "type": { "type": ["string", "null"] @@ -22,16 +19,13 @@ "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "billing_cycles": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "charge_on_event": { "type": ["string", "null"] @@ -50,20 +44,9 @@ }, "object": { "type": ["string", "null"] - } - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json new file mode 100644 index 000000000000..48cc655b1d0e --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Comment", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "entity_type": { + "type": ["string", "null"] + }, + "added_by": { + "type": ["string", "null"] + }, + "notes": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["integer", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "entity_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/contact.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/contact.json index c862f291ace7..9225306f8fbe 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/contact.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/contact.json @@ -15,6 +15,9 @@ "id": { "type": ["string", "null"] }, + "customer_id": { + "type": ["string", "null"] + }, "label": { "type": ["string", "null"] }, @@ -30,19 +33,11 @@ "send_billing_email": { "type": ["boolean", "null"] }, + "phone": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json index f6d484b980a1..512860182a47 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json @@ -4,16 +4,13 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "name": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "invoice_name": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "discount_type": { "type": ["string", "null"] @@ -25,8 +22,7 @@ "type": ["integer", "null"] }, "currency_code": { - "type": ["string", "null"], - "max-length": 3 + "type": ["string", "null"] }, "duration_type": { "type": ["string", "null"] @@ -71,8 +67,10 @@ "type": ["integer", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "max-length": 2000 + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] }, "item_constraints": { "type": ["array", "null"], @@ -116,18 +114,7 @@ } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json index 3e3496697263..fbd4309027fd 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json @@ -4,20 +4,16 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "reference_invoice_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "type": { "type": ["string", "null"] @@ -29,8 +25,7 @@ "type": ["string", "null"] }, "vat_number": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "date": { "type": ["integer", "null"] @@ -39,24 +34,19 @@ "type": ["string", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_allocated": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_refunded": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_available": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refunded_at": { "type": ["integer", "null"] @@ -74,27 +64,23 @@ "type": ["integer", "null"] }, "sub_total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "sub_total_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "total_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "local_currency_code": { "type": ["string", "null"] }, "round_off_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "fractional_correction": { "type": ["integer", "null"], - "minimum": 0 + "maximum": 50000 }, "deleted": { "type": ["boolean", "null"] @@ -103,8 +89,32 @@ "type": ["string", "null"] }, "vat_number_prefix": { - "type": ["string", "null"], - "maxLength": 10 + "type": ["string", "null"] + }, + "base_currency_code": { + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "exchange_rate": { + "type": ["number", "null"] + }, + "is_digital": { + "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "is_vat_moss_registered": { + "type": ["boolean", "null"], + "$comment": "Only available for accounts which have enabled taxes for EU Region for taxes." + }, + "customer_notes": { + "type": ["string", "null"] }, "line_items": { "type": ["array", "null"], @@ -112,12 +122,10 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "date_from": { "type": ["integer", "null"] @@ -141,41 +149,31 @@ "type": ["boolean", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_rate": { - "type": ["number", "null"], - "minimum": 0.0, - "maximum": 100.0 + "type": ["number", "null"] }, "unit_amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "item_level_discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "entity_description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "entity_type": { "type": ["string", "null"] @@ -184,12 +182,19 @@ "type": ["string", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] + }, + "metered": { + "type": ["boolean", "null"] + }, + "reference_line_item_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -200,19 +205,16 @@ "type": ["object", "null"], "properties": { "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "entity_type": { "type": ["string", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } } @@ -223,19 +225,16 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "discount_type": { "type": ["string", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } } @@ -246,39 +245,31 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "starting_unit": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "quantity_used": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unit_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "starting_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "ending_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "quantity_used_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "unit_amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] } } } @@ -289,16 +280,13 @@ "type": ["object", "null"], "properties": { "name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] } } } @@ -309,16 +297,13 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "tax_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "tax_rate": { "type": ["number", "null"], - "minimum": 0.0, "maximum": 100.0 }, "is_partial_tax_applied": { @@ -328,27 +313,22 @@ "type": ["boolean", "null"] }, "taxable_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_juris_type": { "type": ["string", "null"] }, "tax_juris_name": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_juris_code": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_amount_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "local_currency-code": { "type": ["string", "null"] @@ -362,12 +342,10 @@ "type": ["object", "null"], "properties": { "txn_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "applied_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "applied_at": { "type": ["integer", "null"] @@ -379,8 +357,7 @@ "type": ["integer", "null"] }, "txn_amount": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "refund_reason_code": { "type": ["string", "null"] @@ -394,12 +371,10 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "description": { "type": ["string", "null"] @@ -419,12 +394,10 @@ "type": ["object", "null"], "properties": { "invoice_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "allocated_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "allocated_at": { "type": ["integer", "null"] @@ -438,19 +411,105 @@ } } }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } + "shipping_address": { + "type": ["object", "null"], + "properties": { + "first_name": { + "type": ["string", "null"] + }, + "last_name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "company": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "line1": { + "type": ["string", "null"] + }, + "line2": { + "type": ["string", "null"] + }, + "line3": { + "type": ["string", "null"] + }, + "city": { + "type": ["string", "null"] + }, + "state_code": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "country": { + "type": ["string", "null"] + }, + "zip": { + "type": ["string", "null"] + }, + "validation_status": { + "type": ["string", "null"] + } + } + }, + "billing_address": { + "type": ["object", "null"], + "properties": { + "first_name": { + "type": ["string", "null"] + }, + "last_name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "company": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "line1": { + "type": ["string", "null"] + }, + "line2": { + "type": ["string", "null"] + }, + "line3": { + "type": ["string", "null"] + }, + "city": { + "type": ["string", "null"] + }, + "state_code": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "country": { + "type": ["string", "null"] + }, + "zip": { + "type": ["string", "null"] + }, + "validation_status": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json index 61f02e49f104..74ea11e34988 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json @@ -4,32 +4,25 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "vat_number": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "auto_collection": { "type": ["string", "null"] @@ -56,8 +49,7 @@ "type": ["integer", "null"] }, "created_from_ip": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "exemption_details": { "type": ["array", "null"], @@ -73,8 +65,7 @@ "type": ["string", "null"] }, "exempt_number": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "resource_version": { "type": ["integer", "null"] @@ -83,8 +74,7 @@ "type": ["integer", "null"] }, "locale": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "billing_date": { "type": ["integer", "null"] @@ -108,36 +98,28 @@ "type": ["string", "null"] }, "primary_payment_source_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "backup_payment_source_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] }, "preferred_currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "promotional_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unbilled_charges": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "excess_payments": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "deleted": { "type": ["boolean", "null"] @@ -155,73 +137,76 @@ "type": ["boolean", "null"] }, "client_profile_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "use_default_hierarchy_settings": { "type": ["boolean", "null"] }, "vat_number_prefix": { - "type": ["string", "null"], - "maxLength": 10 + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "mrr": { + "type": ["integer", "null"] + }, + "tax_providers_fields": { + "type": ["array", "null"] }, "billing_address": { "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -231,12 +216,10 @@ "type": ["object", "null"], "properties": { "external_customer_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "referral_sharing_url": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "created_at": { "type": ["integer", "null"] @@ -245,16 +228,13 @@ "type": ["integer", "null"] }, "referral_campaign_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "referral_account_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "referral_external_campaign_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "referral_system": { "type": ["string", "null"] @@ -268,28 +248,22 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "label": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "enabled": { "type": ["boolean", "null"] @@ -299,6 +273,9 @@ }, "send_billing_email": { "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -313,15 +290,16 @@ "type": ["string", "null"] }, "gateway_account_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] }, "reference_id": { - "type": ["string", "null"], - "maxLength": 200 + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -331,24 +309,25 @@ "type": ["object", "null"], "properties": { "promotional_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "excess_payments": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unbilled_charges": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] + }, + "balance_currency_code": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -357,16 +336,13 @@ "type": ["object", "null"], "properties": { "parent_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "payment_owner_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "invoice_owner_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] } } }, @@ -413,24 +389,12 @@ "card_status": { "type": ["string", "null"] }, - "meta_data": { "type": ["object", "null"], "properties": {} }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json new file mode 100644 index 000000000000..7897c345600f --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json @@ -0,0 +1,80 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Add-on", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "item_price_id": { + "type": ["string", "null"] + }, + "parent_item_id": { + "type": ["string", "null"] + }, + "price": { + "type": ["integer", "null"] + }, + "price_in_decimal": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "resource_version": { + "type": ["integer", "null"] + }, + "updated_at": { + "type": ["integer", "null"] + }, + "created_at": { + "type": ["integer", "null"] + }, + "modified_at": { + "type": ["integer", "null"] + }, + "currency_code": { + "type": ["string", "null"] + }, + "tiers": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "starting_unit": { + "type": ["integer", "null"] + }, + "ending_unit": { + "type": ["integer", "null"] + }, + "price": { + "type": ["integer", "null"] + } + } + } + }, + "parent_periods": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "period_unit": { + "type": ["string", "null"] + }, + "period": { + "type": ["array", "null"], + "items": { + "type": ["integer", "null"] + } + } + } + } + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json index f76bd77e4799..bc5181a48763 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json @@ -4,8 +4,7 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "occurred_at": { "type": ["integer", "null"] @@ -14,8 +13,7 @@ "type": ["string", "null"] }, "user": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "event_type": { "type": ["string", "null"] @@ -29,14 +27,16 @@ "content": { "type": ["object", "null"] }, + "object": { + "type": ["string", "null"] + }, "webhooks": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "webhook_status": { "type": ["string", "null"] @@ -45,18 +45,7 @@ } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json index 32fde3838b79..339d01ac750d 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json @@ -7,8 +7,7 @@ "type": ["string", "null"] }, "status": { - "type": ["string", "null"], - "enum": ["scheduled", "unclaimed", "claimed", "cancelled", "expired"] + "type": ["string", "null"] }, "scheduled_at": { "type": ["integer", "null"] @@ -42,6 +41,9 @@ }, "note": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -62,6 +64,9 @@ }, "email": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -71,34 +76,22 @@ "type": ["object", "null"], "properties": { "status": { - "type": ["string", "null"], - "enum": [ - "scheduled", - "unclaimed", - "claimed", - "cancelled", - "expired" - ] + "type": ["string", "null"] }, "occurred_at": { "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json index 8ad0b53168de..b5c838f97186 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json @@ -7,26 +7,13 @@ "type": ["string", "null"] }, "type": { - "type": ["string", "null"], - "enum": [ - "checkout_new", - "checkout_existing", - "update_payment_method", - "manage_payment_sources", - "collect_now", - "extend_subscription", - "checkout_gift", - "claim_gift", - "checkout_one_time", - "pre_cancel" - ] + "type": ["string", "null"] }, "url": { "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "enum": ["created", "requested", "succeeded", "cancelled", "acknowledged"] + "type": ["string", "null"] }, "pass_thru_content": { "type": ["string", "null"] @@ -55,19 +42,11 @@ "business_entity_id": { "type": ["string", "null"] }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json index a3815605c08c..5ad0238ce30c 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json @@ -2,22 +2,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "name": "Invoice", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "po_number": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "recurring": { "type": ["boolean", "null"] @@ -26,8 +23,7 @@ "type": ["string", "null"] }, "vat_number": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "price_type": { "type": ["string", "null"] @@ -45,32 +41,25 @@ "type": ["number", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_paid": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_adjusted": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "write_off_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "credits_applied": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_due": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "paid_at": { "type": ["integer", "null"] @@ -91,30 +80,28 @@ "type": ["integer", "null"] }, "sub_total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "sub_total_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "total_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "local_currency_code": { "type": ["string", "null"] }, "tax": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] + }, + "local_currency_exchange_rate": { + "type": ["number", "null"] }, "first_invoice": { "type": ["boolean", "null"] }, "new_sales_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "has_advance_charges": { "type": ["boolean", "null"] @@ -125,6 +112,9 @@ "is_gifted": { "type": ["boolean", "null"] }, + "is_digital": { + "type": ["boolean", "null"] + }, "generated_at": { "type": ["integer", "null"] }, @@ -132,27 +122,37 @@ "type": ["integer", "null"] }, "amount_to_collect": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "round_off_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "payment_owner": { "type": ["string", "null"] }, - "void_reason_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "deleted": { "type": ["boolean", "null"] }, + "tax_category": { + "type": ["string", "null"] + }, "vat_number_prefix": { - "type": ["string", "null"], - "maxLength": 10 + "type": ["string", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "base_currency_code": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] }, "line_items": { "type": ["array", "null"], @@ -160,12 +160,10 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "date_from": { "type": ["integer", "null"] @@ -189,41 +187,31 @@ "type": ["boolean", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_rate": { - "type": ["number", "null"], - "minimum": 0.0, - "maximum": 100.0 + "type": ["number", "null"] }, "unit_amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "item_level_discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "entity_description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "entity_type": { "type": ["string", "null"] @@ -232,12 +220,16 @@ "type": ["string", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] + }, + "metered": { + "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -248,19 +240,28 @@ "type": ["object", "null"], "properties": { "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "entity_type": { "type": ["string", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] + }, + "coupon_set_code": { + "type": ["string", "null"] + }, + "discount_type": { + "type": ["string", "null"] + }, + "discount_percentage": { + "type": ["number", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -271,20 +272,22 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "discount_type": { "type": ["string", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, - "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] + }, + "coupon_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -295,16 +298,13 @@ "type": ["object", "null"], "properties": { "name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] } } } @@ -315,16 +315,23 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "tax_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "tax_rate": { "type": ["number", "null"] }, + "date_to": { + "type": ["integer", "null"] + }, + "date_from": { + "type": ["integer", "null"] + }, + "prorated_taxable_amount": { + "type": ["number", "null"] + }, "is_partial_tax_applied": { "type": ["boolean", "null"] }, @@ -332,29 +339,24 @@ "type": ["boolean", "null"] }, "taxable_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_juris_type": { "type": ["string", "null"] }, "tax_juris_name": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_juris_code": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_amount_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, - "local_currency-code": { + "local_currency_code": { "type": ["string", "null"] } } @@ -366,39 +368,34 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "starting_unit": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "quantity_used": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unit_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "starting_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "ending_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "quantity_used_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "unit_amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -409,12 +406,10 @@ "type": ["object", "null"], "properties": { "txn_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "applied_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "applied_at": { "type": ["integer", "null"] @@ -426,8 +421,7 @@ "type": ["integer", "null"] }, "txn_amount": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] } } } @@ -441,8 +435,7 @@ "type": ["integer", "null"] }, "transaction_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "dunning_type": { "type": ["string", "null"] @@ -454,8 +447,7 @@ "type": ["string", "null"] }, "txn_amount": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] } } } @@ -466,12 +458,10 @@ "type": ["object", "null"], "properties": { "cn_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "applied_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "applied_at": { "type": ["integer", "null"] @@ -480,8 +470,7 @@ "type": ["string", "null"] }, "cn_create_reason_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "cn_date": { "type": ["integer", "null"] @@ -498,22 +487,19 @@ "type": ["object", "null"], "properties": { "cn_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "cn_reason_code": { "type": ["string", "null"] }, "cn_create_reason_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "cn_date": { "type": ["integer", "null"] }, "cn_total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "cn_status": { "type": ["string", "null"] @@ -527,22 +513,19 @@ "type": ["object", "null"], "properties": { "cn_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "cn_reason_code": { "type": ["string", "null"] }, "cn_create_reason_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "cn_date": { "type": ["integer", "null"] }, "cn_total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "cn_status": { "type": ["string", "null"] @@ -556,12 +539,10 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "document_number": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -570,16 +551,13 @@ "type": ["string", "null"] }, "reference_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "fulfillment_status": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "batch_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "created_at": { "type": ["integer", "null"] @@ -596,12 +574,10 @@ "type": ["string", "null"] }, "note": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } } @@ -610,59 +586,63 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 180 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + } + } + }, + "statement_descriptor": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "descriptor": { + "type": ["string", "null"] + }, + "additional_info": { + "type": ["string", "null"] } } }, @@ -670,75 +650,94 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, - "custom_fields": { - "type": ["null", "array"], + "einvoice": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "reference_number": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"] + } + } + }, + "linked_taxes_withheld": { + "type": ["array", "null"], "items": { - "type": ["null", "object"], + "type": ["object", "null"], "properties": { - "name": { - "type": ["null", "string"] + "id": { + "type": ["string", "null"] }, - "value": { - "type": ["null", "string"] + "amount": { + "type": ["integer", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "date": { + "type": ["integer", "null"] + }, + "reference_number": { + "type": ["string", "null"] } } } + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json index 4607fb50c467..7c74ff778665 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json @@ -4,16 +4,13 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "name": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -25,8 +22,7 @@ "type": ["integer", "null"] }, "item_family_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "type": { "type": ["string", "null"] @@ -38,8 +34,7 @@ "type": ["boolean", "null"] }, "redirect_url": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "enabled_for_checkout": { "type": ["boolean", "null"] @@ -54,12 +49,10 @@ "type": ["string", "null"] }, "gift_claim_redirect_url": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "unit": { - "type": ["string", "null"], - "maxLength": 30 + "type": ["string", "null"] }, "metered": { "type": ["boolean", "null"] @@ -74,14 +67,16 @@ "type": ["object", "null"], "properties": {} }, + "external_name": { + "type": ["string", "null"] + }, "applicable_items": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } } @@ -93,18 +88,7 @@ "type": ["string", "null"] }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json new file mode 100644 index 000000000000..bb9b57b37006 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Item Families", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "resource_version": { + "type": ["integer", "null"] + }, + "updated_at": { + "type": ["integer", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json index 357e643c7a9a..be2eecb56103 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_price.json @@ -4,57 +4,46 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "name": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "item_family_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "item_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] }, "external_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "pricing_model": { "type": ["string", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "period_unit": { "type": ["string", "null"] }, "trial_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "trial_period_unit": { "type": ["string", "null"] @@ -63,23 +52,19 @@ "type": ["string", "null"] }, "shipping_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "shipping_period_unit": { "type": ["string", "null"] }, "billing_cycles": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "free_quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "free_quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "resource_version": { "type": ["integer", "null"] @@ -94,8 +79,7 @@ "type": ["integer", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] }, "is_taxable": { "type": ["boolean", "null"] @@ -119,15 +103,13 @@ "type": ["object", "null"], "properties": { "starting_unit": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] } } } @@ -136,8 +118,7 @@ "type": ["object", "null"], "properties": { "tax_profile_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "avalara_sale_type": { "type": ["string", "null"] @@ -149,12 +130,10 @@ "type": ["integer", "null"] }, "avalara_tax_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "taxjar_product_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] } } }, @@ -162,28 +141,22 @@ "type": ["object", "null"], "properties": { "sku": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category1": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category2": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category3": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category4": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } }, @@ -194,18 +167,7 @@ "type": ["string", "null"] }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json index 6b55529a9aab..b985c3fabca9 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json @@ -2,26 +2,22 @@ "$schema": "http://json-schema.org/draft-07/schema#", "name": "Order", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "document_number": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "invoice_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -39,12 +35,10 @@ "type": ["string", "null"] }, "reference_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "fulfillment_status": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "order_date": { "type": ["integer", "null"] @@ -53,53 +47,40 @@ "type": ["integer", "null"] }, "note": { - "type": ["string", "null"], - "maxLength": 600 + "type": ["string", "null"] }, "tracking_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "tracking_url": { - "type": ["string", "null"], - "maxLength": 255 + "type": ["string", "null"] }, "batch_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "created_by": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "shipment_carrier": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, - "invoice_round_off_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_paid": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_adjusted": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits_issued": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "rounding_adjustement": { "type": ["integer", "null"] @@ -138,42 +119,46 @@ "type": ["boolean", "null"] }, "original_order_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "discount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "sub_total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "total": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "deleted": { "type": ["boolean", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "is_gifted": { "type": ["boolean", "null"] }, "gift_note": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "gift_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "resend_reason": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "base_currency_code": { + "type": ["string", "null"] + }, + "exchange_rate": { + "type": ["number", "null"] + }, + "object": { + "type": ["string", "null"] }, "order_line_items": { "type": ["array", "null"], @@ -181,65 +166,50 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "invoice_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "invoice_line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "unit_price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "fulfillment_quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "fulfillment_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_paid": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_adjusted": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits_issued": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "refundable_credits": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "is_shippable": { "type": ["boolean", "null"] }, "sku": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, - "status": { "type": ["string", "null"] }, @@ -247,16 +217,16 @@ "type": ["string", "null"] }, "item_level_discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "entity_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -265,59 +235,49 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 180 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -325,59 +285,49 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -387,17 +337,13 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "tax_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "tax_rate": { - "type": ["number", "null"], - "minimum": 0.0, - "maximum": 100.0 + "type": ["number", "null"] }, "is_partial_tax_applied": { "type": ["boolean", "null"] @@ -406,27 +352,22 @@ "type": ["boolean", "null"] }, "taxable_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "tax_juris_type": { "type": ["string", "null"] }, "tax_juris_name": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_juris_code": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "tax_amount_in_local_currency": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "local_currency-code": { "type": ["string", "null"] @@ -440,19 +381,22 @@ "type": ["object", "null"], "properties": { "line_item_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "discount_type": { "type": ["string", "null"] }, "coupon_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "discount_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] + }, + "entity_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -463,26 +407,25 @@ "type": ["object", "null"], "properties": { "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "type": { "type": ["string", "null"] }, "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] }, "amount_adjusted": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_refunded": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -493,33 +436,19 @@ "type": ["object", "null"], "properties": { "order_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "reason": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] } } } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json index 26095b14a47e..2e4f1bbfeb68 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json @@ -18,90 +18,20 @@ "customer_id": { "type": ["string", "null"] }, + "object": { + "type": ["string", "null"] + }, "type": { - "type": ["string", "null"], - "enum": [ - "card", - "paypal_express_checkout", - "amazon_payments", - "direct_debit", - "generic", - "alipay", - "unionpay", - "apple_pay", - "wechat_pay", - "ideal", - "google_pay", - "sofort", - "bancontact", - "giropay", - "dotpay", - "upi", - "netbanking_emandates" - ] + "type": ["string", "null"] }, "reference_id": { "type": ["string", "null"] }, "status": { - "type": ["string", "null"], - "enum": [ - "valid", - "expiring", - "expired", - "invalid", - "pending_verification" - ] + "type": ["string", "null"] }, "gateway": { - "type": ["string", "null"], - "enum": [ - "chargebee", - "chargebee_payments", - "stripe", - "wepay", - "braintree", - "authorize_net", - "paypal_pro", - "pin", - "eway", - "eway_rapid", - "worldpay", - "balanced_payments", - "beanstream", - "bluepay", - "elavon", - "first_data_global", - "hdfc", - "migs", - "nmi", - "ogone", - "paymill", - "paypal_payflow_pro", - "sage_pay", - "tco", - "wirecard", - "amazon_payments", - "paypal_express_checkout", - "gocardless", - "adyen", - "orbital", - "moneris_us", - "moneris", - "bluesnap", - "cybersource", - "vantiv", - "checkout_com", - "paypal", - "ingenico_direct", - "exact", - "mollie", - "quickbooks", - "razorpay", - "global_payments", - "bank_of_america", - "not_applicable" - ] + "type": ["string", "null"] }, "gateway_account_id": { "type": ["string", "null"] @@ -134,22 +64,10 @@ "type": ["string", "null"] }, "brand": { - "type": ["string", "null"], - "enum": [ - "visa", - "mastercard", - "american_express", - "discover", - "jcb", - "diners_club", - "other", - "bancontact", - "not_applicable" - ] + "type": ["string", "null"] }, "funding_type": { - "type": ["string", "null"], - "enum": ["credit", "dedit", "prepaid", "not_known", "not_applicable"] + "type": ["string", "null"] }, "expiry_month": { "type": ["integer", "null"] @@ -180,6 +98,9 @@ }, "masked_number": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -205,16 +126,13 @@ "type": ["string", "null"] }, "account_type": { - "type": ["string", "null"], - "enum": ["checking", "savings", "business_checking", "current"] + "type": ["string", "null"] }, "echeck_type": { - "type": ["string", "null"], - "enum": ["web", "ppd", "ccd"] + "type": ["string", "null"] }, "account_holder_type": { - "type": ["string", "null"], - "enum": ["individual", "company"] + "type": ["string", "null"] }, "email": { "type": ["string", "null"] @@ -266,18 +184,7 @@ } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json index 6892dcc357ca..705049a84e59 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json @@ -4,39 +4,31 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "name": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "invoice_name": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "description": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "period_unit": { "type": ["string", "null"] }, "trial_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "trial_period_unit": { "type": ["string", "null"] @@ -48,12 +40,10 @@ "type": ["string", "null"] }, "free_quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "setup_cost": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "status": { "type": ["string", "null"] @@ -62,12 +52,10 @@ "type": ["integer", "null"] }, "billing_cycles": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "redirect_url": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "enabled_in_hosted_pages": { "type": ["boolean", "null"] @@ -79,12 +67,10 @@ "type": ["string", "null"] }, "tax_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "taxjar_product_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "avalara_sale_type": { "type": ["string", "null"] @@ -96,35 +82,28 @@ "type": ["integer", "null"] }, "sku": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category1": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category2": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category3": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "accounting_category4": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "is_shippable": { "type": ["boolean", "null"] }, "shipping_frequency_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "shipping_frequency_period_unit": { "type": ["string", "null"] @@ -139,27 +118,31 @@ "type": ["boolean", "null"] }, "claim_url": { - "type": ["string", "null"], - "maxLength": 500 + "type": ["string", "null"] }, "free_quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "charge_model": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] }, "taxable": { "type": ["boolean", "null"] }, "tax_profile_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "meta_data": { "type": ["object", "null"], @@ -177,27 +160,22 @@ "type": ["object", "null"], "properties": { "starting_unit": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "starting_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "ending_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } @@ -208,8 +186,7 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] } } } @@ -220,23 +197,19 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "billing_cycles": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "type": { "type": ["string", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } @@ -247,12 +220,10 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "on_event": { "type": ["string", "null"] @@ -261,25 +232,13 @@ "type": ["boolean", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json index 7089ea9ef12c..526d523fee1a 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json @@ -10,8 +10,7 @@ "type": ["string", "null"] }, "type": { - "type": ["string", "null"], - "enum": ["increment", "decrement"] + "type": ["string", "null"] }, "amount_in_decimal": { "type": ["string", "null"] @@ -26,8 +25,7 @@ "type": ["string", "null"] }, "credit_type": { - "type": ["string", "null"], - "enum": ["loyalty_credits", "referral_rewards", "general"] + "type": ["string", "null"] }, "reference": { "type": ["string", "null"] @@ -41,19 +39,11 @@ "created_at": { "type": ["integer", "null"] }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json index 0e4198814f2c..aa67b7fbb82a 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json @@ -22,23 +22,16 @@ "type": ["string", "null"] }, "status": { - "type": ["string", "null"], - "enum": ["open", "accepted", "declined", "invoiced", "closed"] + "type": ["string", "null"] }, "operation_type": { - "type": ["string", "null"], - "enum": [ - "create_subscription_for_customer", - "change_subscription", - "onetime_invoice" - ] + "type": ["string", "null"] }, "vat_number": { "type": ["string", "null"] }, "price_type": { - "type": ["string", "null"], - "enum": ["tax_exclusive", "tax_inclusive"] + "type": ["string", "null"] }, "valid_till": { "type": ["integer", "null"] @@ -124,8 +117,7 @@ "type": ["integer", "null"] }, "pricing_model": { - "type": ["string", "null"], - "enum": ["flat_fee", "per_unit", "tiered", "volume", "stairstep"] + "type": ["string", "null"] }, "is_taxed": { "type": ["boolean", "null"] @@ -161,33 +153,22 @@ "type": ["string", "null"] }, "entity_type": { - "type": ["string", "null"], - "enum": [ - "adhoc", - "plan_item_price", - "addon_item_price", - "charge_item_price" - ] + "type": ["string", "null"] }, "tax_exempt_reason": { - "type": ["string", "null"], - "enum": [ - "tax_not_configured", - "region_non_taxable", - "export", - "customer_exempt", - "product_exempt", - "zero_rated", - "reverse_charge", - "high_value_physical_goods", - "zero_value_item" - ] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "customer_id": { "type": ["string", "null"] + }, + "metered": { + "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -207,25 +188,19 @@ "type": ["string", "null"] }, "entity_type": { - "type": ["string", "null"], - "enum": [ - "item_level_coupon", - "document_level_coupon", - "promotional_credits", - "prorated_credits", - "item_level_discount", - "document_level_discount" - ] + "type": ["string", "null"] }, "discount_type": { - "type": ["string", "null"], - "enum": ["fixed_amount", "percentage"] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "coupon_set_code": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -239,21 +214,16 @@ "type": ["string", "null"] }, "discount_type": { - "type": ["string", "null"], - "enum": [ - "item_level_coupon", - "document_level_coupon", - "promotional_credits", - "prorated_credits", - "item_level_discount", - "document_level_discount" - ] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "discount_amount": { "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -302,17 +272,7 @@ "type": ["integer", "null"] }, "tax_juris_type": { - "type": ["string", "null"], - "enum": [ - "country", - "federal", - "state", - "county", - "city", - "special", - "unincorporated", - "other" - ] + "type": ["string", "null"] }, "tax_juris_name": { "type": ["string", "null"] @@ -368,63 +328,52 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { - "type": ["string", "null"], - "enum": ["not_validated", "valid", "partially_valid", "invalid"] + "type": ["string", "null"] }, "index": { "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } }, @@ -432,76 +381,57 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { - "type": ["string", "null"], - "enum": ["not_validated", "valid", "partially_valid", "invalid"] + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json index 588c38bd7d4d..b44249f63a04 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json @@ -9,6 +9,9 @@ "id": { "type": ["string", "null"] }, + "quote_id": { + "type": ["string", "null"] + }, "sub_total": { "type": ["integer", "null"] }, @@ -25,15 +28,7 @@ "type": ["integer", "null"] }, "charge_event": { - "type": ["string", "null"], - "enum": [ - "immediate", - "subscription_creation", - "trial_start", - "subscription_change", - "subscription_renewal", - "subscription_cancel" - ] + "type": ["string", "null"] }, "billing_cycle_number": { "type": ["integer", "null"] @@ -65,8 +60,7 @@ "type": ["integer", "null"] }, "pricing_model": { - "type": ["string", "null"], - "enum": ["flat_fee", "per_unit", "tiered", "volume", "stairstep"] + "type": ["string", "null"] }, "is_taxed": { "type": ["boolean", "null"] @@ -102,33 +96,22 @@ "type": ["string", "null"] }, "entity_type": { - "type": ["string", "null"], - "enum": [ - "adhoc", - "plan_item_price", - "addon_item_price", - "charge_item_price" - ] + "type": ["string", "null"] }, "tax_exempt_reason": { - "type": ["string", "null"], - "enum": [ - "tax_not_configured", - "region_non_taxable", - "export", - "customer_exempt", - "product_exempt", - "zero_rated", - "reverse_charge", - "high_value_physical_goods", - "zero_value_item" - ] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "customer_id": { "type": ["string", "null"] + }, + "metered": { + "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -148,25 +131,19 @@ "type": ["string", "null"] }, "entity_type": { - "type": ["string", "null"], - "enum": [ - "item_level_coupon", - "document_level_coupon", - "promotional_credits", - "prorated_credits", - "item_level_discount", - "document_level_discount" - ] + "type": ["string", "null"] }, "discount_type": { - "type": ["string", "null"], - "enum": ["fixed_amount", "percentage"] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "coupon_set_code": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -180,21 +157,16 @@ "type": ["string", "null"] }, "discount_type": { - "type": ["string", "null"], - "enum": [ - "item_level_coupon", - "document_level_coupon", - "promotional_credits", - "prorated_credits", - "item_level_discount", - "document_level_discount" - ] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] }, "discount_amount": { "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -243,17 +215,7 @@ "type": ["integer", "null"] }, "tax_juris_type": { - "type": ["string", "null"], - "enum": [ - "country", - "federal", - "state", - "county", - "city", - "special", - "unincorporated", - "other" - ] + "type": ["string", "null"] }, "tax_juris_name": { "type": ["string", "null"] @@ -270,19 +232,11 @@ } } }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/shared/_definitions.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/shared/_definitions.json new file mode 100644 index 000000000000..0a1f15a6e1a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/shared/_definitions.json @@ -0,0 +1,18 @@ +{ + "definitions": { + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json new file mode 100644 index 000000000000..557bb96391c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Site Migration Detail", + "type": "object", + "properties": { + "entity_id": { + "type": ["string", "null"] + }, + "other_site_name": { + "type": ["string", "null"] + }, + "entity_id_at_other_site": { + "type": ["string", "null"] + }, + "migrated_at": { + "type": ["integer", "null"] + }, + "entity_type": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "$ref": "_definitions.json#/definitions/custom_fields" + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json index 4e7761b5d99e..336b90019484 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json @@ -4,12 +4,10 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "start_date": { "type": ["integer", "null"] @@ -18,24 +16,19 @@ "type": ["integer", "null"] }, "remaining_billing_cycles": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "po_number": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "plan_quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "plan_unit_price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -83,8 +76,7 @@ "type": ["string", "null"] }, "created_from_ip": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "resource_version": { "type": ["integer", "null"] @@ -99,16 +91,13 @@ "type": ["boolean", "null"] }, "payment_source_id": { - "type": ["string", "null"], - "maxLength": 40 + "type": ["string", "null"] }, "plan_free_quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "plan_amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "cancel_schedule_created_at": { "type": ["integer", "null"] @@ -120,23 +109,19 @@ "type": ["integer", "null"] }, "total_dues": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "mrr": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "exchange_rate": { "type": ["number", "null"] }, "base_currency_code": { - "type": ["string", "null"], - "maxLength": 3 + "type": ["string", "null"] }, "invoice_notes": { - "type": ["string", "null"], - "maxLength": 2000 + "type": ["string", "null"] }, "metadata": { "type": ["object", "null"], @@ -152,8 +137,7 @@ "type": ["string", "null"] }, "cancel_reason_code": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "free_period": { "type": ["integer", "null"] @@ -167,14 +151,19 @@ "auto_close_invoices": { "type": ["boolean", "null"] }, + "business_entity_id": { + "type": ["string", "null"] + }, + "channel": { + "type": ["string", "null"] + }, "coupons": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "coupon_id": { - "type": "string", - "maxLength": 50 + "type": "string" }, "apply_till": { "type": ["integer", "null"] @@ -183,8 +172,13 @@ "type": ["integer", "null"] }, "coupon_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] + }, + "applied_count": { + "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -193,56 +187,43 @@ "type": ["object", "null"], "properties": { "first_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "last_name": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "email": { - "type": ["string", "null"], - "maxLength": 70 + "type": ["string", "null"] }, "company": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "phone": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "line1": { - "type": ["string", "null"], - "maxLength": 180 + "type": ["string", "null"] }, "line2": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "line3": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "city": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "state": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "country": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "zip": { - "type": ["string", "null"], - "maxLength": 20 + "type": ["string", "null"] }, "validation_status": { "type": ["string", "null"] @@ -253,20 +234,16 @@ "type": ["object", "null"], "properties": { "referral_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "coupon_code": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "referral_id": { - "type": ["string", "null"], - "maxLength": 19 + "type": ["string", "null"] }, "external_reference_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "reward_status": { "type": ["string", "null"] @@ -275,16 +252,13 @@ "type": ["string", "null"] }, "account_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "campaign_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "external_campaign_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "friend_offer_type": { "type": ["string", "null"] @@ -296,8 +270,7 @@ "type": ["string", "null"] }, "destination_url": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "post_purchase_widget_enabled": { "type": ["boolean", "null"] @@ -308,8 +281,7 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -321,15 +293,13 @@ "type": ["integer", "null"] }, "billing_cycle": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "action_at_term_end": { "type": ["string", "null"] }, "total_contract_value": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "cancellation_cutoff_period": { "type": ["integer", "null"] @@ -338,71 +308,56 @@ "type": ["integer", "null"] }, "subscription_id": { - "type": ["string", "null"], - "maxLength": 50 + "type": ["string", "null"] }, "remaining_billing_cycles": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] } } }, - "subscription_items": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "item_price_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "item_type": { "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "unit_price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unit_price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "free_quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "free_quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "trial_end": { "type": ["integer", "null"] }, "billing_cycles": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "service_period_days": { - "type": ["integer", "null"], - "minimum": 0, - "maximum": 730 + "type": ["integer", "null"] }, "charge_on_event": { "type": ["string", "null"] @@ -412,6 +367,9 @@ }, "charge_on_option": { "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -422,31 +380,28 @@ "type": ["object", "null"], "properties": { "item_price_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "starting_unit": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "ending_unit": { "type": ["integer", "null"] }, "price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "starting_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "ending_unit_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] } } } @@ -457,35 +412,31 @@ "type": ["object", "null"], "properties": { "item_price_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "last_charged_at": { "type": ["integer", "null"] + }, + "object": { + "type": ["string", "null"] } } } }, - "plan_id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "plan_quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "plan_unit_price": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "setup_fee": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "billing_period": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "billing_period_unit": { "type": ["string", "null"] @@ -494,20 +445,16 @@ "type": ["string", "null"] }, "plan_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "plan_free_quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "gift_id": { - "type": ["string", "null"], - "maxLength": 150 + "type": ["string", "null"] }, "affiliate_token": { - "type": ["string", "null"], - "maxLength": 250 + "type": ["string", "null"] }, "offline_payment_method": { "type": ["string", "null"] @@ -522,39 +469,31 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 1 + "type": ["integer", "null"] }, "unit_price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "trial_end": { "type": ["integer", "null"] }, "remaining_billing_cycles": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "unit_price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "amount_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } @@ -565,16 +504,13 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "quantity": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "unit_price": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "on_event": { "type": ["string", "null"] @@ -583,12 +519,10 @@ "type": ["boolean", "null"] }, "quantity_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] }, "unit_price_in_decimal": { - "type": ["string", "null"], - "maxLength": 33 + "type": ["string", "null"] } } } @@ -599,8 +533,7 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "maxLength": 100 + "type": ["string", "null"] }, "last_charged_at": { "type": ["integer", "null"] @@ -620,8 +553,7 @@ "type": ["string", "null"] }, "type": { - "type": ["string", "null"], - "enum": ["fixed_amount", "percentage"] + "type": ["string", "null"] }, "percentage": { "type": ["number", "null"] @@ -633,22 +565,19 @@ "type": ["string", "null"] }, "duration_type": { - "type": ["string", "null"], - "enum": ["one_time", "forever", "limited_period"] + "type": ["string", "null"] }, "period": { "type": ["integer", "null"] }, "period_unit": { - "type": ["string", "null"], - "enum": ["day", "week", "month", "year"] + "type": ["string", "null"] }, "included_in_mrr": { "type": ["boolean", "null"] }, "apply_on": { - "type": ["string", "null"], - "enum": ["invoice_amount", "specific_item_price"] + "type": ["string", "null"] }, "item_price_id": { "type": ["string", "null"] @@ -672,18 +601,7 @@ } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json index 4237fe31f793..3a54a5e2782c 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json @@ -4,31 +4,25 @@ "type": "object", "properties": { "id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "customer_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "subscription_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "gateway_account_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "payment_source_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "payment_method": { "type": ["string", "null"] }, "refrence_number": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "gateway": { "type": ["string", "null"] @@ -52,8 +46,7 @@ "type": ["integer", "null"] }, "id_at_gateway": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -64,19 +57,17 @@ "initiator_type": { "type": ["string", "null"] }, - "three_d_source": { + "three_d_secure": { "type": ["boolean", "null"] }, "authorization_reason": { "type": ["string", "null"] }, "error_code": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "error-text": { - "type": ["string", "null"], - "max-length": 65000 + "type": ["string", "null"] }, "voided_at": { "type": ["integer", "null"] @@ -88,49 +79,58 @@ "type": ["integer", "null"] }, "fraud_reason": { - "type": ["string", "null"], - "max-length": 250 + "type": ["string", "null"] }, "amount_unused": { "type": ["integer", "null"] }, "masked_card_number": { - "type": ["string", "null"], - "max-length": 20 + "type": ["string", "null"] }, "reference_transaction_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "refunded_txn_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "reference_authorization_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "amount_capturable": { "type": ["integer", "null"] }, "reversal_transaction_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "deleted": { "type": ["boolean", "null"] }, "iin": { - "type": ["string", "null"], - "max-length": 6 + "type": ["string", "null"] }, "last4": { - "type": ["string", "null"], - "max-length": 4 + "type": ["string", "null"] }, "merchant_reference_id": { - "type": ["string", "null"], - "max-length": 500 + "type": ["string", "null"] + }, + "base_currency_code": { + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "error_text": { + "type": ["string", "null"] + }, + "payment_method_details": { + "type": ["string", "null"] + }, + "reference_number": { + "type": ["string", "null"] }, "linked_invoices": { "type": ["array", "null"], @@ -138,8 +138,7 @@ "type": ["object", "null"], "properties": { "invoice_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "applied_amount": { "type": ["integer", "null"] @@ -165,8 +164,7 @@ "type": ["object", "null"], "properties": { "cn_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "applied_amount": { "type": ["integer", "null"] @@ -181,8 +179,7 @@ } }, "cn_create_reason_code": { - "type": ["string", "null"], - "max-length": 100 + "type": ["string", "null"] }, "cn_date": { "type": ["integer", "null"] @@ -194,8 +191,7 @@ "type": ["string", "null"] }, "cn_reference_invoice_id": { - "type": ["string", "null"], - "max-length": 50 + "type": ["string", "null"] }, "linked_refunds": { "type": ["array", "null"], @@ -203,8 +199,7 @@ "type": ["object", "null"], "properties": { "txn_id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "txn_status": { "type": ["string", "null"] @@ -224,8 +219,7 @@ "type": ["object", "null"], "properties": { "id": { - "type": ["string", "null"], - "max-length": 40 + "type": ["string", "null"] }, "status": { "type": ["string", "null"] @@ -234,18 +228,7 @@ } }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json index 37c94c393aab..0a7a91e47e34 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json @@ -22,8 +22,7 @@ "type": ["integer", "null"] }, "pricing_model": { - "type": ["string", "null"], - "enum": ["flat_fee", "per_unit", "tiered", "volume", "stairstep"] + "type": ["string", "null"] }, "quantity": { "type": ["integer", "null"] @@ -41,13 +40,7 @@ "type": ["string", "null"] }, "entity_type": { - "type": ["string", "null"], - "enum": [ - "adhoc", - "plan_item_price", - "addon_item_price", - "charge_item_price" - ] + "type": ["string", "null"] }, "entity_id": { "type": ["string", "null"] @@ -105,19 +98,11 @@ } } }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json index daaa14b122fa..af2570f8ebb2 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json @@ -13,8 +13,7 @@ "type": ["string", "null"] }, "scheme": { - "type": ["string", "null"], - "enum": ["ach_credit", "sepa_credit"] + "type": ["string", "null"] }, "bank_name": { "type": ["string", "null"] @@ -29,54 +28,7 @@ "type": ["string", "null"] }, "gateway": { - "type": ["string", "null"], - "enum": [ - "chargebee", - "chargebee_payments", - "stripe", - "wepay", - "braintree", - "authorize_net", - "paypal_pro", - "pin", - "eway", - "eway_rapid", - "worldpay", - "balanced_payments", - "beanstream", - "bluepay", - "elavon", - "first_data_global", - "hdfc", - "migs", - "nmi", - "ogone", - "paymill", - "paypal_payflow_pro", - "sage_pay", - "tco", - "wirecard", - "amazon_payments", - "paypal_express_checkout", - "gocardless", - "adyen", - "orbital", - "moneris_us", - "moneris", - "bluesnap", - "cybersource", - "vantiv", - "checkout_com", - "paypal", - "ingenico_direct", - "exact", - "mollie", - "quickbooks", - "razorpay", - "global_payments", - "bank_of_america", - "not_applicable" - ] + "type": ["string", "null"] }, "resource_version": { "type": ["integer", "null"] @@ -94,18 +46,7 @@ "type": ["boolean", "null"] }, "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } + "$ref": "_definitions.json#/definitions/custom_fields" } } } diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml index 0f8e20d15d36..34e151cf6c6c 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml @@ -7,7 +7,6 @@ connectionSpecification: - site - site_api_key - start_date - - product_catalog additionalProperties: true properties: site_api_key: @@ -27,7 +26,7 @@ connectionSpecification: type: string format: date-time title: Start Date - description: UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated. + description: UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ examples: - "2021-01-25T00:00:00Z" @@ -35,6 +34,7 @@ connectionSpecification: product_catalog: type: string title: Product Catalog - description: Product Catalog version of your Chargebee site. Instructions on how to find your version you may find here under `API Version` section. + description: Product Catalog version of your Chargebee site. Instructions on how to find your version you may find here under `API Version` section. If left blank, the product catalog version will be set to 2.0. enum: ["1.0", "2.0"] + default: "2.0" order: 3 diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py index dcca547a7d16..2d7752d6dabf 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py @@ -2,8 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any, MutableMapping + import pytest -from source_chargebee.components import CustomFieldTransformation +from source_chargebee.components import CustomFieldTransformation, IncrementalSingleSliceCursor @pytest.mark.parametrize( @@ -27,3 +29,59 @@ def test_field_transformation(record, expected_record): transformer = CustomFieldTransformation() transformed_record = transformer.transform(record) assert transformed_record == expected_record + +def test_slicer(): + date_time_dict = {"updated_at": 1662459010} + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") + slicer.close_slice(date_time_dict, date_time_dict) + assert slicer.get_stream_state() == date_time_dict + assert slicer.get_request_headers() == {} + assert slicer.get_request_body_data() == {} + assert slicer.get_request_params() == {} + assert slicer.get_request_body_json() == {} + +@pytest.mark.parametrize( + "first_record, second_record, expected", + [ + ({"pk": 1, "name": "example", "updated_at": 1662459010}, + {"pk": 2, "name": "example2", "updated_at": 1662460000}, + True), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, + {"pk": 2, "name": "example2", "updated_at": 1662440000}, + False), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, + {"pk": 2, "name": "example2"}, + False), + ({"pk": 1, "name": "example"}, + {"pk": 2, "name": "example2", "updated_at": 1662459010}, + True), + ] +) +def test_is_greater_than_or_equal(first_record, second_record, expected): + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") + assert slicer.is_greater_than_or_equal(second_record, first_record) == expected + +def test_set_initial_state(): + cursor_field = "updated_at" + cursor_value = 999999999 + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field=cursor_field) + slicer.set_initial_state(stream_state={cursor_field: cursor_value}) + assert slicer._state[cursor_field] == cursor_value + +@pytest.mark.parametrize( + "record, expected", + [ + ({"pk": 1, "name": "example", "updated_at": 1662459010}, + True), + ] +) +def test_should_be_synced(record, expected): + cursor_field = "updated_at" + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field=cursor_field) + assert slicer.should_be_synced(record) == expected + +def test_stream_slices(): + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") + stream_slices_instance = slicer.stream_slices() + actual = next(stream_slices_instance) + assert actual == {} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_source.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_source.py new file mode 100644 index 000000000000..62feab67ad1d --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_source.py @@ -0,0 +1,9 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +from source_chargebee import SourceChargebee + + +def test_source(): + assert SourceChargebee() \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-chargify/main.py b/airbyte-integrations/connectors/source-chargify/main.py index 1a4568ff615b..44e0f1156271 100644 --- a/airbyte-integrations/connectors/source-chargify/main.py +++ b/airbyte-integrations/connectors/source-chargify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chargify import SourceChargify +from source_chargify.run import run if __name__ == "__main__": - source = SourceChargify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chargify/metadata.yaml b/airbyte-integrations/connectors/source-chargify/metadata.yaml index f211a8bfecbc..ea6cf3005986 100644 --- a/airbyte-integrations/connectors/source-chargify/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargify/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${domain} + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chargify registries: oss: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/chargify tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-chargify/setup.py b/airbyte-integrations/connectors/source-chargify/setup.py index 521c61646023..deda5abcc0b0 100644 --- a/airbyte-integrations/connectors/source-chargify/setup.py +++ b/airbyte-integrations/connectors/source-chargify/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-chargify=source_chargify.run:run", + ], + }, name="source_chargify", description="Source implementation for Chargify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/run.py b/airbyte-integrations/connectors/source-chargify/source_chargify/run.py new file mode 100644 index 000000000000..88f4450c9bcf --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chargify import SourceChargify + + +def run(): + source = SourceChargify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chartmogul/main.py b/airbyte-integrations/connectors/source-chartmogul/main.py index 5dc03e8a74a9..bf13dab9878d 100644 --- a/airbyte-integrations/connectors/source-chartmogul/main.py +++ b/airbyte-integrations/connectors/source-chartmogul/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chartmogul import SourceChartmogul +from source_chartmogul.run import run if __name__ == "__main__": - source = SourceChartmogul() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml index 42e6a35f9b9f..3a68eecb886c 100644 --- a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml @@ -20,6 +20,10 @@ data: icon: chartmogul.svg license: MIT name: Chartmogul + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chartmogul registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py index fa0d73f436c9..624ab8c53ba2 100644 --- a/airbyte-integrations/connectors/source-chartmogul/setup.py +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-chartmogul=source_chartmogul.run:run", + ], + }, name="source_chartmogul", description="Source implementation for Chartmogul.", author="Titas Skrebe", author_email="titas@omnisend.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py new file mode 100644 index 000000000000..f8e5bbd30e63 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chartmogul import SourceChartmogul + + +def run(): + source = SourceChartmogul() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index c4d71fd3e5b9..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-clickhouse-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle index a6c2dfa8b3d7..5941a31d383a 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -26,8 +16,7 @@ application { dependencies { implementation project(':airbyte-integrations:connectors:source-clickhouse') - implementation group: 'com.clickhouse', name: 'clickhouse-jdbc', version: '0.3.2-patch9' + implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') - integrationTestJavaImplementation libs.testcontainers.clickhouse + testImplementation 'org.testcontainers:clickhouse:1.19.4' } diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml index beeb8dd90751..3f67d3b3ead3 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c - dockerImageTag: 0.1.17 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-clickhouse-strict-encrypt githubIssueLabel: source-clickhouse icon: clickhouse.svg diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index 8365085a3277..63a644f1fd1a 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -4,54 +4,50 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static io.airbyte.integrations.io.airbyte.integration_tests.sources.ClickHouseStrictEncryptTestDatabase.DEFAULT_DB_NAME; +import static io.airbyte.integrations.io.airbyte.integration_tests.sources.ClickHouseStrictEncryptTestDatabase.HTTPS_PORT; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.factory.DataSourceFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.ssh.SshHelpers; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.sql.JDBCType; import java.time.Duration; import java.util.List; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; -public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest + extends JdbcSourceAcceptanceTest { public static final Integer HTTP_PORT = 8123; public static final Integer NATIVE_PORT = 9000; - public static final Integer HTTPS_PORT = 8443; + public static final Integer NATIVE_SECURE_PORT = 9440; - private static final String DEFAULT_DB_NAME = "default"; - private static final String DEFAULT_USER_NAME = "default"; - private static GenericContainer container; - private static JdbcDatabase db; - private JsonNode config; - private String dbName; + @Override + protected ClickHouseStrictEncryptTestDatabase createTestDatabase() { + final ClickHouseContainer db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .withEnv("TZ", "UTC") + .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) + .withCopyFileToContainer(MountableFile.forClasspathResource("/docker/clickhouse_certs.sh"), + "/docker-entrypoint-initdb.d/clickhouse_certs.sh") + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + db.start(); + return new ClickHouseStrictEncryptTestDatabase(db).initialized(); + } @Override public boolean supportsSchemas() { @@ -59,13 +55,13 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public String getDriverClass() { - return ClickHouseSource.DRIVER_CLASS; + protected ClickHouseStrictEncryptSource source() { + return new ClickHouseStrictEncryptSource(); } @Override @@ -75,7 +71,7 @@ public String createTableQuery(final String tableName, // ClickHouse requires Engine to be mentioned as part of create table query. // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information return String.format("CREATE TABLE %s(%s) %s", - dbName + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + DEFAULT_DB_NAME + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + primaryKeyClause); } @@ -83,60 +79,9 @@ public String createTableQuery(final String tableName, @BeforeAll static void init() { CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);)"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);"; CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; - - container = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") - .withFileFromClasspath("Dockerfile", "docker/Dockerfile") - .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) - .withEnv("TZ", "UTC") - .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) - .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) - .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); - container.start(); - } - - @BeforeEach - public void setup() throws Exception { - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(container)) - .put(JdbcUtils.PORT_KEY, HTTPS_PORT) - .put(JdbcUtils.USERNAME_KEY, DEFAULT_USER_NAME) - .put("database", DEFAULT_DB_NAME) - .put(JdbcUtils.PASSWORD_KEY, "") - .build()); - - db = new DefaultJdbcDatabase( - DataSourceFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - ClickHouseSource.DRIVER_CLASS, - String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString() + "?sslmode=none", - ClickHouseSource.HTTPS_PROTOCOL, - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt(), - configWithoutDbName.get("database").asText()))); - - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - db.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - - super.setup(); - } - - @AfterEach - public void tearDownMySql() throws Exception { - db.execute(ctx -> ctx.createStatement().execute(String.format("DROP DATABASE %s;", dbName))); - super.tearDown(); - } - - @AfterAll - public static void cleanUp() throws Exception { - container.close(); } @Override @@ -157,19 +102,9 @@ public String primaryKeyClause(final List columns) { return clause.toString(); } - @Override - public AbstractJdbcSource getJdbcSource() { - return new ClickHouseSource(); - } - - @Override - public Source getSource() { - return new ClickHouseStrictEncryptSource(); - } - @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java new file mode 100644 index 000000000000..ffadf1685ca7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.clickhouse.ClickHouseContainer; + +public class ClickHouseStrictEncryptTestDatabase extends + TestDatabase { + + private static final String SCHEMA_NAME = "default"; + public static final Integer HTTPS_PORT = 8443; + public static final String DEFAULT_DB_NAME = "default"; + private static final String DEFAULT_USER_NAME = "default"; + private final ClickHouseContainer container; + + protected ClickHouseStrictEncryptTestDatabase(final ClickHouseContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return SCHEMA_NAME; + } + + @Override + public ClickHouseConfigBuilder configBuilder() { + return new ClickHouseConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getMappedPort(HTTPS_PORT)) + .with(JdbcUtils.USERNAME_KEY, DEFAULT_USER_NAME) + .with(JdbcUtils.DATABASE_KEY, DEFAULT_DB_NAME) + .with(JdbcUtils.PASSWORD_KEY, ""); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.CLICKHOUSE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + container.close(); + } + + @Override + public ClickHouseConfigBuilder integrationTestConfigBuilder() { + return super.integrationTestConfigBuilder(); + } + + static public class ClickHouseConfigBuilder extends ConfigBuilder { + + protected ClickHouseConfigBuilder(final ClickHouseStrictEncryptTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml deleted file mode 100644 index a329f4e6b0db..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-clickhouse:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index ae4d2a7b12ca..4e16af080d96 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -1,31 +1,20 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' - integrationTestJavaImplementation libs.testcontainers.clickhouse + testImplementation 'org.testcontainers:clickhouse:1.19.4' } diff --git a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml index 07806c1237f3..6a6416ef1660 100644 --- a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c - dockerImageTag: 0.1.17 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-clickhouse documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse githubIssueLabel: source-clickhouse @@ -18,7 +18,7 @@ data: name: ClickHouse registries: cloud: - dockerImageTag: 0.1.8 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-clickhouse-strict-encrypt enabled: true oss: diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index e2219bd282b0..9e7ad5a46345 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -31,10 +31,12 @@ import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; +@Disabled public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { private ClickHouseContainer db; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 8a294e339ea9..cee17bd1b6db 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -11,29 +11,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; -import java.sql.JDBCType; -import java.sql.SQLException; import java.time.Duration; import java.util.List; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; -public class ClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +public class ClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final String SCHEMA_NAME = "default"; - private ClickHouseContainer db; - private JsonNode config; + @BeforeAll + static void init() { + CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);"; + CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; + INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; + } @Override public boolean supportsSchemas() { @@ -41,13 +39,17 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + protected JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public String getDriverClass() { - return ClickHouseSource.DRIVER_CLASS; + protected ClickHouseTestDatabase createTestDatabase() { + final ClickHouseContainer db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + db.start(); + return new ClickHouseTestDatabase(db).initialized(); } @Override @@ -60,22 +62,6 @@ public String createTableQuery(final String tableName, final String columnClause + primaryKeyClause); } - @BeforeAll - static void init() { - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);)"; - CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; - } - - @Override - @AfterEach - public void tearDown() throws SQLException { - db.close(); - db.stop(); - super.tearDown(); - } - @Override public String primaryKeyClause(final List columns) { if (columns.isEmpty()) { @@ -95,27 +81,7 @@ public String primaryKeyClause(final List columns) { } @Override - @BeforeEach - public void setup() throws Exception { - db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") - .waitingFor(Wait.forHttp("/ping").forPort(8123) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); - db.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, SCHEMA_NAME) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - super.setup(); - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected ClickHouseSource source() { return new ClickHouseSource(); } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index d9f0772777bb..69950397b24d 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -30,9 +30,11 @@ import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; +@Disabled public class ClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { private ClickHouseContainer db; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java new file mode 100644 index 000000000000..35bc1595e6ff --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.clickhouse.ClickHouseContainer; + +public class ClickHouseTestDatabase extends + TestDatabase { + + private static final String SCHEMA_NAME = "default"; + + private final ClickHouseContainer container; + + protected ClickHouseTestDatabase(final ClickHouseContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return SCHEMA_NAME; + } + + @Override + public ClickHouseConfigBuilder configBuilder() { + return new ClickHouseConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with(JdbcUtils.DATABASE_KEY, SCHEMA_NAME) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SSL_KEY, false); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.CLICKHOUSE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + container.close(); + } + + @Override + public ClickHouseConfigBuilder integrationTestConfigBuilder() { + return super.integrationTestConfigBuilder(); + } + + static public class ClickHouseConfigBuilder extends TestDatabase.ConfigBuilder { + + protected ClickHouseConfigBuilder(final ClickHouseTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java index 648836c51c51..ebcb3f632c67 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshKeyClickhouseSourceAcceptanceTest extends AbstractSshClickHouseSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java index c394da9c0d97..7223031d1735 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshPasswordClickhouseSourceAcceptanceTest extends AbstractSshClickHouseSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index ca1eb6116551..1359111fca8a 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -19,8 +19,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; +@Disabled public class SslClickHouseJdbcSourceAcceptanceTest extends ClickHouseJdbcSourceAcceptanceTest { private static GenericContainer container; diff --git a/airbyte-integrations/connectors/source-clickup-api/main.py b/airbyte-integrations/connectors/source-clickup-api/main.py index 01d656f16c43..76c09b38846f 100644 --- a/airbyte-integrations/connectors/source-clickup-api/main.py +++ b/airbyte-integrations/connectors/source-clickup-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_clickup_api import SourceClickupApi +from source_clickup_api.run import run if __name__ == "__main__": - source = SourceClickupApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml index a59f15841d5d..dc76068faa84 100644 --- a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: clickup.svg license: MIT name: ClickUp + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-clickup-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-clickup-api/setup.py b/airbyte-integrations/connectors/source-clickup-api/setup.py index 4fb2367a7516..b23f4ed53a1a 100644 --- a/airbyte-integrations/connectors/source-clickup-api/setup.py +++ b/airbyte-integrations/connectors/source-clickup-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-clickup-api=source_clickup_api.run:run", + ], + }, name="source_clickup_api", description="Source implementation for Clickup Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py b/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py new file mode 100644 index 000000000000..3767b66395e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clickup_api import SourceClickupApi + + +def run(): + source = SourceClickupApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clockify/main.py b/airbyte-integrations/connectors/source-clockify/main.py index 9f04d4eb176c..486525f0e293 100644 --- a/airbyte-integrations/connectors/source-clockify/main.py +++ b/airbyte-integrations/connectors/source-clockify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_clockify import SourceClockify +from source_clockify.run import run if __name__ == "__main__": - source = SourceClockify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-clockify/metadata.yaml b/airbyte-integrations/connectors/source-clockify/metadata.yaml index f26a0e6f214c..be8435177779 100644 --- a/airbyte-integrations/connectors/source-clockify/metadata.yaml +++ b/airbyte-integrations/connectors/source-clockify/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.clockify.me + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-clockify registries: oss: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/clockify tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py index bcd38b28c29a..940c87ba74b4 100644 --- a/airbyte-integrations/connectors/source-clockify/setup.py +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-clockify=source_clockify.run:run", + ], + }, name="source_clockify", description="Source implementation for Clockify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/run.py b/airbyte-integrations/connectors/source-clockify/source_clockify/run.py new file mode 100644 index 000000000000..ef88995f79dd --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clockify import SourceClockify + + +def run(): + source = SourceClockify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-close-com/Dockerfile b/airbyte-integrations/connectors/source-close-com/Dockerfile index e77535415cb0..44603bb80be5 100644 --- a/airbyte-integrations/connectors/source-close-com/Dockerfile +++ b/airbyte-integrations/connectors/source-close-com/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.3 +LABEL io.airbyte.version=0.5.0 LABEL io.airbyte.name=airbyte/source-close-com diff --git a/airbyte-integrations/connectors/source-close-com/main.py b/airbyte-integrations/connectors/source-close-com/main.py index 22787dfbd89e..f80e76315939 100644 --- a/airbyte-integrations/connectors/source-close-com/main.py +++ b/airbyte-integrations/connectors/source-close-com/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_close_com import SourceCloseCom +from source_close_com.run import run if __name__ == "__main__": - source = SourceCloseCom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-close-com/metadata.yaml b/airbyte-integrations/connectors/source-close-com/metadata.yaml index 97847eafefac..ad645a347058 100644 --- a/airbyte-integrations/connectors/source-close-com/metadata.yaml +++ b/airbyte-integrations/connectors/source-close-com/metadata.yaml @@ -8,13 +8,17 @@ data: connectorSubtype: api connectorType: source definitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9 - dockerImageTag: 0.4.3 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-close-com documentationUrl: https://docs.airbyte.com/integrations/sources/close-com githubIssueLabel: source-close-com icon: close.svg license: MIT name: Close.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-close-com registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-close-com/setup.py b/airbyte-integrations/connectors/source-close-com/setup.py index b9d9aaf53e7d..a6ad55159cd5 100644 --- a/airbyte-integrations/connectors/source-close-com/setup.py +++ b/airbyte-integrations/connectors/source-close-com/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-close-com=source_close_com.run:run", + ], + }, name="source_close_com", description="Source implementation for Close.com.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py index 290f7d5f74e0..26f244576b38 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py @@ -22,6 +22,6 @@ from .datetime_incremental_sync import CustomDatetimeIncrementalSync -from .source_lc import SourceCloseCom +from .source import SourceCloseCom __all__ = ["SourceCloseCom", "CustomDatetimeIncrementalSync"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/run.py b/airbyte-integrations/connectors/source-close-com/source_close_com/run.py new file mode 100644 index 000000000000..eb80c6eab53e --- /dev/null +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_close_com import SourceCloseCom + + +def run(): + source = SourceCloseCom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py index 2fe9c2f85ab8..9754c114bbd8 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py @@ -58,7 +58,6 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {} if self.number_of_items_per_page: params.update({"_limit": self.number_of_items_per_page}) @@ -87,8 +86,24 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return backoff_time -class IncrementalCloseComStream(CloseComStream): +class CloseComStreamCustomFields(CloseComStream): + """Class to get custom fields for close objects that support them.""" + + def get_custom_field_schema(self) -> Mapping[str, Any]: + """Get custom field schema if it exists.""" + resp = requests.request("GET", url=f"{self.url_base}/custom_field/{self.path()}/", headers=self.authenticator.get_auth_header()) + resp.raise_for_status() + resp_json: Mapping[str, Any] = resp.json()["data"] + return {f"custom.{data['id']}": {"type": ["null", "string", "number", "boolean"]} for data in resp_json} + + def get_json_schema(self): + """Override default get_json_schema method to add custom fields to schema.""" + schema = super().get_json_schema() + schema["properties"].update(self.get_custom_field_schema()) + return schema + +class IncrementalCloseComStream(CloseComStream): cursor_field = "date_updated" def get_updated_state( @@ -105,6 +120,10 @@ def get_updated_state( return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} +class IncrementalCloseComStreamCustomFields(CloseComStreamCustomFields, IncrementalCloseComStream): + """Class to get custom fields for close objects using incremental stream.""" + + class CloseComActivitiesStream(IncrementalCloseComStream): """ General class for activities. Define request params based on cursor_field value. @@ -233,7 +252,7 @@ def request_params(self, stream_state=None, **kwargs): return params -class Leads(IncrementalCloseComStream): +class Leads(IncrementalCloseComStreamCustomFields): """ Get leads on a specific date API Docs: https://developer.close.com/#leads @@ -404,7 +423,7 @@ def path(self, **kwargs) -> str: return "user" -class Contacts(CloseComStream): +class Contacts(CloseComStreamCustomFields): """ Get contacts for Close.com account organization API Docs: https://developer.close.com/#contacts @@ -416,7 +435,7 @@ def path(self, **kwargs) -> str: return "contact" -class Opportunities(IncrementalCloseComStream): +class Opportunities(IncrementalCloseComStreamCustomFields): """ Get opportunities on a specific date API Docs: https://developer.close.com/#opportunities diff --git a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml deleted file mode 100644 index 56c47e4288c9..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-cockroachdb:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index 18e96169143e..e1185f883ef0 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -1,36 +1,20 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { + implementation 'org.postgresql:postgresql:42.6.0' - implementation 'org.apache.commons:commons-lang3:3.11' - implementation libs.postgresql - - testImplementation libs.testcontainers.cockroachdb - testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.testcontainers:cockroachdb:1.19.4' } diff --git a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml index 8ef95ce6b9e1..a64ed788d537 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml +++ b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 - dockerImageTag: 0.1.22 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-cockroachdb githubIssueLabel: source-cockroachdb icon: cockroachdb.svg diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java index 7fd590a209fb..9cf188fd497e 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java @@ -27,6 +27,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -91,6 +92,7 @@ public Set getExcludedInternalNameSpaces() { } @Override + @SuppressWarnings("unchecked") public Set getPrivilegesTableForCurrentUser(final JdbcDatabase database, final String schema) throws SQLException { try (final Stream stream = database.unsafeQuery(getPrivileges(database), sourceOperations::rowToJson)) { return stream.map(this::getPrivilegeDto).collect(Collectors.toSet()); @@ -105,14 +107,15 @@ protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); - + final Map connectionProperties = JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY); // Create the JDBC data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); dataSources.add(dataSource); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource, sourceOperations); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java index 5036469dadc7..5a69a4b83c67 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java @@ -15,6 +15,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.function.Function; import java.util.stream.Stream; /** @@ -81,4 +82,9 @@ public Stream unsafeQuery(final String sql, final String... params) th } + @Override + public T executeMetadataQuery(Function function) throws SQLException { + return database.executeMetadataQuery(function); + } + } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java index b1ea109784f9..52829fa17be3 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java @@ -27,8 +27,10 @@ import java.util.Objects; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.CockroachContainer; +@Disabled public class CockroachDbSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "public.id_and_name"; @@ -57,7 +59,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.SSL_KEY, false) .build()); - try (final DSLContext dslContext = DSLContextFactory.create( + final DSLContext dslContext = DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.POSTGRESQL.getDriverClassName(), @@ -65,19 +67,19 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } + SQLDialect.POSTGRES); + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } @Override diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index 28d4564d0e0e..5ebbf01cd858 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -20,10 +20,12 @@ import java.util.Set; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.CockroachContainer; +@Disabled public class CockroachDbSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private CockroachContainer container; @@ -85,7 +87,6 @@ protected JsonNode getConfig() { @Override protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); container.close(); } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java index db5eb5fe3b06..932d7fb1cace 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java @@ -12,43 +12,26 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.cdk.db.factory.DataSourceFactory; -import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.JDBCType; import java.util.*; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.CockroachContainer; -class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static CockroachContainer PSQL_DB; public static String COL_ROW_ID = "rowid"; public static Long ID_VALUE_1 = 1L; @@ -57,77 +40,34 @@ class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { public static Long ID_VALUE_4 = 4L; public static Long ID_VALUE_5 = 5L; - private JsonNode config; - private String dbName; - - @BeforeAll - static void init() { - PSQL_DB = new CockroachContainer("cockroachdb/cockroach:v20.2.18"); - PSQL_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(PSQL_DB.getContainerInfo() - .getNetworkSettings() - .getNetworks() - .entrySet().stream() - .findFirst() - .get().getValue().getIpAddress())) - .put(JdbcUtils.PORT_KEY, PSQL_DB.getExposedPorts().get(1)) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, PSQL_DB.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, PSQL_DB.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - final JsonNode clone = Jsons.clone(config); - ((ObjectNode) clone).put("database", PSQL_DB.getDatabaseName()); - final JsonNode jdbcConfig = getToDatabaseConfigFunction().apply(clone); - - database = new DefaultJdbcDatabase( - DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - getDriverClass(), - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY))); - database.execute(connection -> connection.createStatement().execute("CREATE DATABASE " + dbName + ";")); - super.setup(); - } + static final String DB_NAME = "postgres"; @Override protected String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { - return String.format("CREATE TABLE " + dbName + ".%s(%s %s %s)", + return String.format("CREATE TABLE " + DB_NAME + ".%s(%s %s %s)", tableName, columnClause, primaryKeyClause.equals("") ? "" : ",", primaryKeyClause); } @Override - public boolean supportsSchemas() { - return true; + protected CockroachDbTestDatabase createTestDatabase() { + final CockroachContainer cockroachContainer = new CockroachContainer("cockroachdb/cockroach:v20.2.18"); + cockroachContainer.start(); + return new CockroachDbTestDatabase(cockroachContainer).initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { - return new CockroachDbSource(); + public boolean supportsSchemas() { + return true; } @Override - public JsonNode getConfig() { - return config; + protected CockroachDbSource source() { + return new CockroachDbSource(); } @Override - public String getDriverClass() { - return CockroachDbSource.DRIVER_CLASS; - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override @@ -163,32 +103,33 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List getTestMessages() { - return Lists.newArrayList( + return List.of( new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", COL_UPDATED_AT, "2006-10-19"))))); } @Test + @Override protected void testDiscoverWithNonCursorFields() throws Exception { /* * this test is not valid for cockroach db, when table has no introduced PK it will add a hidden @@ -199,6 +140,7 @@ protected void testDiscoverWithNonCursorFields() throws Exception { } @Test + @Override protected void testDiscoverWithNullableCursorFields() throws Exception { /* * this test is not valid for cockroach db, when table has no introduced PK it will add a hidden @@ -209,20 +151,23 @@ protected void testDiscoverWithNullableCursorFields() throws Exception { } @Test - void testCheckFailure() throws Exception { + @Override + protected void testCheckFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus actual = source.check(config); + final AirbyteConnectionStatus actual = source().check(config); assertEquals(Status.FAILED, actual.getStatus()); } @Test - void testReadOneColumn() throws Exception { + @Override + protected void testReadOneColumn() throws Exception { final ConfiguredAirbyteCatalog catalog = CatalogHelpers - .createConfiguredAirbyteCatalog(streamName, getDefaultNamespace(), + .createConfiguredAirbyteCatalog(streamName(), getDefaultNamespace(), Field.of(COL_ID, JsonSchemaType.NUMBER)); final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -241,7 +186,8 @@ void testReadOneColumn() throws Exception { } @Test - void testTablesWithQuoting() throws Exception { + @Override + protected void testTablesWithQuoting() throws Exception { final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() @@ -249,7 +195,7 @@ void testTablesWithQuoting() throws Exception { getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0), streamForTableWithSpaces)); final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -274,7 +220,8 @@ void testTablesWithQuoting() throws Exception { } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -285,38 +232,34 @@ void testReadOneTableIncrementallyTwice() throws Exception { final DbState state = new DbState() .withStreams(Lists.newArrayList( - new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); assertEquals(2, (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", @@ -324,11 +267,19 @@ void testReadOneTableIncrementallyTwice() throws Exception { expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursor("5") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -342,29 +293,24 @@ void testReadOneTableIncrementallyTwice() throws Exception { } @Test - void testReadMultipleTables() throws Exception { + @Override + protected void testReadMultipleTables() throws Exception { final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithOneStream( getDefaultNamespace()); final List expectedMessages = new ArrayList<>(getTestMessages()); for (int i = 2; i < 10; i++) { final int iFinal = i; - final String streamName2 = streamName + i; - database.execute(connection -> { - connection.createStatement() - .execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME + iFinal), - "id INTEGER, name VARCHAR(200)", "")); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - }); + final String streamName2 = streamName() + i; + testdb.with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME + iFinal), + "id INTEGER, name VARCHAR(200)", "")) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (1,'picard')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (2, 'crusher')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (3, 'vash')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))); + catalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( streamName2, getDefaultNamespace(), @@ -386,7 +332,7 @@ void testReadMultipleTables() throws Exception { } final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -396,23 +342,18 @@ void testReadMultipleTables() throws Exception { } @Test - void testReadMultipleTablesIncrementally() throws Exception { + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { final String tableName2 = TABLE_NAME + 2; - final String streamName2 = streamName + 2; - database.execute(ctx -> { - ctx.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(tableName2), "id INTEGER, name VARCHAR(200)", - "")); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(tableName2))); - }); + final String streamName2 = streamName() + 2; + testdb.with(createTableQuery(getFullyQualifiedTableName(tableName2), "id INTEGER, name VARCHAR(200)", + "")) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (1,'picard')", + getFullyQualifiedTableName(tableName2))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (2, 'crusher')", + getFullyQualifiedTableName(tableName2))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (3, 'vash')", + getFullyQualifiedTableName(tableName2))); final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream( @@ -430,9 +371,9 @@ void testReadMultipleTablesIncrementally() throws Exception { final DbState state = new DbState() .withStreams(Lists.newArrayList( - new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); // get last state message. final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() @@ -456,12 +397,20 @@ void testReadMultipleTablesIncrementally() throws Exception { expectedMessagesFirstSync.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(namespace).withName(streamName())) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursor("3") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList( new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("3") @@ -475,12 +424,20 @@ void testReadMultipleTablesIncrementally() throws Exception { expectedMessagesFirstSync.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(namespace).withName(streamName2)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursor("3") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList( new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("3") @@ -500,30 +457,19 @@ void testReadMultipleTablesIncrementally() throws Exception { } @Test - void testDiscoverWithMultipleSchemas() throws Exception { - // clickhouse and mysql do not have a concept of schemas, so this test does not make sense for them. - if (getDriverClass().toLowerCase().contains("mysql") || getDriverClass().toLowerCase() - .contains("clickhouse")) { - return; - } - + @Override + protected void testDiscoverWithMultipleSchemas() throws Exception { // add table and data to a separate schema. - database.execute(connection -> { - connection.createStatement().execute( - String.format("CREATE TABLE " + dbName + ".%s(id VARCHAR(200), name VARCHAR(200))", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('1','picard')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('2', 'crusher')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('3', 'vash')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - }); - - final AirbyteCatalog actual = source.discover(config); + testdb.with(String.format("CREATE TABLE " + DB_NAME + ".%s(id VARCHAR(200), name VARCHAR(200))", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('1','picard')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('2', 'crusher')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('3', 'vash')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); + + final AirbyteCatalog actual = source().discover(config()); final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); expected.getStreams().add(CatalogHelpers diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java index 7f47f9e557a0..739f0206fecb 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java @@ -36,9 +36,11 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.CockroachContainer; +@Disabled class CockroachDbSourceTest { private static final String SCHEMA_NAME = "public"; @@ -101,32 +103,31 @@ void setup() throws Exception { dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); final JsonNode config = getConfig(PSQL_DB, null); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + dbName + ";"); - ctx.fetch( - "CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);"); - ctx.fetch( - "INSERT INTO " + dbName - + ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch( - "CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); - ctx.fetch( - "INSERT INTO " + dbName - + ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch( - "CREATE TABLE " + dbName - + ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( - "INSERT INTO " + dbName - + ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - } + final DSLContext dslContext = getDslContext(config); + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE DATABASE " + dbName + ";"); + ctx.fetch( + "CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);"); + ctx.fetch( + "INSERT INTO " + dbName + + ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + + ctx.fetch( + "CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); + ctx.fetch( + "INSERT INTO " + dbName + + ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + + ctx.fetch( + "CREATE TABLE " + dbName + + ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); + ctx.fetch( + "INSERT INTO " + dbName + + ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); + return null; + }); } private static Database getDatabase(final DSLContext dslContext) { @@ -182,15 +183,14 @@ public void testCanReadUtf8() throws Exception { // .withCommand("postgres -c client_encoding=sql_ascii") db.start(); final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');"); - return null; - }); - } + final DSLContext dslContext = getDslContext(config); + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');"); + return null; + }); final Set actualMessages = MoreIterators .toSet(new CockroachDbSource().read(config, CONFIGURED_CATALOG, null)); @@ -223,23 +223,22 @@ void testDiscoverWithPk() throws Exception { @Test void testDiscoverWithPermissions() throws Exception { final JsonNode config = getConfig(PSQL_DB, dbName); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch( - "CREATE USER cock;"); - ctx.fetch( - "CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch( - "CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch( - "CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("grant all on database " + dbName + " to cock;"); - ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;"); - ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;"); - return null; - }); - } + final DSLContext dslContext = getDslContext(config); + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch( + "CREATE USER cock;"); + ctx.fetch( + "CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch( + "CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch( + "CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("grant all on database " + dbName + " to cock;"); + ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;"); + ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;"); + return null; + }); final List expected = List.of("id_and_name_perm1", "id_and_name_perm2"); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java index ccf223ecbd42..92037e322f92 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java @@ -21,12 +21,14 @@ import java.nio.file.Files; import java.nio.file.Path; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** * Tests that the postgres spec passes JsonSchema validation. While this may seem like overkill, we * are doing it because there are some gotchas in correctly configuring the oneOf. */ +@Disabled public class CockroachDbSpecTest { private static final String CONFIGURATION = "{ " diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java new file mode 100644 index 000000000000..0f1f87622a96 --- /dev/null +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.cockroachdb; + +import static io.airbyte.integrations.source.cockroachdb.CockroachDbJdbcSourceAcceptanceTest.DB_NAME; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.CockroachContainer; + +public class CockroachDbTestDatabase extends + TestDatabase { + + private final CockroachContainer container; + + protected CockroachDbTestDatabase(final CockroachContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return DB_NAME; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public void close() { + container.close(); + } + + @Override + public CockroachDbConfigBuilder configBuilder() { + return new CockroachDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getMappedPort(26257)) + .with(JdbcUtils.DATABASE_KEY, DB_NAME) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SSL_KEY, false); + } + + static public class CockroachDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected CockroachDbConfigBuilder(final CockroachDbTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-coda/main.py b/airbyte-integrations/connectors/source-coda/main.py index aae501071c7d..dbc36f9d6886 100644 --- a/airbyte-integrations/connectors/source-coda/main.py +++ b/airbyte-integrations/connectors/source-coda/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coda import SourceCoda +from source_coda.run import run if __name__ == "__main__": - source = SourceCoda() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coda/metadata.yaml b/airbyte-integrations/connectors/source-coda/metadata.yaml index c16c1748fef9..4fcb66aba68c 100644 --- a/airbyte-integrations/connectors/source-coda/metadata.yaml +++ b/airbyte-integrations/connectors/source-coda/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://coda.io/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coda registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-coda/setup.py b/airbyte-integrations/connectors/source-coda/setup.py index 446b5807dcec..92e0b6526fe7 100644 --- a/airbyte-integrations/connectors/source-coda/setup.py +++ b/airbyte-integrations/connectors/source-coda/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coda=source_coda.run:run", + ], + }, name="source_coda", description="Source implementation for Coda.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coda/source_coda/run.py b/airbyte-integrations/connectors/source-coda/source_coda/run.py new file mode 100644 index 000000000000..0a1547db7b47 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coda import SourceCoda + + +def run(): + source = SourceCoda() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coin-api/Dockerfile b/airbyte-integrations/connectors/source-coin-api/Dockerfile index d84bd8a41fba..f6c16b5dc062 100644 --- a/airbyte-integrations/connectors/source-coin-api/Dockerfile +++ b/airbyte-integrations/connectors/source-coin-api/Dockerfile @@ -34,5 +34,5 @@ COPY source_coin_api ./source_coin_api ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-coin-api diff --git a/airbyte-integrations/connectors/source-coin-api/main.py b/airbyte-integrations/connectors/source-coin-api/main.py index 44b7d1ff5d1a..0a62eaad6b92 100644 --- a/airbyte-integrations/connectors/source-coin-api/main.py +++ b/airbyte-integrations/connectors/source-coin-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coin_api import SourceCoinApi +from source_coin_api.run import run if __name__ == "__main__": - source = SourceCoinApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coin-api/metadata.yaml b/airbyte-integrations/connectors/source-coin-api/metadata.yaml index fc65c06c61c2..9da3a7feac04 100644 --- a/airbyte-integrations/connectors/source-coin-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-coin-api/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: 919984ef-53a2-479b-8ffe-9c1ddb9fc3f3 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-coin-api githubIssueLabel: source-coin-api icon: coinapi.svg license: MIT name: Coin API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coin-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-coin-api/setup.py b/airbyte-integrations/connectors/source-coin-api/setup.py index b4c098be7797..904de0b88661 100644 --- a/airbyte-integrations/connectors/source-coin-api/setup.py +++ b/airbyte-integrations/connectors/source-coin-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coin-api=source_coin_api.run:run", + ], + }, name="source_coin_api", description="Source implementation for Coin Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py b/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py new file mode 100644 index 000000000000..d0d729797ece --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coin_api import SourceCoinApi + + +def run(): + source = SourceCoinApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/quotes_historical_data.json b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/quotes_historical_data.json index d050df17a7bb..b361eb1f855c 100644 --- a/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/quotes_historical_data.json +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/quotes_historical_data.json @@ -19,10 +19,10 @@ "ask_size": { "type": ["null", "number"] }, - "big_price": { + "bid_price": { "type": ["null", "number"] }, - "big_size": { + "bid_size": { "type": ["null", "number"] } } diff --git a/airbyte-integrations/connectors/source-coingecko-coins/main.py b/airbyte-integrations/connectors/source-coingecko-coins/main.py index 6ab339e39a73..0317d9bbf40b 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/main.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coingecko_coins import SourceCoingeckoCoins +from source_coingecko_coins.run import run if __name__ == "__main__": - source = SourceCoingeckoCoins() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml index b7619d147910..ac0035a6f5ff 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml +++ b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml @@ -8,6 +8,10 @@ data: icon: coingeckocoins.svg license: MIT name: CoinGecko Coins + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coingecko-coins registries: cloud: enabled: false # Did not pass acceptance tests diff --git a/airbyte-integrations/connectors/source-coingecko-coins/setup.py b/airbyte-integrations/connectors/source-coingecko-coins/setup.py index 39a959dcdc27..efc1c015a4df 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/setup.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coingecko-coins=source_coingecko_coins.run:run", + ], + }, name="source_coingecko_coins", description="Source implementation for Coingecko Coins.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py new file mode 100644 index 000000000000..c652fd9af6d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coingecko_coins import SourceCoingeckoCoins + + +def run(): + source = SourceCoingeckoCoins() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coinmarketcap/main.py b/airbyte-integrations/connectors/source-coinmarketcap/main.py index e2c19d866514..908e01700613 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/main.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coinmarketcap import SourceCoinmarketcap +from source_coinmarketcap.run import run if __name__ == "__main__": - source = SourceCoinmarketcap() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml index a73b7184a363..d12360ad7599 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml +++ b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml @@ -8,6 +8,10 @@ data: icon: coinmarketcap.svg license: MIT name: CoinMarketCap + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coinmarketcap registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-coinmarketcap/setup.py b/airbyte-integrations/connectors/source-coinmarketcap/setup.py index 64fd1b41273b..608c7f92baef 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/setup.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coinmarketcap=source_coinmarketcap.run:run", + ], + }, name="source_coinmarketcap", description="Source implementation for Coinmarketcap.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py b/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py new file mode 100644 index 000000000000..523d670c3aa9 --- /dev/null +++ b/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coinmarketcap import SourceCoinmarketcap + + +def run(): + source = SourceCoinmarketcap() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-commcare/main.py b/airbyte-integrations/connectors/source-commcare/main.py index 362386c57b33..edd438bde5be 100644 --- a/airbyte-integrations/connectors/source-commcare/main.py +++ b/airbyte-integrations/connectors/source-commcare/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_commcare import SourceCommcare +from source_commcare.run import run if __name__ == "__main__": - source = SourceCommcare() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-commcare/metadata.yaml b/airbyte-integrations/connectors/source-commcare/metadata.yaml index e1bc67ca7ec8..3e78f837b0e1 100644 --- a/airbyte-integrations/connectors/source-commcare/metadata.yaml +++ b/airbyte-integrations/connectors/source-commcare/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-commcare license: MIT name: Commcare + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commcare registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-commcare/setup.py b/airbyte-integrations/connectors/source-commcare/setup.py index fd011eec2b1f..44ba9e72640a 100644 --- a/airbyte-integrations/connectors/source-commcare/setup.py +++ b/airbyte-integrations/connectors/source-commcare/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-commcare=source_commcare.run:run", + ], + }, name="source_commcare", description="Source implementation for Commcare.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/run.py b/airbyte-integrations/connectors/source-commcare/source_commcare/run.py new file mode 100644 index 000000000000..d42251049d08 --- /dev/null +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_commcare import SourceCommcare + + +def run(): + source = SourceCommcare() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-commercetools/main.py b/airbyte-integrations/connectors/source-commercetools/main.py index 32aac6ed9f30..44dd2fb8f952 100644 --- a/airbyte-integrations/connectors/source-commercetools/main.py +++ b/airbyte-integrations/connectors/source-commercetools/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_commercetools import SourceCommercetools +from source_commercetools.run import run if __name__ == "__main__": - source = SourceCommercetools() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index dcbb88d3fafe..f28b88a916c9 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - auth.${region}.${host}.commercetools.com - api.${region}.${host}.commercetools.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commercetools registries: oss: enabled: true @@ -20,7 +24,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/commercetools tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-commercetools/setup.py b/airbyte-integrations/connectors/source-commercetools/setup.py index 9622d7ad2c24..386e9738b744 100644 --- a/airbyte-integrations/connectors/source-commercetools/setup.py +++ b/airbyte-integrations/connectors/source-commercetools/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-commercetools=source_commercetools.run:run", + ], + }, name="source_commercetools", description="Source implementation for Commercetools.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py new file mode 100644 index 000000000000..0d264787f978 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_commercetools import SourceCommercetools + + +def run(): + source = SourceCommercetools() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-configcat/main.py b/airbyte-integrations/connectors/source-configcat/main.py index b398a6dc8c92..9b554d1d713f 100644 --- a/airbyte-integrations/connectors/source-configcat/main.py +++ b/airbyte-integrations/connectors/source-configcat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_configcat import SourceConfigcat +from source_configcat.run import run if __name__ == "__main__": - source = SourceConfigcat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-configcat/metadata.yaml b/airbyte-integrations/connectors/source-configcat/metadata.yaml index b4725144c2f1..3c1f707095be 100644 --- a/airbyte-integrations/connectors/source-configcat/metadata.yaml +++ b/airbyte-integrations/connectors/source-configcat/metadata.yaml @@ -8,6 +8,10 @@ data: icon: configcat.svg license: MIT name: ConfigCat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-configcat registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-configcat/setup.py b/airbyte-integrations/connectors/source-configcat/setup.py index 0bfbad2e7170..0f65dc21183f 100644 --- a/airbyte-integrations/connectors/source-configcat/setup.py +++ b/airbyte-integrations/connectors/source-configcat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-configcat=source_configcat.run:run", + ], + }, name="source_configcat", description="Source implementation for Configcat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-configcat/source_configcat/run.py b/airbyte-integrations/connectors/source-configcat/source_configcat/run.py new file mode 100644 index 000000000000..9167f73fa44c --- /dev/null +++ b/airbyte-integrations/connectors/source-configcat/source_configcat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_configcat import SourceConfigcat + + +def run(): + source = SourceConfigcat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-confluence/main.py b/airbyte-integrations/connectors/source-confluence/main.py index 5cf3cd879713..eb38dbbac0f0 100644 --- a/airbyte-integrations/connectors/source-confluence/main.py +++ b/airbyte-integrations/connectors/source-confluence/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_confluence import SourceConfluence +from source_confluence.run import run if __name__ == "__main__": - source = SourceConfluence() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index 87d324b5ebec..be0b48fb334a 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -5,6 +5,10 @@ data: allowedHosts: hosts: - ${subdomain}.atlassian.net + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-confluence registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-confluence/setup.py b/airbyte-integrations/connectors/source-confluence/setup.py index aaacdff5bbe3..993c131b35f9 100644 --- a/airbyte-integrations/connectors/source-confluence/setup.py +++ b/airbyte-integrations/connectors/source-confluence/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-confluence=source_confluence.run:run", + ], + }, name="source_confluence", description="Source implementation for Confluence.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/run.py b/airbyte-integrations/connectors/source-confluence/source_confluence/run.py new file mode 100644 index 000000000000..b52381028de6 --- /dev/null +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_confluence import SourceConfluence + + +def run(): + source = SourceConfluence() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convertkit/main.py b/airbyte-integrations/connectors/source-convertkit/main.py index 95b40e4a8c42..0338150054eb 100644 --- a/airbyte-integrations/connectors/source-convertkit/main.py +++ b/airbyte-integrations/connectors/source-convertkit/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_convertkit import SourceConvertkit +from source_convertkit.run import run if __name__ == "__main__": - source = SourceConvertkit() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-convertkit/metadata.yaml b/airbyte-integrations/connectors/source-convertkit/metadata.yaml index 66d558a615f6..8a0327f0580c 100644 --- a/airbyte-integrations/connectors/source-convertkit/metadata.yaml +++ b/airbyte-integrations/connectors/source-convertkit/metadata.yaml @@ -8,6 +8,10 @@ data: icon: convertkit.svg license: MIT name: ConvertKit + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convertkit registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-convertkit/setup.py b/airbyte-integrations/connectors/source-convertkit/setup.py index a4a3f97a2b7d..1f8512be419f 100644 --- a/airbyte-integrations/connectors/source-convertkit/setup.py +++ b/airbyte-integrations/connectors/source-convertkit/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-convertkit=source_convertkit.run:run", + ], + }, name="source_convertkit", description="Source implementation for Convertkit.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py b/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py new file mode 100644 index 000000000000..0db38a005bc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_convertkit import SourceConvertkit + + +def run(): + source = SourceConvertkit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convex/Dockerfile b/airbyte-integrations/connectors/source-convex/Dockerfile index 6108e5911ada..3b46ff759568 100644 --- a/airbyte-integrations/connectors/source-convex/Dockerfile +++ b/airbyte-integrations/connectors/source-convex/Dockerfile @@ -34,5 +34,5 @@ COPY source_convex ./source_convex ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-convex diff --git a/airbyte-integrations/connectors/source-convex/main.py b/airbyte-integrations/connectors/source-convex/main.py index 90037cfed046..751ae667fae2 100644 --- a/airbyte-integrations/connectors/source-convex/main.py +++ b/airbyte-integrations/connectors/source-convex/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_convex import SourceConvex +from source_convex.run import run if __name__ == "__main__": - source = SourceConvex() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index 9e454dd6e454..14d34525a641 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: c332628c-f55c-4017-8222-378cfafda9b2 - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-convex githubIssueLabel: source-convex icon: convex.svg license: MIT name: Convex + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convex registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-convex/setup.py b/airbyte-integrations/connectors/source-convex/setup.py index 22937fa853f1..82f937d289d9 100644 --- a/airbyte-integrations/connectors/source-convex/setup.py +++ b/airbyte-integrations/connectors/source-convex/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-convex=source_convex.run:run", + ], + }, name="source_convex", description="Source implementation for Convex.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-convex/source_convex/run.py b/airbyte-integrations/connectors/source-convex/source_convex/run.py new file mode 100644 index 000000000000..94ff036e088c --- /dev/null +++ b/airbyte-integrations/connectors/source-convex/source_convex/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_convex import SourceConvex + + +def run(): + source = SourceConvex() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convex/source_convex/source.py b/airbyte-integrations/connectors/source-convex/source_convex/source.py index 94f08c3fe77b..664f5bf3ca16 100644 --- a/airbyte-integrations/connectors/source-convex/source_convex/source.py +++ b/airbyte-integrations/connectors/source-convex/source_convex/source.py @@ -31,7 +31,7 @@ }, ) -CONVEX_CLIENT_VERSION = "0.3.0" +CONVEX_CLIENT_VERSION = "0.4.0" # Source @@ -153,7 +153,8 @@ def next_page_token(self, response: requests.Response) -> Optional[ConvexState]: else: self._delta_cursor_value = resp_json["cursor"] self._delta_has_more = resp_json["hasMore"] - return cast(ConvexState, self.state) if self._delta_has_more else None + has_more = self._snapshot_has_more or self._delta_has_more + return cast(ConvexState, self.state) if has_more else None def path( self, diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index 89f2c9a66e56..17512d01cf07 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -6,13 +6,15 @@ from unittest.mock import MagicMock import pytest +import requests +import responses +from airbyte_cdk.models import SyncMode from source_convex.source import ConvexStream @pytest.fixture def patch_base_class(mocker): # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(ConvexStream, "path", "v0/example_endpoint") mocker.patch.object(ConvexStream, "primary_key", "test_primary_key") mocker.patch.object(ConvexStream, "__abstractmethods__", set()) @@ -62,6 +64,66 @@ def test_next_page_token(patch_base_class): assert stream.state == {"snapshot_cursor": 1235, "snapshot_has_more": False, "delta_cursor": 7000} +@responses.activate +def test_read_records_full_refresh(patch_base_class): + stream = ConvexStream("http://mocked_base_url:8080", "accesskey", "json", "messages", None) + snapshot0_resp = {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json", + json=snapshot0_resp, + ) + snapshot1_resp = {"values": [{"_id": "an_id", "field": "b", "_ts": 100}], "cursor": 2345, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=1234&snapshot=5000", + json=snapshot1_resp, + ) + snapshot2_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 300}], "cursor": 3456, "snapshot": 5000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=2345&snapshot=5000", + json=snapshot2_resp, + ) + records = list(stream.read_records(SyncMode.full_refresh)) + assert len(records) == 3 + assert [record["field"] for record in records] == ["f", "b", "x"] + assert stream.state == {"delta_cursor": 5000, "snapshot_cursor": 3456, "snapshot_has_more": False} + + +@responses.activate +def test_read_records_incremental(patch_base_class): + stream = ConvexStream("http://mocked_base_url:8080", "accesskey", "json", "messages", None) + snapshot0_resp = {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json", + json=snapshot0_resp, + ) + snapshot1_resp = {"values": [{"_id": "an_id", "field": "b", "_ts": 100}], "cursor": 2345, "snapshot": 5000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=1234&snapshot=5000", + json=snapshot1_resp, + ) + delta0_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 300}], "cursor": 6000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/document_deltas?tableName=messages&format=json&cursor=5000", + json=delta0_resp, + ) + delta1_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 400}], "cursor": 7000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/document_deltas?tableName=messages&format=json&cursor=6000", + json=delta1_resp, + ) + records = list(stream.read_records(SyncMode.incremental)) + assert len(records) == 4 + assert [record["field"] for record in records] == ["f", "b", "x", "x"] + assert stream.state == {"delta_cursor": 7000, "snapshot_cursor": 2345, "snapshot_has_more": False} + + def test_parse_response(patch_base_class): stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) resp = MagicMock() @@ -75,7 +137,7 @@ def test_parse_response(patch_base_class): def test_request_headers(patch_base_class): stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - assert stream.request_headers(**inputs) == {"Convex-Client": "airbyte-export-0.3.0"} + assert stream.request_headers(**inputs) == {"Convex-Client": "airbyte-export-0.4.0"} def test_http_method(patch_base_class): diff --git a/airbyte-integrations/connectors/source-copper/main.py b/airbyte-integrations/connectors/source-copper/main.py index 5c45e3420a77..090988115e85 100644 --- a/airbyte-integrations/connectors/source-copper/main.py +++ b/airbyte-integrations/connectors/source-copper/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_copper import SourceCopper +from source_copper.run import run if __name__ == "__main__": - source = SourceCopper() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index cfa55715d0c4..6298afd8c8b5 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.copper.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-copper registries: oss: enabled: true @@ -20,5 +24,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/copper tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-copper/setup.py b/airbyte-integrations/connectors/source-copper/setup.py index 15fe92381858..f4a0f506acbb 100644 --- a/airbyte-integrations/connectors/source-copper/setup.py +++ b/airbyte-integrations/connectors/source-copper/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-copper=source_copper.run:run", + ], + }, name="source_copper", description="Source implementation for Copper.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-copper/source_copper/run.py b/airbyte-integrations/connectors/source-copper/source_copper/run.py new file mode 100644 index 000000000000..1267cf2c4722 --- /dev/null +++ b/airbyte-integrations/connectors/source-copper/source_copper/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_copper import SourceCopper + + +def run(): + source = SourceCopper() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-courier/main.py b/airbyte-integrations/connectors/source-courier/main.py index df87edbfaf1d..d4de8fbdc917 100644 --- a/airbyte-integrations/connectors/source-courier/main.py +++ b/airbyte-integrations/connectors/source-courier/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_courier import SourceCourier +from source_courier.run import run if __name__ == "__main__": - source = SourceCourier() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-courier/metadata.yaml b/airbyte-integrations/connectors/source-courier/metadata.yaml index d2badc1d80db..06a78d2cfc9d 100644 --- a/airbyte-integrations/connectors/source-courier/metadata.yaml +++ b/airbyte-integrations/connectors/source-courier/metadata.yaml @@ -8,11 +8,15 @@ data: icon: courier.svg license: MIT name: Courier - registries: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-courier + registries: # Removed from registries due to LEGACY STATE cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/courier tags: diff --git a/airbyte-integrations/connectors/source-courier/setup.py b/airbyte-integrations/connectors/source-courier/setup.py index 883332ece7a6..9bb4390f1913 100644 --- a/airbyte-integrations/connectors/source-courier/setup.py +++ b/airbyte-integrations/connectors/source-courier/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-courier=source_courier.run:run", + ], + }, name="source_courier", description="Source implementation for Courier.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-courier/source_courier/run.py b/airbyte-integrations/connectors/source-courier/source_courier/run.py new file mode 100644 index 000000000000..c6e85fbde238 --- /dev/null +++ b/airbyte-integrations/connectors/source-courier/source_courier/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_courier import SourceCourier + + +def run(): + source = SourceCourier() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-customer-io/main.py b/airbyte-integrations/connectors/source-customer-io/main.py index 835bc1df92ba..ce54e7ce6db6 100644 --- a/airbyte-integrations/connectors/source-customer-io/main.py +++ b/airbyte-integrations/connectors/source-customer-io/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_customer_io import SourceCustomerIo +from source_customer_io.run import run if __name__ == "__main__": - source = SourceCustomerIo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-customer-io/metadata.yaml b/airbyte-integrations/connectors/source-customer-io/metadata.yaml index 548f40de65ea..6bb0d1e26176 100644 --- a/airbyte-integrations/connectors/source-customer-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-customer-io/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.customer.io/v1/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-customer-io registries: oss: enabled: false @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/customer-io tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-customer-io/setup.py b/airbyte-integrations/connectors/source-customer-io/setup.py index 04cd8664c8a1..eb13cd73bbea 100644 --- a/airbyte-integrations/connectors/source-customer-io/setup.py +++ b/airbyte-integrations/connectors/source-customer-io/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-customer-io=source_customer_io.run:run", + ], + }, name="source_customer_io", description="Source implementation for Customer Io.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py b/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py new file mode 100644 index 000000000000..b122bbd03497 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_customer_io import SourceCustomerIo + + +def run(): + source = SourceCustomerIo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-datadog/main.py b/airbyte-integrations/connectors/source-datadog/main.py index 768db52beb4f..d8e7d33f093e 100644 --- a/airbyte-integrations/connectors/source-datadog/main.py +++ b/airbyte-integrations/connectors/source-datadog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_datadog import SourceDatadog +from source_datadog.run import run if __name__ == "__main__": - source = SourceDatadog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index 88bef85da8ba..da27ba470328 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -6,6 +6,10 @@ data: - us5.datadoghq.com - datadoghq.eu - ddog-gov.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datadog registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-datadog/setup.py b/airbyte-integrations/connectors/source-datadog/setup.py index 432515ebd8f0..59b397984d78 100644 --- a/airbyte-integrations/connectors/source-datadog/setup.py +++ b/airbyte-integrations/connectors/source-datadog/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-datadog=source_datadog.run:run", + ], + }, name="source_datadog", description="Source implementation for Datadog.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/run.py b/airbyte-integrations/connectors/source-datadog/source_datadog/run.py new file mode 100644 index 000000000000..62adbe197666 --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_datadog import SourceDatadog + + +def run(): + source = SourceDatadog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-datascope/main.py b/airbyte-integrations/connectors/source-datascope/main.py index dbdd6b9cc1a5..59b428ca2396 100644 --- a/airbyte-integrations/connectors/source-datascope/main.py +++ b/airbyte-integrations/connectors/source-datascope/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_datascope import SourceDatascope +from source_datascope.run import run if __name__ == "__main__": - source = SourceDatascope() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-datascope/metadata.yaml b/airbyte-integrations/connectors/source-datascope/metadata.yaml index 2d61ac39b220..2649a980d2dd 100644 --- a/airbyte-integrations/connectors/source-datascope/metadata.yaml +++ b/airbyte-integrations/connectors/source-datascope/metadata.yaml @@ -8,6 +8,10 @@ data: icon: datascope.svg license: MIT name: Datascope + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datascope registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-datascope/setup.py b/airbyte-integrations/connectors/source-datascope/setup.py index 4a3a8864ddf4..d999c1e72290 100644 --- a/airbyte-integrations/connectors/source-datascope/setup.py +++ b/airbyte-integrations/connectors/source-datascope/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-datascope=source_datascope.run:run", + ], + }, name="source_datascope", description="Source implementation for Datascope.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/run.py b/airbyte-integrations/connectors/source-datascope/source_datascope/run.py new file mode 100644 index 000000000000..e1543c70c39c --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_datascope import SourceDatascope + + +def run(): + source = SourceDatascope() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml b/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml deleted file mode 100644 index 37095f7ef91f..000000000000 --- a/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-db2:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 886cecdd301c..3b9d58fbbf7b 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -1,26 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.db2.Db2Source' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -30,7 +17,5 @@ dependencies { implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' - testImplementation libs.testcontainers.db2 - - integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.testcontainers:db2:1.19.4' } diff --git a/airbyte-integrations/connectors/source-db2/metadata.yaml b/airbyte-integrations/connectors/source-db2/metadata.yaml index 5aa18ba1f25a..fee9127e66de 100644 --- a/airbyte-integrations/connectors/source-db2/metadata.yaml +++ b/airbyte-integrations/connectors/source-db2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 - dockerImageTag: 0.1.20 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-db2 githubIssueLabel: source-db2 icon: db2.svg diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index 9437232493e3..a6858e314eee 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -95,6 +95,7 @@ public Set getExcludedInternalNameSpaces() { } @Override + @SuppressWarnings("unchecked") public Set getPrivilegesTableForCurrentUser(final JdbcDatabase database, final String schema) throws SQLException { try (final Stream stream = database.unsafeQuery(getPrivileges(), sourceOperations::rowToJson)) { return stream.map(this::getPrivilegeDto).collect(Collectors.toSet()); @@ -170,7 +171,7 @@ private static void convertAndImportCertificate(final String certificate, final } private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { - final Process pr = run.exec(cmd); + final Process pr = run.exec(cmd.split(" ")); if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java index 60ea2ac4cf4b..48230d65505a 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java @@ -34,9 +34,11 @@ import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java index fc1d876f8d68..011033264271 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java @@ -31,8 +31,10 @@ import java.util.HashMap; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceCertificateAcceptanceTest extends SourceAcceptanceTest { private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 85c0fb6ad529..155ec3e1e909 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -18,8 +18,10 @@ import io.airbyte.protocol.models.JsonSchemaType; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private static final String CREATE_TABLE_SQL = "CREATE TABLE %1$s(%2$s INTEGER NOT NULL PRIMARY KEY, %3$s %4$s)"; @@ -41,7 +43,6 @@ protected JsonNode getConfig() throws Exception { @Override protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); container.close(); } diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java index 680514248099..3d53bc09582e 100644 --- a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java @@ -7,32 +7,25 @@ import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import java.sql.JDBCType; import java.util.Collections; import java.util.Set; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; -class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + private final static Db2Container DB_2_CONTAINER = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); + private static final String QUOTE_STRING = "\""; private static Set TEST_TABLES = Collections.emptySet(); - private static Db2Container db; - private JsonNode config; @BeforeAll static void init() { - db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); - db.start(); - // Db2 transforms names to upper case, so we need to use upper case name to retrieve data later. SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; @@ -64,76 +57,63 @@ static void init() { INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(true)"; } - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put("db", db.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) - .build()); - - super.setup(); + @AfterAll + static void cleanUp() { + DB_2_CONTAINER.close(); } - @AfterEach - public void clean() throws Exception { + static void deleteTablesAndSchema(final Db2TestDatabase testdb) { // In Db2 before dropping a schema, all objects that were in that schema must be dropped or moved to // another schema. for (final String tableName : TEST_TABLES) { final String dropTableQuery = String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, tableName); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + testdb.with(dropTableQuery); } for (int i = 2; i < 10; i++) { final String dropTableQuery = String .format("DROP TABLE IF EXISTS %s.%s%s", SCHEMA_NAME, TABLE_NAME, i); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + testdb.with(dropTableQuery); } - super.database.execute(connection -> connection.createStatement().execute(String + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_SPACES, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITH_SPACES, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_SPACES + 2, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITH_SPACES + 2, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME2, - enquoteIdentifier(TABLE_NAME, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITHOUT_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITHOUT_CURSOR_TYPE, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.tearDown(); - } + enquoteIdentifier(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE, QUOTE_STRING))); + for (final String schemaName : TEST_SCHEMAS) { + testdb.with(DROP_SCHEMA_QUERY, schemaName); + } - @AfterAll - static void cleanUp() { - db.close(); } @Override - public boolean supportsSchemas() { - return true; + protected Db2TestDatabase createTestDatabase() { + DB_2_CONTAINER.start(); + return new Db2TestDatabase(DB_2_CONTAINER).initialized(); } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public boolean supportsSchemas() { + return true; } @Override - public String getDriverClass() { - return Db2Source.DRIVER_CLASS; + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public AbstractJdbcSource getJdbcSource() { + protected Db2Source source() { return new Db2Source(); } diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java index f3a985c4cd01..c11d7736a9ef 100644 --- a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java @@ -20,8 +20,10 @@ import java.nio.file.Path; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class Db2SpecTest { private static JsonNode schema; diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java new file mode 100644 index 000000000000..216ed0ee5319 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2; + +import static io.airbyte.integrations.source.db2.Db2JdbcSourceAcceptanceTest.deleteTablesAndSchema; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.Db2Container; + +public class Db2TestDatabase extends + TestDatabase { + + private final Db2Container container; + + protected Db2TestDatabase(final Db2Container container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.DB2; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + deleteTablesAndSchema(this); + } + + @Override + public Db2DbConfigBuilder configBuilder() { + return new Db2DbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("db", container.getDatabaseName()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())); + } + + static public class Db2DbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected Db2DbConfigBuilder(final Db2TestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-delighted/main.py b/airbyte-integrations/connectors/source-delighted/main.py index 2ba66b4bf80a..f2a80745b8cd 100644 --- a/airbyte-integrations/connectors/source-delighted/main.py +++ b/airbyte-integrations/connectors/source-delighted/main.py @@ -2,34 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_delighted import SourceDelighted +from source_delighted.run import run if __name__ == "__main__": - source = SourceDelighted() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index 2a050c2b4814..c9d006b900ee 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -15,6 +15,10 @@ data: icon: delighted.svg license: MIT name: Delighted + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-delighted registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index ca92c54ec6fe..fdffdc368480 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-delighted=source_delighted.run:run", + ], + }, name="source_delighted", description="Source implementation for Delighted.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/run.py b/airbyte-integrations/connectors/source-delighted/source_delighted/run.py new file mode 100644 index 000000000000..b38535e5962d --- /dev/null +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/run.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_delighted import SourceDelighted + + +def run(): + source = SourceDelighted() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dixa/main.py b/airbyte-integrations/connectors/source-dixa/main.py index 5ec7b113ae37..cf2dc464235a 100644 --- a/airbyte-integrations/connectors/source-dixa/main.py +++ b/airbyte-integrations/connectors/source-dixa/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dixa import SourceDixa +from source_dixa.run import run if __name__ == "__main__": - source = SourceDixa() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index 2bd71f0c367a..bb4821ee8c81 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - exports.dixa.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dixa registries: oss: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/dixa tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-dixa/setup.py b/airbyte-integrations/connectors/source-dixa/setup.py index be6537c6a3cd..5a08f6fc5552 100644 --- a/airbyte-integrations/connectors/source-dixa/setup.py +++ b/airbyte-integrations/connectors/source-dixa/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dixa=source_dixa.run:run", + ], + }, name="source_dixa", description="Source implementation for Dixa.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/run.py b/airbyte-integrations/connectors/source-dixa/source_dixa/run.py new file mode 100644 index 000000000000..f264dd78af8e --- /dev/null +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dixa import SourceDixa + + +def run(): + source = SourceDixa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dockerhub/main.py b/airbyte-integrations/connectors/source-dockerhub/main.py index fffced0a26b7..c7bf0a5fadc9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/main.py +++ b/airbyte-integrations/connectors/source-dockerhub/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dockerhub import SourceDockerhub +from source_dockerhub.run import run if __name__ == "__main__": - source = SourceDockerhub() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index 3afe52544412..7800c7d8f8d9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - hub.docker.com - auth.docker.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dockerhub registries: oss: enabled: true @@ -21,7 +25,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/dockerhub tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-dockerhub/setup.py b/airbyte-integrations/connectors/source-dockerhub/setup.py index ea4b2d5873c8..9d382d831668 100644 --- a/airbyte-integrations/connectors/source-dockerhub/setup.py +++ b/airbyte-integrations/connectors/source-dockerhub/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dockerhub=source_dockerhub.run:run", + ], + }, name="source_dockerhub", description="Source implementation for Dockerhub.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py new file mode 100644 index 000000000000..40df913e37e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dockerhub import SourceDockerhub + + +def run(): + source = SourceDockerhub() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dremio/main.py b/airbyte-integrations/connectors/source-dremio/main.py index 10c2231becd9..2a7f7fbaabd4 100644 --- a/airbyte-integrations/connectors/source-dremio/main.py +++ b/airbyte-integrations/connectors/source-dremio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dremio import SourceDremio +from source_dremio.run import run if __name__ == "__main__": - source = SourceDremio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dremio/metadata.yaml b/airbyte-integrations/connectors/source-dremio/metadata.yaml index 4315e72f6be3..406e3707fdda 100644 --- a/airbyte-integrations/connectors/source-dremio/metadata.yaml +++ b/airbyte-integrations/connectors/source-dremio/metadata.yaml @@ -8,6 +8,10 @@ data: icon: dremio.svg license: MIT name: Dremio + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dremio registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-dremio/setup.py b/airbyte-integrations/connectors/source-dremio/setup.py index 290c8fcf4fbe..0d89e8a42fe6 100644 --- a/airbyte-integrations/connectors/source-dremio/setup.py +++ b/airbyte-integrations/connectors/source-dremio/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dremio=source_dremio.run:run", + ], + }, name="source_dremio", description="Source implementation for Dremio.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dremio/source_dremio/run.py b/airbyte-integrations/connectors/source-dremio/source_dremio/run.py new file mode 100644 index 000000000000..8df023f7eff9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dremio/source_dremio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dremio import SourceDremio + + +def run(): + source = SourceDremio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-drift/main.py b/airbyte-integrations/connectors/source-drift/main.py index 0e0072ac04c3..7f9fdd2ad3df 100644 --- a/airbyte-integrations/connectors/source-drift/main.py +++ b/airbyte-integrations/connectors/source-drift/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_drift import SourceDrift +from source_drift.run import run if __name__ == "__main__": - source = SourceDrift() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index f17ce487e236..e22ab893eae5 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://driftapi.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-drift registries: oss: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/drift tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index 5407ab8ae1e5..7e6632fa4725 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-drift=source_drift.run:run", + ], + }, name="source_drift", description="Source implementation for Drift.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-drift/source_drift/run.py b/airbyte-integrations/connectors/source-drift/source_drift/run.py new file mode 100644 index 000000000000..b9b04289c276 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/source_drift/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_drift import SourceDrift + + +def run(): + source = SourceDrift() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dv-360/main.py b/airbyte-integrations/connectors/source-dv-360/main.py index 483df6f4de96..4d7158d2a11d 100644 --- a/airbyte-integrations/connectors/source-dv-360/main.py +++ b/airbyte-integrations/connectors/source-dv-360/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dv_360 import SourceDV360 +from source_dv_360.run import run if __name__ == "__main__": - source = SourceDV360() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dv-360/metadata.yaml b/airbyte-integrations/connectors/source-dv-360/metadata.yaml index acaa90cec05d..21de388e91c2 100644 --- a/airbyte-integrations/connectors/source-dv-360/metadata.yaml +++ b/airbyte-integrations/connectors/source-dv-360/metadata.yaml @@ -8,11 +8,15 @@ data: icon: dv360.svg license: MIT name: DV 360 - registries: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dv-360 + registries: # Removed from registries due to LEGACY STATE cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/dv-360 tags: diff --git a/airbyte-integrations/connectors/source-dv-360/setup.py b/airbyte-integrations/connectors/source-dv-360/setup.py index a8198206559e..850d55c1e665 100644 --- a/airbyte-integrations/connectors/source-dv-360/setup.py +++ b/airbyte-integrations/connectors/source-dv-360/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock"] setup( + entry_points={ + "console_scripts": [ + "source-dv-360=source_dv_360.run:run", + ], + }, name="source_dv_360", description="Source implementation for Display & Video 360.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py new file mode 100644 index 000000000000..a869331c4dbf --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dv_360 import SourceDV360 + + +def run(): + source = SourceDV360() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml deleted file mode 100644 index 5ec5326f613a..000000000000 --- a/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-dynamodb:dev -acceptance-tests: - spec: - tests: - - spec_path: "main/resources/spec.json" diff --git a/airbyte-integrations/connectors/source-dynamodb/build.gradle b/airbyte-integrations/connectors/source-dynamodb/build.gradle index f555ca183d4d..9ab96a5b6693 100644 --- a/airbyte-integrations/connectors/source-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/source-dynamodb/build.gradle @@ -1,30 +1,17 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.dynamodb.DynamodbSource' } -def testContainersVersion = '1.17.5' -def assertVersion = '3.23.1' - dependencies { implementation platform('software.amazon.awssdk:bom:2.18.1') @@ -32,14 +19,7 @@ dependencies { implementation 'software.amazon.awssdk:dynamodb' testImplementation 'org.skyscreamer:jsonassert:1.5.1' + testImplementation "org.assertj:assertj-core:3.23.1" + testImplementation "org.testcontainers:localstack:1.19.4" - - // https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind - implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2' - // https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core - implementation 'com.fasterxml.jackson.core:jackson-core:2.13.4' - - - testImplementation "org.assertj:assertj-core:${assertVersion}" - testImplementation "org.testcontainers:localstack:${testContainersVersion}" } diff --git a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml index e71ffe623c56..82d7bcf42909 100644 --- a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 50401137-8871-4c5a-abb7-1f5fda35545a - dockerImageTag: 0.1.2 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-dynamodb documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb githubIssueLabel: source-dynamodb diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java index 719b71a1deba..f3ffab950c66 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java @@ -16,8 +16,6 @@ import io.airbyte.cdk.integrations.source.relationaldb.StateDecoratingIterator; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; @@ -42,8 +40,6 @@ public class DynamodbSource extends BaseConnector implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(DynamodbSource.class); - private final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); - private final ObjectMapper objectMapper = new ObjectMapper(); public static void main(final String[] args) throws Exception { @@ -98,7 +94,7 @@ public AutoCloseableIterator read(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JsonNode state) { - final var streamState = DynamodbUtils.deserializeStreamState(state, featureFlags.useStreamCapableState()); + final var streamState = DynamodbUtils.deserializeStreamState(state); final StateManager stateManager = StateManagerFactory .createStateManager(streamState.airbyteStateType(), streamState.airbyteStateMessages(), catalog); diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java index c664682deab7..e79dc1833af0 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.source.dynamodb; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.StateWrapper; import io.airbyte.configoss.helpers.StateMessageHelper; @@ -53,9 +52,9 @@ public static AirbyteMessage mapAirbyteMessage(final String stream, final JsonNo .withData(data)); } - public static StreamState deserializeStreamState(final JsonNode state, final boolean useStreamCapableState) { + public static StreamState deserializeStreamState(final JsonNode state) { final Optional typedState = - StateMessageHelper.getTypedState(state, useStreamCapableState); + StateMessageHelper.getTypedState(state); return typedState.map(stateWrapper -> switch (stateWrapper.getStateType()) { case STREAM: yield new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, @@ -68,15 +67,10 @@ yield new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( throw new UnsupportedOperationException("Unsupported stream state"); }).orElseGet(() -> { // create empty initial state - if (useStreamCapableState) { - return new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState()))); - } else { - return new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState())))); - } + return new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, List.of( + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState()))); + }); } diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json b/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json index 6b6a09f075fb..25745616e501 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json @@ -21,31 +21,39 @@ "description": "The region of the Dynamodb database", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ] }, "access_key_id": { diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java index d2784fbeddce..814dbcff8458 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java @@ -15,12 +15,14 @@ import org.json.JSONException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbOperationsTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java index 5036d609dc87..86c86fae53be 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java @@ -13,10 +13,12 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.HashMap; import java.util.Map; +import org.junit.jupiter.api.Disabled; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbSourceAcceptanceTest extends SourceAcceptanceTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java index 7e37aa47e338..713e84ed53b6 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java @@ -17,11 +17,13 @@ import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbSourceTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle index 1a3b3f3109eb..fc2ce55fc81d 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle @@ -1,29 +1,17 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.e2e_test.CloudTestingSources' } dependencies { implementation project(':airbyte-integrations:connectors:source-e2e-test') - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-e2e-test-cloud') } diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml index a988b663087e..fe08670b05a0 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 50bd8338-7c4e-46f1-8c7f-3ef95de19fdd - dockerImageTag: 2.1.5 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-e2e-test-cloud githubIssueLabel: source-e2e-test-cloud icon: airbyte.svg diff --git a/airbyte-integrations/connectors/source-e2e-test/build.gradle b/airbyte-integrations/connectors/source-e2e-test/build.gradle index 148e0cf1922b..bf277e0cb036 100644 --- a/airbyte-integrations/connectors/source-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test/build.gradle @@ -1,39 +1,20 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.e2e_test.TestingSources' } dependencies { - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'com.networknt:json-schema-validator:1.0.72' - // random Json object generation from Json schema // https://github.com/airbytehq/jsongenerator implementation 'net.jimblackler.jsonschemafriend:core:0.12.1' - implementation 'org.mozilla:rhino-engine:1.7.14' implementation group: 'com.github.airbytehq', name: 'jsongenerator', version: '1.0.2' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-e2e-test') } diff --git a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml index cde52423510d..83d67fc62fc1 100644 --- a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a - dockerImageTag: 2.1.5 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-e2e-test githubIssueLabel: source-e2e-test icon: airbyte.svg diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java index 2119222ae514..a2775f18a121 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java @@ -12,15 +12,9 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.SyncMode; import java.time.Instant; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -78,7 +72,12 @@ protected AirbyteMessage computeNext() { hasEmittedStateAtCount.set(true); return new AirbyteMessage() .withType(Type.STATE) - .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))); + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(LegacyConstants.DEFAULT_STREAM)) + .withStreamState(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))) + .withData(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))); } else if (throwAfterNRecords > recordsEmitted.get()) { recordsEmitted.incrementAndGet(); recordValue.incrementAndGet(); diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSourceTest.java b/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSourceTest.java index 09ae68fc0343..036a784af072 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSourceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/test/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSourceTest.java @@ -98,6 +98,7 @@ void testDiscover() throws Exception { } @Test + @SuppressWarnings("try") void testSource() throws Exception { final SpeedBenchmarkSource speedBenchmarkSource = new SpeedBenchmarkSource(); diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle index 31f1cfca5fcc..be2a7153ed4a 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/source-elasticsearch/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.elasticsearch.ElasticsearchSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -26,6 +16,7 @@ application { dependencies { implementation 'co.elastic.clients:elasticsearch-java:7.15.0' + implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.15.2' implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3' // EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 @@ -39,6 +30,5 @@ dependencies { // MIT // https://www.testcontainers.org/ - testImplementation libs.testcontainers.elasticsearch - integrationTestJavaImplementation libs.testcontainers.elasticsearch + testImplementation 'org.testcontainers:elasticsearch:1.19.4' } diff --git a/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml b/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml index 1c7dad3da8d3..887dc9c9fa35 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml +++ b/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 7cf88806-25f5-4e1a-b422-b2fa9e1b0090 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-elasticsearch githubIssueLabel: source-elasticsearch icon: elasticsearch.svg diff --git a/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchConnection.java b/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchConnection.java index 15ea98bc4d74..a745fda1db93 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchConnection.java +++ b/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchConnection.java @@ -21,7 +21,7 @@ import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; @@ -44,6 +44,7 @@ public class ElasticsearchConnection { * * @param config Configuration parameters for connecting to the Elasticsearch host */ + @SuppressWarnings("this-escape") public ElasticsearchConnection(ConnectorConfiguration config) { log.info(String.format( "creating ElasticsearchConnection: %s", config.getEndpoint())); diff --git a/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java index 980a71a9c016..83b7db8cc7a0 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -47,7 +48,7 @@ protected String getImageName() { @Override protected JsonNode getConfig() { var configJson = mapper.createObjectNode(); - configJson.put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))); + configJson.put("endpoint", String.format("http://%s:%s", HostPortResolver.resolveHost(container), HostPortResolver.resolvePort(container))); return configJson; } diff --git a/airbyte-integrations/connectors/source-emailoctopus/main.py b/airbyte-integrations/connectors/source-emailoctopus/main.py index 18d45ef8ae6f..c0c08dfdd774 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/main.py +++ b/airbyte-integrations/connectors/source-emailoctopus/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_emailoctopus import SourceEmailoctopus +from source_emailoctopus.run import run if __name__ == "__main__": - source = SourceEmailoctopus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml index 6b3d33420107..55a5037b7c05 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml +++ b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml @@ -8,6 +8,10 @@ data: icon: emailoctopus.svg license: MIT name: EmailOctopus + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-emailoctopus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-emailoctopus/setup.py b/airbyte-integrations/connectors/source-emailoctopus/setup.py index 9cd42d8b7050..4c4d0b6e863c 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/setup.py +++ b/airbyte-integrations/connectors/source-emailoctopus/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-emailoctopus=source_emailoctopus.run:run", + ], + }, name="source_emailoctopus", description="Source implementation for Emailoctopus.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py new file mode 100644 index 000000000000..407e0e086c97 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_emailoctopus import SourceEmailoctopus + + +def run(): + source = SourceEmailoctopus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-everhour/main.py b/airbyte-integrations/connectors/source-everhour/main.py index 8c44913d9ddf..d69ca6247ce6 100644 --- a/airbyte-integrations/connectors/source-everhour/main.py +++ b/airbyte-integrations/connectors/source-everhour/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_everhour import SourceEverhour +from source_everhour.run import run if __name__ == "__main__": - source = SourceEverhour() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-everhour/metadata.yaml b/airbyte-integrations/connectors/source-everhour/metadata.yaml index a41a9283a3d1..edc9128f054f 100644 --- a/airbyte-integrations/connectors/source-everhour/metadata.yaml +++ b/airbyte-integrations/connectors/source-everhour/metadata.yaml @@ -11,6 +11,10 @@ data: icon: everhour.svg license: MIT name: Everhour + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-everhour registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-everhour/setup.py b/airbyte-integrations/connectors/source-everhour/setup.py index e1838cba9eda..40c9560ff254 100644 --- a/airbyte-integrations/connectors/source-everhour/setup.py +++ b/airbyte-integrations/connectors/source-everhour/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-everhour=source_everhour.run:run", + ], + }, name="source_everhour", description="Source implementation for Everhour.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-everhour/source_everhour/run.py b/airbyte-integrations/connectors/source-everhour/source_everhour/run.py new file mode 100644 index 000000000000..4e36ccaaf781 --- /dev/null +++ b/airbyte-integrations/connectors/source-everhour/source_everhour/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_everhour import SourceEverhour + + +def run(): + source = SourceEverhour() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-exchange-rates/main.py b/airbyte-integrations/connectors/source-exchange-rates/main.py index ffdcfd706cff..ce703c9da571 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/main.py +++ b/airbyte-integrations/connectors/source-exchange-rates/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_exchange_rates import SourceExchangeRates +from source_exchange_rates.run import run if __name__ == "__main__": - source = SourceExchangeRates() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index dc7062891b8f..32efbf5e53ff 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - ${subdomain}.apilayer.com - apilayer.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-exchange-rates registries: oss: enabled: true @@ -22,5 +26,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/exchange-rates tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py index 120437119f72..c7832abdf32d 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-exchange-rates/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-exchange-rates=source_exchange_rates.run:run", + ], + }, name="source_exchange_rates", description="Source implementation for Exchange Rates.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py new file mode 100644 index 000000000000..c710a7e23d0d --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_exchange_rates import SourceExchangeRates + + +def run(): + source = SourceExchangeRates() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/README.md b/airbyte-integrations/connectors/source-facebook-marketing/README.md index 1e31681a3fb2..be9a7cc5e992 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/README.md +++ b/airbyte-integrations/connectors/source-facebook-marketing/README.md @@ -1,118 +1,55 @@ -# Facebook Marketing Source +# Facebook-Marketing source connector -This is the repository for the Facebook Marketing source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing). + +This is the repository for the Facebook-Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/facebook-marketing). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/facebook-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-marketing test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-facebook-marketing spec +poetry run source-facebook-marketing check --config secrets/config.json +poetry run source-facebook-marketing discover --config secrets/config.json +poetry run source-facebook-marketing read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-facebook-marketing build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-facebook-marketing:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-facebook-marketing:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-facebook-marketing:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-facebook-marketing:dev . -# Running the spec command against your patched connector -docker run airbyte/source-facebook-marketing:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-facebook-marketing:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-facebook-marketing test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-marketing test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-marketing.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-marketing.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index e5b56c73cece..b4ec84009b78 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -7,8 +7,8 @@ acceptance_tests: tests: - spec_path: "integration_tests/spec.json" backward_compatibility_tests_config: - disable_for_version: "1.1.12" - previous_connector_version: "1.1.11" + disable_for_version: "1.2.2" + previous_connector_version: "1.2.1" connection: tests: - config_path: "secrets/config.json" @@ -21,8 +21,8 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: "1.1.12" - previous_connector_version: "1.1.11" + disable_for_version: "1.2.2" + previous_connector_version: "1.2.1" basic_read: tests: - config_path: "secrets/config.json" @@ -37,6 +37,8 @@ acceptance_tests: ad_creatives: - name: thumbnail_url bypass_reason: is changeable + - name: image_url + bypass_reason: is changeable images: - name: permalink_url bypass_reason: is changeable @@ -81,9 +83,9 @@ acceptance_tests: bypass_reason: is changeable empty_streams: - name: "ads_insights_action_product_id" - bypass_reason: "Data not permanent" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" - name: "videos" - bypass_reason: "Cannot populate" + bypass_reason: "Data cannot be seeded in the test account, integration tests added for the stream instead" timeout_seconds: 4800 expect_records: path: "integration_tests/expected_records.jsonl" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py index f58ee7224089..e7d6bc323d60 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py @@ -6,12 +6,15 @@ import json import pytest +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray @pytest.fixture(scope="session", name="config") def config_fixture(): with open("secrets/config.json", "r") as config_file: - return json.load(config_file) + config = json.load(config_file) + migrated_config = MigrateAccountIdToArray.transform(config) + return migrated_config @pytest.fixture(scope="session", name="config_with_wrong_token") @@ -21,12 +24,45 @@ def config_with_wrong_token_fixture(config): @pytest.fixture(scope="session", name="config_with_wrong_account") def config_with_wrong_account_fixture(config): - return {**config, "account_id": "WRONG_ACCOUNT"} + return {**config, "account_ids": ["WRONG_ACCOUNT"]} @pytest.fixture(scope="session", name="config_with_include_deleted") -def config_with_include_deleted_fixture(config): - new_config = {**config, "include_deleted": True} +def config_with_include_deleted(config): + new_config = { + **config, + "campaign_statuses": [ + "ACTIVE", + "ARCHIVED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", + ], + "adset_statuses": [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", + ], + "ad_statuses": [ + "ACTIVE", + "ADSET_PAUSED", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "DISAPPROVED", + "IN_PROCESS", + "PAUSED", + "PENDING_BILLING_INFO", + "PENDING_REVIEW", + "PREAPPROVED", + "WITH_ISSUES", + ], + } new_config.pop("_limit", None) new_config.pop("end_date", None) return new_config diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl index 6bed2dde636d..4859b8984362 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl @@ -1,10 +1,10 @@ -{"stream": "ad_account", "data": {"id": "act_212551616838260", "account_id": "212551616838260", "account_status": 1, "age": 1305.7507638889, "amount_spent": "39125", "balance": "0", "business": {"id": "1506473679510495", "name": "Airbyte"}, "business_city": "", "business_country_code": "US", "business_name": "", "business_street": "", "business_street2": "", "can_create_brand_lift_study": false, "capabilities": ["CAN_CREATE_CALL_ADS", "CAN_SEE_GROWTH_OPPORTUNITY_DATA", "ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT", "CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE", "CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S", "ENABLE_BIZ_DISCO_ADS", "ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS", "ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS", "ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS", "ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE", "ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION", "ENABLE_PAC_FOR_BIZ_DISCO_ADS", "CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_REACH_AND_FREQUENCY", "CAN_USE_RECURRING_BUDGET", "HAS_VALID_PAYMENT_METHODS", "CAN_USE_LINK_CLICK_BILLING_EVENT", "CAN_USE_CPA_BILLING_EVENT", "CAN_SEE_NEW_CONVERSION_WINDOW_NUX", "ADS_INSTREAM_INTERFACE_INTEGRITY", "ADS_INSTREAM_LINK_CLICK", "ADS_INSTREAM_LINK_CLICK_IMAGE", "ADS_IN_OBJECTIVES_DEPRECATION", "MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES", "CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL", "ALLOW_INSTREAM_ONLY_FOR_REACH", "ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS", "CAN_CREATE_INSTAGRAM_EXPLORE_ADS", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY", "ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN", "INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW", "ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION", "ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY", "WHATSAPP_DESTINATION_ADS", "CTM_ADS_CREATION_CLICK_TO_DIRECT", "CTW_ADS_ENABLE_IG_FEED_PLACEMENT", "CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE", "CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER", "CTW_ADS_TRUSTED_TIER_ADVERTISER", "ADS_PLACEMENT_MARKETPLACE", "ADNW_DISABLE_INSTREAM_AND_WEB_PLACEMENT", "CAN_CHANGE_BILLING_THRESHOLD", "CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING", "CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING", "ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D", "ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT", "ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET", "CAN_SEE_APP_AD_EVENTS", "CAN_SEE_NEW_STANDARD_EVENTS_BETA", "CAN_SEE_VCK_HOLIDAY_TEMPLATES", "ENABLE_DCO_FOR_FB_STORY_ADS", "CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT", "CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY", "CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES", "CAN_USE_IG_SEARCH_PLACEMENT", "CAN_USE_IG_SEARCH_GRID_ADS", "CAN_USE_IG_SEARCH_RESULTS_AUTO_PLACEMENT", "CAN_USE_IG_REELS_PAC_CAROUSEL", "CAN_USE_IG_REELS_POSITION", "CAN_SEE_CONVERSION_LIFT_SUMMARY", "CAN_USE_IG_PROFILE_FEED_POSITION", "CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT", "CAN_USE_IG_PROFILE_FEED_ADDITIONAL_OBJECTIVES", "CAN_USE_IG_REELS_REACH_AND_FREQUENCY", "CAN_USE_IG_REELS_OVERLAY_POSITION", "CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT", "CAN_USE_IG_REELS_OVERLAY_PAC", "CAN_USE_IG_SHOP_TAB_PAC", "CAN_SEE_LEARNING_STAGE", "ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS", "ENABLE_MESSENGER_INBOX_VIDEO_ADS", "ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS", "ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS", "ENABLE_REACH_FOR_FB_STORY_ADS", "CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT", "ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY", "ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS", "ENABLE_LEAD_GEN_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW", "CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT", "ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION", "CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT", "RF_ALLOW_MARKETPLACE_ACCOUNT", "RF_ALLOW_SEARCH_ACCOUNT", "VERTICAL_VIDEO_PAC_INSTREAM_UPSELL", "IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH", "ADS_BM_REQUIREMENTS_OCT_15_RELEASE", "ENABLE_POST_ENGAGEMENT_FOR_FB_STORY", "ENBABLE_CATALOG_SALES_FOR_FB_STORY", "CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS", "CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS", "IS_NON_TAIL_AD_ACCOUNT", "IS_IN_DSA_GK", "IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT", "IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT", "IS_IN_ODAX_EXPERIENCE", "IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT", "CAN_USE_MARKETPLACE_DESKTOP", "ADS_MERCHANT_OVERLAYS_DEPRECATION", "CONNECTIONS_DEPRECATION_V2", "CAN_USE_LIVE_VIDEO_FOR_THRUPLAY", "CAN_SEE_HEC_AM_FLOW", "CAN_SEE_POLITICAL_FLOW", "ADS_INSTREAM_PLACEMENT_CATALOG_SALES", "ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS", "ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS", "ENABLE_REACH_FOR_FB_GROUP_TAB_ADS", "CAN_USE_CONVERSATIONS_OPTIMIZATION", "ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS", "CAN_USE_IG_STORY_POLLS_PAC_CREATION", "IOS14_CEO_CAMPAIGN_CREATION", "ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS", "DIGITAL_CIRCULAR_ADS", "CAN_SEE_SAFR_V3_FLOW", "CAN_USE_FB_REELS_POSITION", "CAN_USE_ADS_ON_FB_REELS_POSITION", "CAN_USE_FB_REELS_AUTO_PLACEMENT", "ENABLE_FB_REELS_CREATION_PAC_ADS", "ENABLE_FB_REELS_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS", "RF_CPA_BILLING_DEPRECATION_PHASE_2", "ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION", "ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX", "ADS_RF_FB_REELS_PLACEMENT", "ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX", "REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY", "ADS_AEMV2_HAS_LAUNCHED", "ELIGIBLE_FOR_TEXT_GEN"], "created_time": "2020-04-13T18:04:59-0700", "currency": "USD", "disable_reason": 0.0, "end_advertiser": 1506473679510495.0, "end_advertiser_name": "Airbyte", "fb_entity": 85.0, "funding_source": 2825262454257003.0, "funding_source_details": {"id": "2825262454257003", "type": 1}, "has_migrated_permissions": true, "is_attribution_spec_system_default": true, "is_direct_deals_enabled": false, "is_in_3ds_authorization_enabled_market": false, "is_notifications_enabled": true, "is_personal": 0.0, "is_prepay_account": false, "is_tax_id_required": false, "min_campaign_group_spend_cap": 10000.0, "min_daily_budget": 100.0, "name": "Airbyte", "offsite_pixels_tos_accepted": true, "owner": 1506473679510495.0, "rf_spec": {"min_reach_limits": {"US": 200000, "CA": 200000, "GB": 200000, "AR": 200000, "AU": 200000, "AT": 200000, "BE": 200000, "BR": 200000, "CL": 200000, "CN": 200000, "CO": 200000, "HR": 200000, "DK": 200000, "DO": 200000, "EG": 200000, "FI": 200000, "FR": 200000, "DE": 200000, "GR": 200000, "HK": 200000, "IN": 200000, "ID": 200000, "IE": 200000, "IL": 200000, "IT": 200000, "JP": 200000, "JO": 200000, "KW": 200000, "LB": 200000, "MY": 200000, "MX": 200000, "NL": 200000, "NZ": 200000, "NG": 200000, "NO": 200000, "PK": 200000, "PA": 200000, "PE": 200000, "PH": 200000, "PL": 200000, "RU": 200000, "SA": 200000, "RS": 200000, "SG": 200000, "ZA": 200000, "KR": 200000, "ES": 200000, "SE": 200000, "CH": 200000, "TW": 200000, "TH": 200000, "TR": 200000, "AE": 200000, "VE": 200000, "PT": 200000, "LU": 200000, "BG": 200000, "CZ": 200000, "SI": 200000, "IS": 200000, "SK": 200000, "LT": 200000, "TT": 200000, "BD": 200000, "LK": 200000, "KE": 200000, "HU": 200000, "MA": 200000, "CY": 200000, "JM": 200000, "EC": 200000, "RO": 200000, "BO": 200000, "GT": 200000, "CR": 200000, "QA": 200000, "SV": 200000, "HN": 200000, "NI": 200000, "PY": 200000, "UY": 200000, "PR": 200000, "BA": 200000, "PS": 200000, "TN": 200000, "BH": 200000, "VN": 200000, "GH": 200000, "MU": 200000, "UA": 200000, "MT": 200000, "BS": 200000, "MV": 200000, "OM": 200000, "MK": 200000, "LV": 200000, "EE": 200000, "IQ": 200000, "DZ": 200000, "AL": 200000, "NP": 200000, "MO": 200000, "ME": 200000, "SN": 200000, "GE": 200000, "BN": 200000, "UG": 200000, "GP": 200000, "BB": 200000, "AZ": 200000, "TZ": 200000, "LY": 200000, "MQ": 200000, "CM": 200000, "BW": 200000, "ET": 200000, "KZ": 200000, "NA": 200000, "MG": 200000, "NC": 200000, "MD": 200000, "FJ": 200000, "BY": 200000, "JE": 200000, "GU": 200000, "YE": 200000, "ZM": 200000, "IM": 200000, "HT": 200000, "KH": 200000, "AW": 200000, "PF": 200000, "AF": 200000, "BM": 200000, "GY": 200000, "AM": 200000, "MW": 200000, "AG": 200000, "RW": 200000, "GG": 200000, "GM": 200000, "FO": 200000, "LC": 200000, "KY": 200000, "BJ": 200000, "AD": 200000, "GD": 200000, "VI": 200000, "BZ": 200000, "VC": 200000, "MN": 200000, "MZ": 200000, "ML": 200000, "AO": 200000, "GF": 200000, "UZ": 200000, "DJ": 200000, "BF": 200000, "MC": 200000, "TG": 200000, "GL": 200000, "GA": 200000, "GI": 200000, "CD": 200000, "KG": 200000, "PG": 200000, "BT": 200000, "KN": 200000, "SZ": 200000, "LS": 200000, "LA": 200000, "LI": 200000, "MP": 200000, "SR": 200000, "SC": 200000, "VG": 200000, "TC": 200000, "DM": 200000, "MR": 200000, "AX": 200000, "SM": 200000, "SL": 200000, "NE": 200000, "CG": 200000, "AI": 200000, "YT": 200000, "CV": 200000, "GN": 200000, "TM": 200000, "BI": 200000, "TJ": 200000, "VU": 200000, "SB": 200000, "ER": 200000, "WS": 200000, "AS": 200000, "FK": 200000, "GQ": 200000, "TO": 200000, "KM": 200000, "PW": 200000, "FM": 200000, "CF": 200000, "SO": 200000, "MH": 200000, "VA": 200000, "TD": 200000, "KI": 200000, "ST": 200000, "TV": 200000, "NR": 200000, "RE": 200000, "LR": 200000, "ZW": 200000, "CI": 200000, "MM": 200000, "AN": 200000, "AQ": 200000, "BQ": 200000, "BV": 200000, "IO": 200000, "CX": 200000, "CC": 200000, "CK": 200000, "CW": 200000, "TF": 200000, "GW": 200000, "HM": 200000, "XK": 200000, "MS": 200000, "NU": 200000, "NF": 200000, "PN": 200000, "BL": 200000, "SH": 200000, "MF": 200000, "PM": 200000, "SX": 200000, "GS": 200000, "SS": 200000, "SJ": 200000, "TL": 200000, "TK": 200000, "UM": 200000, "WF": 200000, "EH": 200000}, "countries": ["US", "CA", "GB", "AR", "AU", "AT", "BE", "BR", "CL", "CN", "CO", "HR", "DK", "DO", "EG", "FI", "FR", "DE", "GR", "HK", "IN", "ID", "IE", "IL", "IT", "JP", "JO", "KW", "LB", "MY", "MX", "NL", "NZ", "NG", "NO", "PK", "PA", "PE", "PH", "PL", "RU", "SA", "RS", "SG", "ZA", "KR", "ES", "SE", "CH", "TW", "TH", "TR", "AE", "VE", "PT", "LU", "BG", "CZ", "SI", "IS", "SK", "LT", "TT", "BD", "LK", "KE", "HU", "MA", "CY", "JM", "EC", "RO", "BO", "GT", "CR", "QA", "SV", "HN", "NI", "PY", "UY", "PR", "BA", "PS", "TN", "BH", "VN", "GH", "MU", "UA", "MT", "BS", "MV", "OM", "MK", "EE", "LV", "IQ", "DZ", "AL", "NP", "MO", "ME", "SN", "GE", "BN", "UG", "GP", "BB", "ZW", "CI", "AZ", "TZ", "LY", "MQ", "MM", "CM", "BW", "ET", "KZ", "NA", "MG", "NC", "MD", "FJ", "BY", "JE", "GU", "YE", "ZM", "IM", "HT", "KH", "AW", "PF", "AF", "BM", "GY", "AM", "MW", "AG", "RW", "GG", "GM", "FO", "LC", "KY", "BJ", "AD", "GD", "VI", "BZ", "VC", "MN", "MZ", "ML", "AO", "GF", "UZ", "DJ", "BF", "MC", "TG", "GL", "GA", "GI", "CD", "KG", "PG", "BT", "KN", "SZ", "LS", "LA", "LI", "MP", "SR", "SC", "VG", "TC", "DM", "MR", "AX", "SM", "SL", "NE", "CG", "AI", "YT", "LR", "CV", "GN", "TM", "BI", "TJ", "VU", "SB", "ER", "WS", "AS", "FK", "GQ", "TO", "KM", "PW", "FM", "CF", "SO", "MH", "VA", "TD", "KI", "ST", "TV", "NR", "RE", "AN", "AQ", "BQ", "BV", "IO", "CX", "CC", "CK", "CW", "TF", "GW", "HM", "XK", "MS", "NU", "NF", "PN", "BL", "SH", "MF", "PM", "SX", "GS", "SS", "SJ", "TL", "TK", "UM", "WF", "EH"], "min_campaign_duration": {"US": 1, "CA": 1, "GB": 1, "AR": 1, "AU": 1, "AT": 1, "BE": 1, "BR": 1, "CL": 1, "CN": 1, "CO": 1, "HR": 1, "DK": 1, "DO": 1, "EG": 1, "FI": 1, "FR": 1, "DE": 1, "GR": 1, "HK": 1, "IN": 1, "ID": 1, "IE": 1, "IL": 1, "IT": 1, "JP": 1, "JO": 1, "KW": 1, "LB": 1, "MY": 1, "MX": 1, "NL": 1, "NZ": 1, "NG": 1, "NO": 1, "PK": 1, "PA": 1, "PE": 1, "PH": 1, "PL": 1, "RU": 1, "SA": 1, "RS": 1, "SG": 1, "ZA": 1, "KR": 1, "ES": 1, "SE": 1, "CH": 1, "TW": 1, "TH": 1, "TR": 1, "AE": 1, "VE": 1, "PT": 1, "LU": 1, "BG": 1, "CZ": 1, "SI": 1, "IS": 1, "SK": 1, "LT": 1, "TT": 1, "BD": 1, "LK": 1, "KE": 1, "HU": 1, "MA": 1, "CY": 1, "JM": 1, "EC": 1, "RO": 1, "BO": 1, "GT": 1, "CR": 1, "QA": 1, "SV": 1, "HN": 1, "NI": 1, "PY": 1, "UY": 1, "PR": 1, "BA": 1, "PS": 1, "TN": 1, "BH": 1, "VN": 1, "GH": 1, "MU": 1, "UA": 1, "MT": 1, "BS": 1, "MV": 1, "OM": 1, "MK": 1, "LV": 1, "EE": 1, "IQ": 1, "DZ": 1, "AL": 1, "NP": 1, "MO": 1, "ME": 1, "SN": 1, "GE": 1, "BN": 1, "UG": 1, "GP": 1, "BB": 1, "AZ": 1, "TZ": 1, "LY": 1, "MQ": 1, "CM": 1, "BW": 1, "ET": 1, "KZ": 1, "NA": 1, "MG": 1, "NC": 1, "MD": 1, "FJ": 1, "BY": 1, "JE": 1, "GU": 1, "YE": 1, "ZM": 1, "IM": 1, "HT": 1, "KH": 1, "AW": 1, "PF": 1, "AF": 1, "BM": 1, "GY": 1, "AM": 1, "MW": 1, "AG": 1, "RW": 1, "GG": 1, "GM": 1, "FO": 1, "LC": 1, "KY": 1, "BJ": 1, "AD": 1, "GD": 1, "VI": 1, "BZ": 1, "VC": 1, "MN": 1, "MZ": 1, "ML": 1, "AO": 1, "GF": 1, "UZ": 1, "DJ": 1, "BF": 1, "MC": 1, "TG": 1, "GL": 1, "GA": 1, "GI": 1, "CD": 1, "KG": 1, "PG": 1, "BT": 1, "KN": 1, "SZ": 1, "LS": 1, "LA": 1, "LI": 1, "MP": 1, "SR": 1, "SC": 1, "VG": 1, "TC": 1, "DM": 1, "MR": 1, "AX": 1, "SM": 1, "SL": 1, "NE": 1, "CG": 1, "AI": 1, "YT": 1, "CV": 1, "GN": 1, "TM": 1, "BI": 1, "TJ": 1, "VU": 1, "SB": 1, "ER": 1, "WS": 1, "AS": 1, "FK": 1, "GQ": 1, "TO": 1, "KM": 1, "PW": 1, "FM": 1, "CF": 1, "SO": 1, "MH": 1, "VA": 1, "TD": 1, "KI": 1, "ST": 1, "TV": 1, "NR": 1, "RE": 1, "LR": 1, "ZW": 1, "CI": 1, "MM": 1, "AN": 1, "AQ": 1, "BQ": 1, "BV": 1, "IO": 1, "CX": 1, "CC": 1, "CK": 1, "CW": 1, "TF": 1, "GW": 1, "HM": 1, "XK": 1, "MS": 1, "NU": 1, "NF": 1, "PN": 1, "BL": 1, "SH": 1, "MF": 1, "PM": 1, "SX": 1, "GS": 1, "SS": 1, "SJ": 1, "TL": 1, "TK": 1, "UM": 1, "WF": 1, "EH": 1}, "max_campaign_duration": {"US": 90, "CA": 90, "GB": 90, "AR": 90, "AU": 90, "AT": 90, "BE": 90, "BR": 90, "CL": 90, "CN": 90, "CO": 90, "HR": 90, "DK": 90, "DO": 90, "EG": 90, "FI": 90, "FR": 90, "DE": 90, "GR": 90, "HK": 90, "IN": 90, "ID": 90, "IE": 90, "IL": 90, "IT": 90, "JP": 90, "JO": 90, "KW": 90, "LB": 90, "MY": 90, "MX": 90, "NL": 90, "NZ": 90, "NG": 90, "NO": 90, "PK": 90, "PA": 90, "PE": 90, "PH": 90, "PL": 90, "RU": 90, "SA": 90, "RS": 90, "SG": 90, "ZA": 90, "KR": 90, "ES": 90, "SE": 90, "CH": 90, "TW": 90, "TH": 90, "TR": 90, "AE": 90, "VE": 90, "PT": 90, "LU": 90, "BG": 90, "CZ": 90, "SI": 90, "IS": 90, "SK": 90, "LT": 90, "TT": 90, "BD": 90, "LK": 90, "KE": 90, "HU": 90, "MA": 90, "CY": 90, "JM": 90, "EC": 90, "RO": 90, "BO": 90, "GT": 90, "CR": 90, "QA": 90, "SV": 90, "HN": 90, "NI": 90, "PY": 90, "UY": 90, "PR": 90, "BA": 90, "PS": 90, "TN": 90, "BH": 90, "VN": 90, "GH": 90, "MU": 90, "UA": 90, "MT": 90, "BS": 90, "MV": 90, "OM": 90, "MK": 90, "LV": 90, "EE": 90, "IQ": 90, "DZ": 90, "AL": 90, "NP": 90, "MO": 90, "ME": 90, "SN": 90, "GE": 90, "BN": 90, "UG": 90, "GP": 90, "BB": 90, "AZ": 90, "TZ": 90, "LY": 90, "MQ": 90, "CM": 90, "BW": 90, "ET": 90, "KZ": 90, "NA": 90, "MG": 90, "NC": 90, "MD": 90, "FJ": 90, "BY": 90, "JE": 90, "GU": 90, "YE": 90, "ZM": 90, "IM": 90, "HT": 90, "KH": 90, "AW": 90, "PF": 90, "AF": 90, "BM": 90, "GY": 90, "AM": 90, "MW": 90, "AG": 90, "RW": 90, "GG": 90, "GM": 90, "FO": 90, "LC": 90, "KY": 90, "BJ": 90, "AD": 90, "GD": 90, "VI": 90, "BZ": 90, "VC": 90, "MN": 90, "MZ": 90, "ML": 90, "AO": 90, "GF": 90, "UZ": 90, "DJ": 90, "BF": 90, "MC": 90, "TG": 90, "GL": 90, "GA": 90, "GI": 90, "CD": 90, "KG": 90, "PG": 90, "BT": 90, "KN": 90, "SZ": 90, "LS": 90, "LA": 90, "LI": 90, "MP": 90, "SR": 90, "SC": 90, "VG": 90, "TC": 90, "DM": 90, "MR": 90, "AX": 90, "SM": 90, "SL": 90, "NE": 90, "CG": 90, "AI": 90, "YT": 90, "CV": 90, "GN": 90, "TM": 90, "BI": 90, "TJ": 90, "VU": 90, "SB": 90, "ER": 90, "WS": 90, "AS": 90, "FK": 90, "GQ": 90, "TO": 90, "KM": 90, "PW": 90, "FM": 90, "CF": 90, "SO": 90, "MH": 90, "VA": 90, "TD": 90, "KI": 90, "ST": 90, "TV": 90, "NR": 90, "RE": 90, "LR": 90, "ZW": 90, "CI": 90, "MM": 90, "AN": 90, "AQ": 90, "BQ": 90, "BV": 90, "IO": 90, "CX": 90, "CC": 90, "CK": 90, "CW": 90, "TF": 90, "GW": 90, "HM": 90, "XK": 90, "MS": 90, "NU": 90, "NF": 90, "PN": 90, "BL": 90, "SH": 90, "MF": 90, "PM": 90, "SX": 90, "GS": 90, "SS": 90, "SJ": 90, "TL": 90, "TK": 90, "UM": 90, "WF": 90, "EH": 90}, "max_days_to_finish": {"US": 180, "CA": 180, "GB": 180, "AR": 180, "AU": 180, "AT": 180, "BE": 180, "BR": 180, "CL": 180, "CN": 180, "CO": 180, "HR": 180, "DK": 180, "DO": 180, "EG": 180, "FI": 180, "FR": 180, "DE": 180, "GR": 180, "HK": 180, "IN": 180, "ID": 180, "IE": 180, "IL": 180, "IT": 180, "JP": 180, "JO": 180, "KW": 180, "LB": 180, "MY": 180, "MX": 180, "NL": 180, "NZ": 180, "NG": 180, "NO": 180, "PK": 180, "PA": 180, "PE": 180, "PH": 180, "PL": 180, "RU": 180, "SA": 180, "RS": 180, "SG": 180, "ZA": 180, "KR": 180, "ES": 180, "SE": 180, "CH": 180, "TW": 180, "TH": 180, "TR": 180, "AE": 180, "VE": 180, "PT": 180, "LU": 180, "BG": 180, "CZ": 180, "SI": 180, "IS": 180, "SK": 180, "LT": 180, "TT": 180, "BD": 180, "LK": 180, "KE": 180, "HU": 180, "MA": 180, "CY": 180, "JM": 180, "EC": 180, "RO": 180, "BO": 180, "GT": 180, "CR": 180, "QA": 180, "SV": 180, "HN": 180, "NI": 180, "PY": 180, "UY": 180, "PR": 180, "BA": 180, "PS": 180, "TN": 180, "BH": 180, "VN": 180, "GH": 180, "MU": 180, "UA": 180, "MT": 180, "BS": 180, "MV": 180, "OM": 180, "MK": 180, "LV": 180, "EE": 180, "IQ": 180, "DZ": 180, "AL": 180, "NP": 180, "MO": 180, "ME": 180, "SN": 180, "GE": 180, "BN": 180, "UG": 180, "GP": 180, "BB": 180, "AZ": 180, "TZ": 180, "LY": 180, "MQ": 180, "CM": 180, "BW": 180, "ET": 180, "KZ": 180, "NA": 180, "MG": 180, "NC": 180, "MD": 180, "FJ": 180, "BY": 180, "JE": 180, "GU": 180, "YE": 180, "ZM": 180, "IM": 180, "HT": 180, "KH": 180, "AW": 180, "PF": 180, "AF": 180, "BM": 180, "GY": 180, "AM": 180, "MW": 180, "AG": 180, "RW": 180, "GG": 180, "GM": 180, "FO": 180, "LC": 180, "KY": 180, "BJ": 180, "AD": 180, "GD": 180, "VI": 180, "BZ": 180, "VC": 180, "MN": 180, "MZ": 180, "ML": 180, "AO": 180, "GF": 180, "UZ": 180, "DJ": 180, "BF": 180, "MC": 180, "TG": 180, "GL": 180, "GA": 180, "GI": 180, "CD": 180, "KG": 180, "PG": 180, "BT": 180, "KN": 180, "SZ": 180, "LS": 180, "LA": 180, "LI": 180, "MP": 180, "SR": 180, "SC": 180, "VG": 180, "TC": 180, "DM": 180, "MR": 180, "AX": 180, "SM": 180, "SL": 180, "NE": 180, "CG": 180, "AI": 180, "YT": 180, "CV": 180, "GN": 180, "TM": 180, "BI": 180, "TJ": 180, "VU": 180, "SB": 180, "ER": 180, "WS": 180, "AS": 180, "FK": 180, "GQ": 180, "TO": 180, "KM": 180, "PW": 180, "FM": 180, "CF": 180, "SO": 180, "MH": 180, "VA": 180, "TD": 180, "KI": 180, "ST": 180, "TV": 180, "NR": 180, "RE": 180, "LR": 180, "ZW": 180, "CI": 180, "MM": 180, "AN": 180, "AQ": 180, "BQ": 180, "BV": 180, "IO": 180, "CX": 180, "CC": 180, "CK": 180, "CW": 180, "TF": 180, "GW": 180, "HM": 180, "XK": 180, "MS": 180, "NU": 180, "NF": 180, "PN": 180, "BL": 180, "SH": 180, "MF": 180, "PM": 180, "SX": 180, "GS": 180, "SS": 180, "SJ": 180, "TL": 180, "TK": 180, "UM": 180, "WF": 180, "EH": 180}, "global_io_max_campaign_duration": 100}, "spend_cap": "0", "tax_id_status": 0.0, "tax_id_type": "0", "timezone_id": 1.0, "timezone_name": "America/Los_Angeles", "timezone_offset_hours_utc": -8.0, "tos_accepted": {"web_custom_audience_tos": 1}, "user_tasks": ["DRAFT", "ANALYZE", "ADVERTISE", "MANAGE"]}, "emitted_at": 1699644186066} -{"stream":"ads","data":{"bid_type":"ABSOLUTE_OCPM","account_id":"212551616838260","campaign_id":"23853619670350398","adset_id":"23853619670380398","status":"ACTIVE","creative":{"id":"23853666125630398"},"id":"23853620198790398","updated_time":"2023-03-21T22:33:56-0700","created_time":"2023-03-17T08:04:29-0700","name":"Don't Compromise Between Cost/Relaibility","targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"effective_status":"ACTIVE","last_updated_by_app_id":"119211728144504","source_ad_id":"0","tracking_specs":[{"action.type":["offsite_conversion"],"fb_pixel":["917042523049733"]},{"action.type":["post_engagement"],"page":["112704783733939"],"post":["660122622785523","662226992575086"]},{"action.type":["link_click"],"post":["660122622785523","662226992575086"],"post.wall":["112704783733939"]}],"conversion_specs":[{"action.type":["offsite_conversion"],"conversion_id":["6015304265216283"]}]},"emitted_at":1682686047377} -{"stream":"ad_sets","data":{"name":"Lookalike audience_Free Connector Program","promoted_object":{"pixel_id":"917042523049733","custom_event_type":"COMPLETE_REGISTRATION"},"id":"23853619670380398","account_id":"212551616838260","updated_time":"2023-03-21T14:20:51-0700","daily_budget":2000,"budget_remaining":2000,"effective_status":"ACTIVE","campaign_id":"23853619670350398","created_time":"2023-03-17T08:04:28-0700","start_time":"2023-03-17T08:04:28-0700","lifetime_budget":0,"targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"bid_strategy":"LOWEST_COST_WITHOUT_CAP"},"emitted_at":1692180821847} +{"stream": "ad_account", "data": {"id": "act_212551616838260", "account_id": "212551616838260", "account_status": 1, "age": 1402.6937847222, "amount_spent": "39125", "balance": "0", "business": {"id": "1506473679510495", "name": "Airbyte"}, "business_city": "", "business_country_code": "US", "business_name": "", "business_street": "", "business_street2": "", "can_create_brand_lift_study": false, "capabilities": ["CAN_CREATE_CALL_ADS", "CAN_SEE_GROWTH_OPPORTUNITY_DATA", "ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT", "CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE", "CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S", "ENABLE_BIZ_DISCO_ADS", "ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS", "ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS", "ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS", "ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE", "ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION", "ENABLE_PAC_FOR_BIZ_DISCO_ADS", "CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_REACH_AND_FREQUENCY", "CAN_USE_RECURRING_BUDGET", "HAS_VALID_PAYMENT_METHODS", "CAN_USE_LINK_CLICK_BILLING_EVENT", "CAN_USE_CPA_BILLING_EVENT", "CAN_SEE_NEW_CONVERSION_WINDOW_NUX", "ADS_INSTREAM_INTERFACE_INTEGRITY", "ADS_INSTREAM_LINK_CLICK", "ADS_INSTREAM_LINK_CLICK_IMAGE", "ADS_IN_OBJECTIVES_DEPRECATION", "MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES", "CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL", "ALLOW_INSTREAM_ONLY_FOR_REACH", "ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS", "CAN_CREATE_INSTAGRAM_EXPLORE_ADS", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY", "ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN", "INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW", "ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION", "ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY", "WHATSAPP_DESTINATION_ADS", "CTM_ADS_CREATION_CLICK_TO_DIRECT", "CTW_ADS_ENABLE_IG_FEED_PLACEMENT", "CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE", "CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER", "CTW_ADS_TRUSTED_TIER_ADVERTISER", "ADS_PLACEMENT_MARKETPLACE", "ADNW_DISABLE_INSTREAM_AND_WEB_PLACEMENT", "CAN_CHANGE_BILLING_THRESHOLD", "CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING", "CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING", "ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D", "ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT", "ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET", "CAN_SEE_APP_AD_EVENTS", "CAN_SEE_NEW_STANDARD_EVENTS_BETA", "CAN_SEE_VCK_HOLIDAY_TEMPLATES", "ENABLE_DCO_FOR_FB_STORY_ADS", "CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT", "CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY", "CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES", "CAN_USE_IG_SEARCH_PLACEMENT", "CAN_USE_IG_SEARCH_RESULTS_AUTO_PLACEMENT", "CAN_USE_IG_REELS_PAC_CAROUSEL", "CAN_USE_IG_REELS_POSITION", "CAN_SEE_CONVERSION_LIFT_SUMMARY", "CAN_USE_IG_PROFILE_FEED_POSITION", "CAN_USE_IG_REELS_REACH_AND_FREQUENCY", "CAN_USE_IG_REELS_OVERLAY_POSITION", "CAN_USE_IG_REELS_OVERLAY_PAC", "CAN_USE_IG_SHOP_TAB_PAC", "CAN_SEE_LEARNING_STAGE", "ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS", "ENABLE_MESSENGER_INBOX_VIDEO_ADS", "ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS", "ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS", "ENABLE_REACH_FOR_FB_STORY_ADS", "CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT", "ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY", "ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS", "ENABLE_LEAD_GEN_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW", "CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT", "ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION", "CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT", "RF_ALLOW_MARKETPLACE_ACCOUNT", "RF_ALLOW_SEARCH_ACCOUNT", "VERTICAL_VIDEO_PAC_INSTREAM_UPSELL", "IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH", "ADS_BM_REQUIREMENTS_OCT_15_RELEASE", "ENABLE_POST_ENGAGEMENT_FOR_FB_STORY", "ENBABLE_CATALOG_SALES_FOR_FB_STORY", "CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS", "CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS", "IS_NON_TAIL_AD_ACCOUNT", "IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT", "IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT", "IS_IN_ODAX_EXPERIENCE", "IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT", "CAN_USE_MARKETPLACE_DESKTOP", "ADS_MERCHANT_OVERLAYS_DEPRECATION", "CONNECTIONS_DEPRECATION_V2", "CAN_USE_LIVE_VIDEO_FOR_THRUPLAY", "CAN_SEE_HEC_AM_FLOW", "CAN_SEE_POLITICAL_FLOW", "ADS_INSTREAM_PLACEMENT_CATALOG_SALES", "ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS", "ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS", "ENABLE_REACH_FOR_FB_GROUP_TAB_ADS", "CAN_USE_CONVERSATIONS_OPTIMIZATION", "ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS", "CAN_USE_IG_STORY_POLLS_PAC_CREATION", "IOS14_CEO_CAMPAIGN_CREATION", "ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS", "DIGITAL_CIRCULAR_ADS", "CAN_SEE_SAFR_V3_FLOW", "CAN_USE_FB_REELS_POSITION", "CAN_USE_ADS_ON_FB_REELS_POSITION", "CAN_USE_FB_REELS_AUTO_PLACEMENT", "ENABLE_FB_REELS_CREATION_PAC_ADS", "ENABLE_FB_REELS_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS", "RF_CPA_BILLING_DEPRECATION_PHASE_2", "ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION", "ADS_RF_FB_REELS_PLACEMENT", "REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY", "ELIGIBLE_FOR_TEXT_GEN", "CAN_USE_BUDGET_SCHEDULING_API", "ADS_AEMV2_HAS_LAUNCHED"], "created_time": "2020-04-13T18:04:59-0700", "currency": "USD", "disable_reason": 0.0, "end_advertiser": 1506473679510495.0, "end_advertiser_name": "Airbyte", "fb_entity": 85.0, "funding_source": 2825262454257003.0, "funding_source_details": {"id": "2825262454257003", "type": 1}, "has_migrated_permissions": true, "is_attribution_spec_system_default": true, "is_direct_deals_enabled": false, "is_in_3ds_authorization_enabled_market": false, "is_notifications_enabled": true, "is_personal": 0.0, "is_prepay_account": false, "is_tax_id_required": false, "min_campaign_group_spend_cap": 10000.0, "min_daily_budget": 100.0, "name": "Airbyte", "offsite_pixels_tos_accepted": true, "owner": 1506473679510495.0, "rf_spec": {"min_reach_limits": {"US": 200000, "CA": 200000, "GB": 200000, "AR": 200000, "AU": 200000, "AT": 200000, "BE": 200000, "BR": 200000, "CL": 200000, "CN": 200000, "CO": 200000, "HR": 200000, "DK": 200000, "DO": 200000, "EG": 200000, "FI": 200000, "FR": 200000, "DE": 200000, "GR": 200000, "HK": 200000, "IN": 200000, "ID": 200000, "IE": 200000, "IL": 200000, "IT": 200000, "JP": 200000, "JO": 200000, "KW": 200000, "LB": 200000, "MY": 200000, "MX": 200000, "NL": 200000, "NZ": 200000, "NG": 200000, "NO": 200000, "PK": 200000, "PA": 200000, "PE": 200000, "PH": 200000, "PL": 200000, "RU": 200000, "SA": 200000, "RS": 200000, "SG": 200000, "ZA": 200000, "KR": 200000, "ES": 200000, "SE": 200000, "CH": 200000, "TW": 200000, "TH": 200000, "TR": 200000, "AE": 200000, "VE": 200000, "PT": 200000, "LU": 200000, "BG": 200000, "CZ": 200000, "SI": 200000, "IS": 200000, "SK": 200000, "LT": 200000, "TT": 200000, "BD": 200000, "LK": 200000, "KE": 200000, "HU": 200000, "MA": 200000, "CY": 200000, "JM": 200000, "EC": 200000, "RO": 200000, "BO": 200000, "GT": 200000, "CR": 200000, "QA": 200000, "SV": 200000, "HN": 200000, "NI": 200000, "PY": 200000, "UY": 200000, "PR": 200000, "BA": 200000, "PS": 200000, "TN": 200000, "BH": 200000, "VN": 200000, "GH": 200000, "MU": 200000, "UA": 200000, "MT": 200000, "BS": 200000, "MV": 200000, "OM": 200000, "MK": 200000, "LV": 200000, "EE": 200000, "IQ": 200000, "DZ": 200000, "AL": 200000, "NP": 200000, "MO": 200000, "ME": 200000, "SN": 200000, "GE": 200000, "BN": 200000, "UG": 200000, "GP": 200000, "BB": 200000, "AZ": 200000, "TZ": 200000, "LY": 200000, "MQ": 200000, "CM": 200000, "BW": 200000, "ET": 200000, "KZ": 200000, "NA": 200000, "MG": 200000, "NC": 200000, "MD": 200000, "FJ": 200000, "BY": 200000, "JE": 200000, "GU": 200000, "YE": 200000, "ZM": 200000, "IM": 200000, "HT": 200000, "KH": 200000, "AW": 200000, "PF": 200000, "AF": 200000, "BM": 200000, "GY": 200000, "AM": 200000, "MW": 200000, "AG": 200000, "RW": 200000, "GG": 200000, "GM": 200000, "FO": 200000, "LC": 200000, "KY": 200000, "BJ": 200000, "AD": 200000, "GD": 200000, "VI": 200000, "BZ": 200000, "VC": 200000, "MN": 200000, "MZ": 200000, "ML": 200000, "AO": 200000, "GF": 200000, "UZ": 200000, "DJ": 200000, "BF": 200000, "MC": 200000, "TG": 200000, "GL": 200000, "GA": 200000, "GI": 200000, "CD": 200000, "KG": 200000, "PG": 200000, "BT": 200000, "KN": 200000, "SZ": 200000, "LS": 200000, "LA": 200000, "LI": 200000, "MP": 200000, "SR": 200000, "SC": 200000, "VG": 200000, "TC": 200000, "DM": 200000, "MR": 200000, "AX": 200000, "SM": 200000, "SL": 200000, "NE": 200000, "CG": 200000, "AI": 200000, "YT": 200000, "CV": 200000, "GN": 200000, "TM": 200000, "BI": 200000, "TJ": 200000, "VU": 200000, "SB": 200000, "ER": 200000, "WS": 200000, "AS": 200000, "FK": 200000, "GQ": 200000, "TO": 200000, "KM": 200000, "PW": 200000, "FM": 200000, "CF": 200000, "SO": 200000, "MH": 200000, "VA": 200000, "TD": 200000, "KI": 200000, "ST": 200000, "TV": 200000, "NR": 200000, "RE": 200000, "LR": 200000, "ZW": 200000, "CI": 200000, "MM": 200000, "AN": 200000, "AQ": 200000, "BQ": 200000, "BV": 200000, "IO": 200000, "CX": 200000, "CC": 200000, "CK": 200000, "CW": 200000, "TF": 200000, "GW": 200000, "HM": 200000, "XK": 200000, "MS": 200000, "NU": 200000, "NF": 200000, "PN": 200000, "BL": 200000, "SH": 200000, "MF": 200000, "PM": 200000, "SX": 200000, "GS": 200000, "SS": 200000, "SJ": 200000, "TL": 200000, "TK": 200000, "UM": 200000, "WF": 200000, "EH": 200000}, "countries": ["US", "CA", "GB", "AR", "AU", "AT", "BE", "BR", "CL", "CN", "CO", "HR", "DK", "DO", "EG", "FI", "FR", "DE", "GR", "HK", "IN", "ID", "IE", "IL", "IT", "JP", "JO", "KW", "LB", "MY", "MX", "NL", "NZ", "NG", "NO", "PK", "PA", "PE", "PH", "PL", "RU", "SA", "RS", "SG", "ZA", "KR", "ES", "SE", "CH", "TW", "TH", "TR", "AE", "VE", "PT", "LU", "BG", "CZ", "SI", "IS", "SK", "LT", "TT", "BD", "LK", "KE", "HU", "MA", "CY", "JM", "EC", "RO", "BO", "GT", "CR", "QA", "SV", "HN", "NI", "PY", "UY", "PR", "BA", "PS", "TN", "BH", "VN", "GH", "MU", "UA", "MT", "BS", "MV", "OM", "MK", "EE", "LV", "IQ", "DZ", "AL", "NP", "MO", "ME", "SN", "GE", "BN", "UG", "GP", "BB", "ZW", "CI", "AZ", "TZ", "LY", "MQ", "MM", "CM", "BW", "ET", "KZ", "NA", "MG", "NC", "MD", "FJ", "BY", "JE", "GU", "YE", "ZM", "IM", "HT", "KH", "AW", "PF", "AF", "BM", "GY", "AM", "MW", "AG", "RW", "GG", "GM", "FO", "LC", "KY", "BJ", "AD", "GD", "VI", "BZ", "VC", "MN", "MZ", "ML", "AO", "GF", "UZ", "DJ", "BF", "MC", "TG", "GL", "GA", "GI", "CD", "KG", "PG", "BT", "KN", "SZ", "LS", "LA", "LI", "MP", "SR", "SC", "VG", "TC", "DM", "MR", "AX", "SM", "SL", "NE", "CG", "AI", "YT", "LR", "CV", "GN", "TM", "BI", "TJ", "VU", "SB", "ER", "WS", "AS", "FK", "GQ", "TO", "KM", "PW", "FM", "CF", "SO", "MH", "VA", "TD", "KI", "ST", "TV", "NR", "RE", "AN", "AQ", "BQ", "BV", "IO", "CX", "CC", "CK", "CW", "TF", "GW", "HM", "XK", "MS", "NU", "NF", "PN", "BL", "SH", "MF", "PM", "SX", "GS", "SS", "SJ", "TL", "TK", "UM", "WF", "EH"], "min_campaign_duration": {"US": 1, "CA": 1, "GB": 1, "AR": 1, "AU": 1, "AT": 1, "BE": 1, "BR": 1, "CL": 1, "CN": 1, "CO": 1, "HR": 1, "DK": 1, "DO": 1, "EG": 1, "FI": 1, "FR": 1, "DE": 1, "GR": 1, "HK": 1, "IN": 1, "ID": 1, "IE": 1, "IL": 1, "IT": 1, "JP": 1, "JO": 1, "KW": 1, "LB": 1, "MY": 1, "MX": 1, "NL": 1, "NZ": 1, "NG": 1, "NO": 1, "PK": 1, "PA": 1, "PE": 1, "PH": 1, "PL": 1, "RU": 1, "SA": 1, "RS": 1, "SG": 1, "ZA": 1, "KR": 1, "ES": 1, "SE": 1, "CH": 1, "TW": 1, "TH": 1, "TR": 1, "AE": 1, "VE": 1, "PT": 1, "LU": 1, "BG": 1, "CZ": 1, "SI": 1, "IS": 1, "SK": 1, "LT": 1, "TT": 1, "BD": 1, "LK": 1, "KE": 1, "HU": 1, "MA": 1, "CY": 1, "JM": 1, "EC": 1, "RO": 1, "BO": 1, "GT": 1, "CR": 1, "QA": 1, "SV": 1, "HN": 1, "NI": 1, "PY": 1, "UY": 1, "PR": 1, "BA": 1, "PS": 1, "TN": 1, "BH": 1, "VN": 1, "GH": 1, "MU": 1, "UA": 1, "MT": 1, "BS": 1, "MV": 1, "OM": 1, "MK": 1, "LV": 1, "EE": 1, "IQ": 1, "DZ": 1, "AL": 1, "NP": 1, "MO": 1, "ME": 1, "SN": 1, "GE": 1, "BN": 1, "UG": 1, "GP": 1, "BB": 1, "AZ": 1, "TZ": 1, "LY": 1, "MQ": 1, "CM": 1, "BW": 1, "ET": 1, "KZ": 1, "NA": 1, "MG": 1, "NC": 1, "MD": 1, "FJ": 1, "BY": 1, "JE": 1, "GU": 1, "YE": 1, "ZM": 1, "IM": 1, "HT": 1, "KH": 1, "AW": 1, "PF": 1, "AF": 1, "BM": 1, "GY": 1, "AM": 1, "MW": 1, "AG": 1, "RW": 1, "GG": 1, "GM": 1, "FO": 1, "LC": 1, "KY": 1, "BJ": 1, "AD": 1, "GD": 1, "VI": 1, "BZ": 1, "VC": 1, "MN": 1, "MZ": 1, "ML": 1, "AO": 1, "GF": 1, "UZ": 1, "DJ": 1, "BF": 1, "MC": 1, "TG": 1, "GL": 1, "GA": 1, "GI": 1, "CD": 1, "KG": 1, "PG": 1, "BT": 1, "KN": 1, "SZ": 1, "LS": 1, "LA": 1, "LI": 1, "MP": 1, "SR": 1, "SC": 1, "VG": 1, "TC": 1, "DM": 1, "MR": 1, "AX": 1, "SM": 1, "SL": 1, "NE": 1, "CG": 1, "AI": 1, "YT": 1, "CV": 1, "GN": 1, "TM": 1, "BI": 1, "TJ": 1, "VU": 1, "SB": 1, "ER": 1, "WS": 1, "AS": 1, "FK": 1, "GQ": 1, "TO": 1, "KM": 1, "PW": 1, "FM": 1, "CF": 1, "SO": 1, "MH": 1, "VA": 1, "TD": 1, "KI": 1, "ST": 1, "TV": 1, "NR": 1, "RE": 1, "LR": 1, "ZW": 1, "CI": 1, "MM": 1, "AN": 1, "AQ": 1, "BQ": 1, "BV": 1, "IO": 1, "CX": 1, "CC": 1, "CK": 1, "CW": 1, "TF": 1, "GW": 1, "HM": 1, "XK": 1, "MS": 1, "NU": 1, "NF": 1, "PN": 1, "BL": 1, "SH": 1, "MF": 1, "PM": 1, "SX": 1, "GS": 1, "SS": 1, "SJ": 1, "TL": 1, "TK": 1, "UM": 1, "WF": 1, "EH": 1}, "max_campaign_duration": {"US": 90, "CA": 90, "GB": 90, "AR": 90, "AU": 90, "AT": 90, "BE": 90, "BR": 90, "CL": 90, "CN": 90, "CO": 90, "HR": 90, "DK": 90, "DO": 90, "EG": 90, "FI": 90, "FR": 90, "DE": 90, "GR": 90, "HK": 90, "IN": 90, "ID": 90, "IE": 90, "IL": 90, "IT": 90, "JP": 90, "JO": 90, "KW": 90, "LB": 90, "MY": 90, "MX": 90, "NL": 90, "NZ": 90, "NG": 90, "NO": 90, "PK": 90, "PA": 90, "PE": 90, "PH": 90, "PL": 90, "RU": 90, "SA": 90, "RS": 90, "SG": 90, "ZA": 90, "KR": 90, "ES": 90, "SE": 90, "CH": 90, "TW": 90, "TH": 90, "TR": 90, "AE": 90, "VE": 90, "PT": 90, "LU": 90, "BG": 90, "CZ": 90, "SI": 90, "IS": 90, "SK": 90, "LT": 90, "TT": 90, "BD": 90, "LK": 90, "KE": 90, "HU": 90, "MA": 90, "CY": 90, "JM": 90, "EC": 90, "RO": 90, "BO": 90, "GT": 90, "CR": 90, "QA": 90, "SV": 90, "HN": 90, "NI": 90, "PY": 90, "UY": 90, "PR": 90, "BA": 90, "PS": 90, "TN": 90, "BH": 90, "VN": 90, "GH": 90, "MU": 90, "UA": 90, "MT": 90, "BS": 90, "MV": 90, "OM": 90, "MK": 90, "LV": 90, "EE": 90, "IQ": 90, "DZ": 90, "AL": 90, "NP": 90, "MO": 90, "ME": 90, "SN": 90, "GE": 90, "BN": 90, "UG": 90, "GP": 90, "BB": 90, "AZ": 90, "TZ": 90, "LY": 90, "MQ": 90, "CM": 90, "BW": 90, "ET": 90, "KZ": 90, "NA": 90, "MG": 90, "NC": 90, "MD": 90, "FJ": 90, "BY": 90, "JE": 90, "GU": 90, "YE": 90, "ZM": 90, "IM": 90, "HT": 90, "KH": 90, "AW": 90, "PF": 90, "AF": 90, "BM": 90, "GY": 90, "AM": 90, "MW": 90, "AG": 90, "RW": 90, "GG": 90, "GM": 90, "FO": 90, "LC": 90, "KY": 90, "BJ": 90, "AD": 90, "GD": 90, "VI": 90, "BZ": 90, "VC": 90, "MN": 90, "MZ": 90, "ML": 90, "AO": 90, "GF": 90, "UZ": 90, "DJ": 90, "BF": 90, "MC": 90, "TG": 90, "GL": 90, "GA": 90, "GI": 90, "CD": 90, "KG": 90, "PG": 90, "BT": 90, "KN": 90, "SZ": 90, "LS": 90, "LA": 90, "LI": 90, "MP": 90, "SR": 90, "SC": 90, "VG": 90, "TC": 90, "DM": 90, "MR": 90, "AX": 90, "SM": 90, "SL": 90, "NE": 90, "CG": 90, "AI": 90, "YT": 90, "CV": 90, "GN": 90, "TM": 90, "BI": 90, "TJ": 90, "VU": 90, "SB": 90, "ER": 90, "WS": 90, "AS": 90, "FK": 90, "GQ": 90, "TO": 90, "KM": 90, "PW": 90, "FM": 90, "CF": 90, "SO": 90, "MH": 90, "VA": 90, "TD": 90, "KI": 90, "ST": 90, "TV": 90, "NR": 90, "RE": 90, "LR": 90, "ZW": 90, "CI": 90, "MM": 90, "AN": 90, "AQ": 90, "BQ": 90, "BV": 90, "IO": 90, "CX": 90, "CC": 90, "CK": 90, "CW": 90, "TF": 90, "GW": 90, "HM": 90, "XK": 90, "MS": 90, "NU": 90, "NF": 90, "PN": 90, "BL": 90, "SH": 90, "MF": 90, "PM": 90, "SX": 90, "GS": 90, "SS": 90, "SJ": 90, "TL": 90, "TK": 90, "UM": 90, "WF": 90, "EH": 90}, "max_days_to_finish": {"US": 180, "CA": 180, "GB": 180, "AR": 180, "AU": 180, "AT": 180, "BE": 180, "BR": 180, "CL": 180, "CN": 180, "CO": 180, "HR": 180, "DK": 180, "DO": 180, "EG": 180, "FI": 180, "FR": 180, "DE": 180, "GR": 180, "HK": 180, "IN": 180, "ID": 180, "IE": 180, "IL": 180, "IT": 180, "JP": 180, "JO": 180, "KW": 180, "LB": 180, "MY": 180, "MX": 180, "NL": 180, "NZ": 180, "NG": 180, "NO": 180, "PK": 180, "PA": 180, "PE": 180, "PH": 180, "PL": 180, "RU": 180, "SA": 180, "RS": 180, "SG": 180, "ZA": 180, "KR": 180, "ES": 180, "SE": 180, "CH": 180, "TW": 180, "TH": 180, "TR": 180, "AE": 180, "VE": 180, "PT": 180, "LU": 180, "BG": 180, "CZ": 180, "SI": 180, "IS": 180, "SK": 180, "LT": 180, "TT": 180, "BD": 180, "LK": 180, "KE": 180, "HU": 180, "MA": 180, "CY": 180, "JM": 180, "EC": 180, "RO": 180, "BO": 180, "GT": 180, "CR": 180, "QA": 180, "SV": 180, "HN": 180, "NI": 180, "PY": 180, "UY": 180, "PR": 180, "BA": 180, "PS": 180, "TN": 180, "BH": 180, "VN": 180, "GH": 180, "MU": 180, "UA": 180, "MT": 180, "BS": 180, "MV": 180, "OM": 180, "MK": 180, "LV": 180, "EE": 180, "IQ": 180, "DZ": 180, "AL": 180, "NP": 180, "MO": 180, "ME": 180, "SN": 180, "GE": 180, "BN": 180, "UG": 180, "GP": 180, "BB": 180, "AZ": 180, "TZ": 180, "LY": 180, "MQ": 180, "CM": 180, "BW": 180, "ET": 180, "KZ": 180, "NA": 180, "MG": 180, "NC": 180, "MD": 180, "FJ": 180, "BY": 180, "JE": 180, "GU": 180, "YE": 180, "ZM": 180, "IM": 180, "HT": 180, "KH": 180, "AW": 180, "PF": 180, "AF": 180, "BM": 180, "GY": 180, "AM": 180, "MW": 180, "AG": 180, "RW": 180, "GG": 180, "GM": 180, "FO": 180, "LC": 180, "KY": 180, "BJ": 180, "AD": 180, "GD": 180, "VI": 180, "BZ": 180, "VC": 180, "MN": 180, "MZ": 180, "ML": 180, "AO": 180, "GF": 180, "UZ": 180, "DJ": 180, "BF": 180, "MC": 180, "TG": 180, "GL": 180, "GA": 180, "GI": 180, "CD": 180, "KG": 180, "PG": 180, "BT": 180, "KN": 180, "SZ": 180, "LS": 180, "LA": 180, "LI": 180, "MP": 180, "SR": 180, "SC": 180, "VG": 180, "TC": 180, "DM": 180, "MR": 180, "AX": 180, "SM": 180, "SL": 180, "NE": 180, "CG": 180, "AI": 180, "YT": 180, "CV": 180, "GN": 180, "TM": 180, "BI": 180, "TJ": 180, "VU": 180, "SB": 180, "ER": 180, "WS": 180, "AS": 180, "FK": 180, "GQ": 180, "TO": 180, "KM": 180, "PW": 180, "FM": 180, "CF": 180, "SO": 180, "MH": 180, "VA": 180, "TD": 180, "KI": 180, "ST": 180, "TV": 180, "NR": 180, "RE": 180, "LR": 180, "ZW": 180, "CI": 180, "MM": 180, "AN": 180, "AQ": 180, "BQ": 180, "BV": 180, "IO": 180, "CX": 180, "CC": 180, "CK": 180, "CW": 180, "TF": 180, "GW": 180, "HM": 180, "XK": 180, "MS": 180, "NU": 180, "NF": 180, "PN": 180, "BL": 180, "SH": 180, "MF": 180, "PM": 180, "SX": 180, "GS": 180, "SS": 180, "SJ": 180, "TL": 180, "TK": 180, "UM": 180, "WF": 180, "EH": 180}, "global_io_max_campaign_duration": 100}, "spend_cap": "0", "tax_id_status": 0.0, "tax_id_type": "0", "timezone_id": 1.0, "timezone_name": "America/Los_Angeles", "timezone_offset_hours_utc": -8.0, "tos_accepted": {"web_custom_audience_tos": 1}, "user_tasks": ["DRAFT", "ANALYZE", "ADVERTISE", "MANAGE"]}, "emitted_at": 1708020062150} +{"stream": "ads", "data": {"id": "23853620229650398", "bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666124230398"}, "updated_time": "2023-03-21T22:41:46-0700", "created_time": "2023-03-17T08:04:31-0700", "name": "With The Highest Standard for Reliability", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["link_click"], "post": ["662226902575095"], "post.wall": ["112704783733939"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["662226902575095"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1707135365030} +{"stream": "ad_sets", "data": {"id": "23853619670380398", "name": "Lookalike audience_Free Connector Program", "promoted_object": {"pixel_id": "917042523049733", "custom_event_type": "COMPLETE_REGISTRATION"}, "account_id": "212551616838260", "updated_time": "2023-03-21T14:20:51-0700", "daily_budget": 2000.0, "budget_remaining": 2000.0, "effective_status": "ACTIVE", "campaign_id": "23853619670350398", "created_time": "2023-03-17T08:04:28-0700", "start_time": "2023-03-17T08:04:28-0700", "lifetime_budget": 0.0, "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "bid_strategy": "LOWEST_COST_WITHOUT_CAP"}, "emitted_at": 1707135364623} {"stream":"campaigns","data":{"id":"23846542053890398","account_id":"212551616838260","budget_rebalance_flag":false,"budget_remaining":0.0,"buying_type":"AUCTION","created_time":"2021-01-18T21:36:42-0800","configured_status":"PAUSED","effective_status":"PAUSED","name":"Fake Campaign 0","objective":"MESSAGES","smart_promotion_type":"GUIDED_CREATION","source_campaign_id":0.0,"special_ad_category":"NONE","start_time":"1969-12-31T15:59:59-0800","status":"PAUSED","updated_time":"2021-02-18T01:00:02-0800"},"emitted_at":1694795155769} {"stream": "custom_audiences", "data": {"id": "23853683587660398", "account_id": "212551616838260", "approximate_count_lower_bound": 4700, "approximate_count_upper_bound": 5500, "customer_file_source": "PARTNER_PROVIDED_ONLY", "data_source": {"type": "UNKNOWN", "sub_type": "ANYTHING", "creation_params": "[]"}, "delivery_status": {"code": 200, "description": "This audience is ready for use."}, "description": "Custom Audience-Web Traffic [ALL] - _copy", "is_value_based": false, "name": "Web Traffic [ALL] - _copy", "operation_status": {"code": 200, "description": "Normal"}, "permission_for_actions": {"can_edit": true, "can_see_insight": "True", "can_share": "True", "subtype_supports_lookalike": "True", "supports_recipient_lookalike": "False"}, "retention_days": 0, "subtype": "CUSTOM", "time_content_updated": 1679433484, "time_created": 1679433479, "time_updated": 1679433484}, "emitted_at": 1698925454024} -{"stream":"ad_creatives","data":{"id":"23844568440620398","account_id":"212551616838260","actor_id":"112704783733939","asset_feed_spec":{"images":[{"adlabels":[{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"191x100":[[0,411],[589,719]]}},{"adlabels":[{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"100x100":[[12,282],[574,844]]}},{"adlabels":[{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[14,72],[562,1046]]}},{"adlabels":[{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[0,0],[589,1047]]}}],"bodies":[{"adlabels":[{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"}],"text":""}],"call_to_action_types":["LEARN_MORE"],"descriptions":[{"text":"Unmatched attribution, ad performances, and lead conversion, by unlocking your ad-blocked traffic across all your tools."}],"link_urls":[{"adlabels":[{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"}],"website_url":"http://dataline.io/","display_url":""}],"titles":[{"adlabels":[{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"}],"text":"Unblock all your adblocked traffic"}],"ad_formats":["AUTOMATIC_FORMAT"],"asset_customization_rules":[{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["instagram","audience_network","messenger"],"instagram_positions":["story"],"messenger_positions":["story"],"audience_network_positions":["classic"]},"image_label":{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"},"body_label":{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},"link_url_label":{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},"title_label":{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},"priority":1},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["right_hand_column","instant_article","search"]},"image_label":{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"},"body_label":{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},"link_url_label":{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},"title_label":{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},"priority":2},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["story"]},"image_label":{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"},"body_label":{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"},"link_url_label":{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"},"title_label":{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"},"priority":3},{"customization_spec":{"age_max":65,"age_min":13},"image_label":{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"},"body_label":{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},"link_url_label":{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},"title_label":{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},"priority":4}],"optimization_type":"PLACEMENT","reasons_to_shop":false,"shops_bundle":false,"additional_data":{"multi_share_end_card":false,"is_click_to_message":false}},"effective_object_story_id":"112704783733939_117519556585795","name":"{{product.name}} 2020-04-21-49cbe5bd90ed9861ea68bb38f7d6fc7c","instagram_actor_id":"3437258706290825","object_story_spec":{"page_id":"112704783733939","instagram_actor_id":"3437258706290825"},"object_type":"SHARE","status":"ACTIVE","thumbnail_url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/93287504_23844521781140398_125048020067680256_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=a3999f&_nc_ohc=-TT4Z0FkPeYAX97qejq&_nc_ht=scontent-dus1-1.xx&edm=AAT1rw8EAAAA&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ur=58080a&oh=00_AfBjMrayWFyOLmIgVt8Owtv2fBSJVyCmtNuPLpCQyggdpg&oe=64E18154"},"emitted_at":1692180825964} -{"stream":"activities","data":{"actor_id":"122043039268043192","actor_name":"Payments RTU Processor","application_id":"0","date_time_in_timezone":"03/13/2023 at 6:30 AM","event_time":"2023-03-13T13:30:47+0000","event_type":"ad_account_billing_charge","extra_data":"{\"currency\":\"USD\",\"new_value\":1188,\"transaction_id\":\"5885578541558696-11785530\",\"action\":67,\"type\":\"payment_amount\"}","object_id":"212551616838260","object_name":"Airbyte","object_type":"ACCOUNT","translated_event_type":"Account billed"},"emitted_at":1696931251153} +{"stream": "ad_creatives", "data": {"id": "23853630774830398", "body": "Until a connector meets our GA reliability standards, you don't pay for it.", "image_url": "https://scontent.fiev6-1.fna.fbcdn.net/v/t45.1600-4/333773383_23853620180320398_4214441850420455541_n.png?_nc_cat=109&ccb=1-7&_nc_sid=c0a1f7&_nc_ohc=qbTWMi-gWi8AX8hFZLQ&_nc_ht=scontent.fiev6-1.fna&edm=ALjApogEAAAA&oh=00_AfC9KndALRjbR5Z4Xz_ZytJTb9rsS_S4_SDvmiegih69vQ&oe=65C8B50F", "account_id": "212551616838260", "actor_id": "112704783733939", "asset_feed_spec": {"bodies": [{"text": "Until a connector meets our GA reliability standards, you don't pay for it."}, {"text": "Reliability is the cornerstone of having an ELT tool you trust."}, {"text": "Don't compromise between cost and connector reliability."}, {"text": "Limitless data movement with free Alpha and Beta connectors"}], "descriptions": [{"text": "Until a connector meets our GA reliability standards, you don't pay for it. "}], "titles": [{"text": "Introducing: our free connector program"}], "optimization_type": "DEGREES_OF_FREEDOM"}, "call_to_action_type": "SIGN_UP", "effective_instagram_story_id": "5605802859523550", "effective_object_story_id": "112704783733939_660115876119531", "title": "Introducing: our free connector program", "name": "Introducing: our free connector program 2023-03-17-ccf7ed52a98e5e699299861a8a323194", "instagram_actor_id": "2185696824778148", "instagram_permalink_url": "https://www.instagram.com/p/Cp5PgWrjU8V/", "object_story_spec": {"page_id": "112704783733939", "instagram_actor_id": "2185696824778148", "link_data": {"link": "https://airbyte.com/free-connector-program?utm_medium=paid_social&utm_source=facebook&utm_campaign=q1_freeconnectorprogram_t", "image_hash": "970937d2f16de20c0a99e598aa876ac0", "call_to_action": {"type": "SIGN_UP"}}}, "object_type": "SHARE", "status": "ACTIVE", "thumbnail_url": "https://external.fiev6-1.fna.fbcdn.net/emg1/v/t13/8568826884261823966?url=https%3A%2F%2Fwww.facebook.com%2Fads%2Fimage%2F%3Fd%3DAQL3nBsTZ0CoQ_uD_vAVwqZKjwi7X3zsqa8EbE4S1aY7w8cjJ7x6BihYqZkQTgC3BzwY5Y_dxv11UvkOL0cMER5tPch9x6_Q2p3xtHYED2DHLT6v9o9CnYB8S5FMSQ91vMBQCbLFVHh_bSr0OT_4bW4V&fb_obo=1&utld=facebook.com&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ccb=13-1&oh=06_AbE-j6xf-dGVCh9dJcOJdFM5v4Sydw74rDQJWynPZayneA&oe=65C511DE&_nc_sid=58080a", "image_hash": "970937d2f16de20c0a99e598aa876ac0"}, "emitted_at": 1707288372517} +{"stream":"activities","data":{"account_id":"212551616838260","actor_id":"122043039268043192","actor_name":"Payments RTU Processor","application_id":"0","date_time_in_timezone":"03/13/2023 at 6:30 AM","event_time":"2023-03-13T13:30:47+0000","event_type":"ad_account_billing_charge","extra_data":"{\"currency\":\"USD\",\"new_value\":1188,\"transaction_id\":\"5885578541558696-11785530\",\"action\":67,\"type\":\"payment_amount\"}","object_id":"212551616838260","object_name":"Airbyte","object_type":"ACCOUNT","translated_event_type":"Account billed"},"emitted_at":1696931251153} {"stream":"custom_conversions","data":{"id":"694166388077667","account_id":"212551616838260","creation_time":"2020-04-22T01:36:00+0000","custom_event_type":"CONTACT","data_sources":[{"id":"2667253716886462","source_type":"PIXEL","name":"Dataline's Pixel"}],"default_conversion_value":0,"event_source_type":"pixel","is_archived":true,"is_unavailable":false,"name":"SubscribedButtonClick","retention_days":0,"rule":"{\"and\":[{\"event\":{\"eq\":\"PageView\"}},{\"or\":[{\"URL\":{\"i_contains\":\"SubscribedButtonClick\"}}]}]}"},"emitted_at":1692180839174} {"stream":"images","data":{"id":"212551616838260:c1e94a8768a405f0f212d71fe8336647","account_id":"212551616838260","name":"Audience_1_Ad_3_1200x1200_blue_CTA_arrow.png_105","creatives":["23853630775340398","23853630871360398","23853666124200398"],"original_height":1200,"original_width":1200,"permalink_url":"https://www.facebook.com/ads/image/?d=AQIDNjjLb7VzVJ26jXb_HpudCEUJqbV_lLF2JVsdruDcBxnXQEKfzzd21VVJnkm0B-JLosUXNNg1BH78y7FxnK3AH-0D_lnk7kn39_bIcOMK7Z9HYyFInfsVY__adup3A5zGTIcHC9Y98Je5qK-yD8F6","status":"ACTIVE","url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfDCqQ6viqrgLcfbO3O5-n030Usq7Zyt2c1TmsatqnYf7Q&oe=64E2779A","created_time":"2023-03-16T13:13:17-0700","hash":"c1e94a8768a405f0f212d71fe8336647","url_128":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?stp=dst-png_s128x128&_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfAY50CMpox2s4w_f18IVx7sZuXlg4quF6YNIJJ8D4PZew&oe=64E2779A","is_associated_creatives_in_adgroups":true,"updated_time":"2023-03-17T08:09:56-0700","height":1200,"width":1200},"emitted_at":1692180839582} {"stream":"ads_insights","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170.0,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_link_clicks":3,"inline_post_engagement":3,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"quality_ranking":"UNKNOWN","reach":1255,"social_spend":0.0,"spend":1.19,"unique_actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"unique_link_clicks_ctr":0.239044,"unique_outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"website_ctr":[{"action_type":"link_click","value":0.227445}],"wish_bid":0.0},"emitted_at":1682686057366} @@ -26,4 +26,4 @@ {"stream":"ads_insights_dma","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0,"cpm":0.0,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recallers":1.0,"frequency":1.0,"impressions":1,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":1,"spend":0.0,"unique_clicks":0,"updated_time":"2021-08-27","dma":"West Palm Beach-Ft. Pierce"},"emitted_at":1696936556045} {"stream":"ads_insights_platform_and_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0,"cpm":0.0,"cpp":0.0,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":12.5,"estimated_ad_recallers":1.0,"frequency":1.0,"impressions":8,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":8,"spend":0.0,"unique_clicks":0,"updated_time":"2021-08-27","publisher_platform":"instagram","platform_position":"feed","impression_device":"android_smartphone"},"emitted_at":1696936579028} {"stream":"ads_insights_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.02,"cpm":1.111111,"cpp":1.111111,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":5.555556,"estimated_ad_recallers":1.0,"frequency":1.0,"impressions":18,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":18,"spend":0.02,"unique_clicks":0,"updated_time":"2021-08-27","region":"New York"},"emitted_at":1696936621899} -{"stream":"customcustom_insight_stream","data":{"account_id":"212551616838260","cpc":0.27,"ad_id":"23846765228310398","clicks":1,"account_name":"Airbyte","date_start":"2021-02-15","date_stop":"2021-02-15","gender":"female"},"emitted_at":1695385890508} \ No newline at end of file +{"stream":"customcustom_insight_stream","data":{"account_id":"212551616838260","cpc":0.27,"ad_id":"23846765228310398","clicks":1,"account_name":"Airbyte","date_start":"2021-02-15","date_stop":"2021-02-15","gender":"female"},"emitted_at":1695385890508} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json index 1cc425ce9f5d..55d23aacd448 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json @@ -4,7 +4,7 @@ "stream": { "stream_state": { "event_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "activities" @@ -16,7 +16,7 @@ "stream": { "stream_state": { "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": ["ARCHIVED"] }, "stream_descriptor": { "name": "campaigns" @@ -28,7 +28,7 @@ "stream": { "stream_state": { "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "images" @@ -40,7 +40,7 @@ "stream": { "stream_state": { "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "videos" @@ -52,7 +52,7 @@ "stream": { "stream_state": { "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ad_creatives" @@ -64,7 +64,7 @@ "stream": { "stream_state": { "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": ["ARCHIVED"] }, "stream_descriptor": { "name": "ad_sets" @@ -75,7 +75,11 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_time": "2121-07-25T13:34:26Z", + "212551616838260": { + "updated_time": "2121-07-25T13:34:26Z", + "filter_statuses": ["ARCHIVED"], + "include_deleted": true + }, "include_deleted": true }, "stream_descriptor": { @@ -88,7 +92,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights" @@ -100,7 +104,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_age_and_gender" @@ -112,7 +116,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_country" @@ -124,7 +128,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_dma" @@ -136,7 +140,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_platform_and_device" @@ -148,7 +152,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_region" @@ -160,7 +164,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_type" @@ -172,20 +176,19 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "custommy_custom_insights" } } }, - { "type": "STREAM", "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_carousel_card" @@ -197,7 +200,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_conversion_device" @@ -209,7 +212,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_product_id" @@ -221,7 +224,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_reaction" @@ -233,7 +236,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_video_sound" @@ -245,7 +248,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_action_video_type" @@ -257,7 +260,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_delivery_device" @@ -269,7 +272,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_delivery_platform" @@ -281,7 +284,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_delivery_platform_and_device_platform" @@ -293,7 +296,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_demographics_age" @@ -305,7 +308,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_demographics_country" @@ -317,7 +320,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_demographics_dma_region" @@ -329,7 +332,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "ads_insights_demographics_gender" @@ -341,7 +344,7 @@ "stream": { "stream_state": { "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true + "filter_statuses": [] }, "stream_descriptor": { "name": "customcustom_insight_stream" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/invalid_config.json index 44d42108c4db..4774cfe38c36 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/invalid_config.json @@ -1,6 +1,5 @@ { "start_date": "2023-04-01T00:00:00Z", "account_id": "account", - "access_token": "wrong_token", - "include_deleted": true + "access_token": "wrong_token" } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 560e10ca2686..4649cf17fdfe 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -5,14 +5,19 @@ "title": "Source Facebook Marketing", "type": "object", "properties": { - "account_id": { - "title": "Ad Account ID", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your Meta Ads Manager. See the docs for more information.", + "account_ids": { + "title": "Ad Account ID(s)", + "description": "The Facebook Ad account ID(s) to pull data from. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your Meta Ads Manager. See the docs for more information.", "order": 0, - "pattern": "^[0-9]+$", - "pattern_descriptor": "1234567890", + "pattern_descriptor": "The Ad Account ID must be a number.", "examples": ["111111111111111"], - "type": "string" + "type": "array", + "minItems": 1, + "items": { + "pattern": "^[0-9]+$", + "type": "string" + }, + "uniqueItems": true }, "access_token": { "title": "Access Token", @@ -39,24 +44,81 @@ "type": "string", "format": "date-time" }, - "include_deleted": { - "title": "Include Deleted Campaigns, Ads, and AdSets", - "description": "Set to active if you want to include data from deleted Campaigns, Ads, and AdSets.", - "default": false, + "campaign_statuses": { + "title": "Campaign Statuses", + "description": "Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + "default": [], "order": 4, - "type": "boolean" + "type": "array", + "items": { + "title": "ValidCampaignStatuses", + "description": "An enumeration.", + "enum": [ + "ACTIVE", + "ARCHIVED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ] + } + }, + "adset_statuses": { + "title": "AdSet Statuses", + "description": "Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + "default": [], + "order": 5, + "type": "array", + "items": { + "title": "ValidAdSetStatuses", + "description": "An enumeration.", + "enum": [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ] + } + }, + "ad_statuses": { + "title": "Ad Statuses", + "description": "Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + "default": [], + "order": 6, + "type": "array", + "items": { + "title": "ValidAdStatuses", + "description": "An enumeration.", + "enum": [ + "ACTIVE", + "ADSET_PAUSED", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "DISAPPROVED", + "IN_PROCESS", + "PAUSED", + "PENDING_BILLING_INFO", + "PENDING_REVIEW", + "PREAPPROVED", + "WITH_ISSUES" + ] + } }, "fetch_thumbnail_images": { "title": "Fetch Thumbnail Images from Ad Creative", "description": "Set to active if you want to fetch the thumbnail_url and store the result in thumbnail_data_url for each Ad Creative.", "default": false, - "order": 5, + "order": 7, "type": "boolean" }, "custom_insights": { "title": "Custom Insights", "description": "A list which contains ad statistics entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns. Click on \"add\" to fill this field.", - "order": 6, + "order": 8, "type": "array", "items": { "title": "InsightConfig", @@ -324,6 +386,15 @@ "mininum": 1, "exclusiveMinimum": 0, "type": "integer" + }, + "insights_job_timeout": { + "title": "Custom Insights Job Timeout", + "description": "The insights job timeout", + "default": 60, + "maximum": 60, + "mininum": 10, + "exclusiveMinimum": 0, + "type": "integer" } }, "required": ["name"] @@ -333,7 +404,7 @@ "title": "Page Size of Requests", "description": "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", "default": 100, - "order": 7, + "order": 10, "exclusiveMinimum": 0, "type": "integer" }, @@ -341,12 +412,22 @@ "title": "Insights Lookback Window", "description": "The attribution window. Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so you can retrieve refreshed insights from the past by setting this parameter. If you set a custom lookback window value in Facebook account, please provide the same value here.", "default": 28, - "order": 8, + "order": 11, "maximum": 28, "mininum": 1, "exclusiveMinimum": 0, "type": "integer" }, + "insights_job_timeout": { + "title": "Insights Job Timeout", + "description": "Insights Job Timeout establishes the maximum amount of time (in minutes) of waiting for the report job to complete. When timeout is reached the job is considered failed and we are trying to request smaller amount of data by breaking the job to few smaller ones. If you definitely know that 60 minutes is not enough for your report to be processed then you can decrease the timeout value, so we start breaking job to smaller parts faster.", + "default": 60, + "order": 12, + "maximum": 60, + "mininum": 10, + "exclusiveMinimum": 0, + "type": "integer" + }, "action_breakdowns_allow_empty": { "title": "Action Breakdowns Allow Empty", "description": "Allows action_breakdowns to be an empty list", @@ -369,7 +450,7 @@ "type": "string" } }, - "required": ["account_id", "access_token"] + "required": ["account_ids", "access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/test_streams.py index 38c08f4e4139..30d7784bb579 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/test_streams.py @@ -31,54 +31,129 @@ def configured_catalog_fixture(config) -> ConfiguredAirbyteCatalog: streams = [] # Prefer incremental if available for stream in catalog.streams: - sync_mode = SyncMode.incremental if SyncMode.incremental in stream.supported_sync_modes else SyncMode.full_refresh - streams.append(ConfiguredAirbyteStream(stream=stream, sync_mode=sync_mode, destination_sync_mode=DestinationSyncMode.append)) + sync_mode = ( + SyncMode.incremental + if SyncMode.incremental in stream.supported_sync_modes + else SyncMode.full_refresh + ) + streams.append( + ConfiguredAirbyteStream( + stream=stream, + sync_mode=sync_mode, + destination_sync_mode=DestinationSyncMode.append, + ) + ) return ConfiguredAirbyteCatalog(streams=streams) class TestFacebookMarketingSource: @pytest.mark.parametrize( - "stream_name, deleted_id", [("ads", "23846756820320398"), ("campaigns", "23846541919710398"), ("ad_sets", "23846541706990398")] + "stream_name, deleted_id", + [ + ("ads", "23846756820320398"), + ("campaigns", "23846541919710398"), + ("ad_sets", "23846541706990398"), + ], ) - def test_streams_with_include_deleted(self, stream_name, deleted_id, config_with_include_deleted, configured_catalog): + def test_streams_with_include_deleted( + self, stream_name, deleted_id, config_with_include_deleted, configured_catalog + ): catalog = self._slice_catalog(configured_catalog, {stream_name}) records, states = self._read_records(config_with_include_deleted, catalog) deleted_records = list(filter(self._deleted_record, records)) is_specific_deleted_pulled = deleted_id in list(map(self._object_id, records)) + account_id = config_with_include_deleted["account_id"] assert states, "incremental read should produce states" for name, state in states[-1].state.data.items(): - assert "include_deleted" in state, f"State for {name} should include `include_deleted` flag" + assert ( + "filter_statuses" in state[account_id] + ), f"State for {name} should include `filter_statuses` flag" - assert deleted_records, f"{stream_name} stream should have deleted records returned" - assert is_specific_deleted_pulled, f"{stream_name} stream should have a deleted record with id={deleted_id}" + assert ( + deleted_records + ), f"{stream_name} stream should have deleted records returned" + assert ( + is_specific_deleted_pulled + ), f"{stream_name} stream should have a deleted record with id={deleted_id}" @pytest.mark.parametrize( - "stream_name, deleted_num, include_deleted_in_state", + "stream_name, deleted_num, filter_statuses", [ ("ads", 2, False), ("campaigns", 3, False), ("ad_sets", 1, False), - ("ads", 0, True), - ("campaigns", 0, True), - ("ad_sets", 0, True), + ( + "ads", + 0, + [ + "ACTIVE", + "ADSET_PAUSED", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "DISAPPROVED", + "IN_PROCESS", + "PAUSED", + "PENDING_BILLING_INFO", + "PENDING_REVIEW", + "PREAPPROVED", + "WITH_ISSUES", + ], + ), + ( + "campaigns", + 0, + [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", + ], + ), + ( + "ad_sets", + 0, + [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", + ], + ), ], ) def test_streams_with_include_deleted_and_state( - self, stream_name, deleted_num, include_deleted_in_state, config_with_include_deleted, configured_catalog, state + self, + stream_name, + deleted_num, + filter_statuses, + config_with_include_deleted, + configured_catalog, + state, ): - """Should ignore state because of include_deleted enabled""" - if include_deleted_in_state: + """Should ignore state because of filter_statuses changed""" + if filter_statuses: state = copy.deepcopy(state) for value in state.values(): - value["include_deleted"] = True + value["filter_statuses"] = filter_statuses catalog = self._slice_catalog(configured_catalog, {stream_name}) - records, states = self._read_records(config_with_include_deleted, catalog, state=state) + records, states = self._read_records( + config_with_include_deleted, catalog, state=state + ) deleted_records = list(filter(self._deleted_record, records)) - assert len(deleted_records) == deleted_num, f"{stream_name} should have {deleted_num} deleted records returned" + assert ( + len(deleted_records) == deleted_num + ), f"{stream_name} should have {deleted_num} deleted records returned" @staticmethod def _deleted_record(record: AirbyteMessage) -> bool: @@ -89,7 +164,9 @@ def _object_id(record: AirbyteMessage) -> str: return str(record.record.data["id"]) @staticmethod - def _slice_catalog(catalog: ConfiguredAirbyteCatalog, streams: Set[str]) -> ConfiguredAirbyteCatalog: + def _slice_catalog( + catalog: ConfiguredAirbyteCatalog, streams: Set[str] + ) -> ConfiguredAirbyteCatalog: sliced_catalog = ConfiguredAirbyteCatalog(streams=[]) for stream in catalog.streams: if stream.stream.name in streams: @@ -97,10 +174,14 @@ def _slice_catalog(catalog: ConfiguredAirbyteCatalog, streams: Set[str]) -> Conf return sliced_catalog @staticmethod - def _read_records(conf, catalog, state=None) -> Tuple[List[AirbyteMessage], List[AirbyteMessage]]: + def _read_records( + conf, catalog, state=None + ) -> Tuple[List[AirbyteMessage], List[AirbyteMessage]]: records = [] states = [] - for message in SourceFacebookMarketing().read(logging.getLogger("airbyte"), conf, catalog, state=state): + for message in SourceFacebookMarketing().read( + logging.getLogger("airbyte"), conf, catalog, state=state + ): if message.type == Type.RECORD: records.append(message) elif message.type == Type.STATE: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/main.py b/airbyte-integrations/connectors/source-facebook-marketing/main.py index 64be48a5343e..fc25c7149e93 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/main.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_facebook_marketing import SourceFacebookMarketing +from source_facebook_marketing.run import run if __name__ == "__main__": - source = SourceFacebookMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index c456f711fb92..4f3f93f23537 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 1.2.1 + dockerImageTag: 1.4.2 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing icon: facebook.svg license: ELv2 name: Facebook Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-facebook-marketing registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock b/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock new file mode 100644 index 000000000000..3cfd31f7261a --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock @@ -0,0 +1,1510 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "airbyte-cdk" +version = "0.62.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cached-property" +version = "1.5.2" +description = "A decorator for caching properties in classes." +optional = false +python-versions = "*" +files = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "curlify" +version = "2.2.1" +description = "Library to convert python requests object to curl command." +optional = false +python-versions = "*" +files = [ + {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, +] + +[package.dependencies] +requests = "*" + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "facebook-business" +version = "17.0.0" +description = "Facebook Business SDK" +optional = false +python-versions = "*" +files = [ + {file = "facebook_business-17.0.0-py3-none-any.whl", hash = "sha256:f4b87a940a068d94ace6dc2dde7e0d43602264da18375ebfb0a8059a48a47012"}, + {file = "facebook_business-17.0.0.tar.gz", hash = "sha256:6a1c11185384325b49d640a7abb60e610b8f8561a8add1206d8e7e5f24626cf2"}, +] + +[package.dependencies] +aiohttp = {version = "*", markers = "python_version >= \"3.5.3\""} +curlify = ">=2.1.0" +pycountry = ">=19.8.18" +requests = ">=2.3.0" +six = ">=1.7.3" + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pycountry" +version = "23.12.11" +description = "ISO country, subdivision, language, currency and script definitions and their translations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycountry-23.12.11-py3-none-any.whl", hash = "sha256:2ff91cff4f40ff61086e773d61e72005fe95de4a57bfc765509db05695dc50ab"}, + {file = "pycountry-23.12.11.tar.gz", hash = "sha256:00569d82eaefbc6a490a311bfa84a9c571cff9ddbf8b0a4f4e7b4f868b4ad925"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "5753d144dc008fabd12b18d9e28d148ee96976d7b83cdcf0a82b3ea22f8f315f" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml new file mode 100644 index 000000000000..04043b38a353 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.4.2" +name = "source-facebook-marketing" +description = "Source implementation for Facebook Marketing." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/facebook-marketing" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_facebook_marketing" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.1" +facebook-business = "==17.0.0" +cached-property = "==1.5.2" +pendulum = "==2.1.2" + +[tool.poetry.scripts] +source-facebook-marketing = "source_facebook_marketing.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.6" +freezegun = "^1.4.0" +pytest = "^6.1" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py deleted file mode 100644 index 144e8b73abc7..000000000000 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.36", - "cached_property==1.5.2", - "facebook_business==17.0.0", - "pendulum>=2,<3", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", "freezegun"] - -setup( - name="source_facebook_marketing", - description="Source implementation for Facebook Marketing.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index e3a6c610e117..31bf4644013d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -9,7 +9,6 @@ import backoff import pendulum -from cached_property import cached_property from facebook_business import FacebookAdsApi from facebook_business.adobjects.adaccount import AdAccount from facebook_business.api import FacebookResponse @@ -35,7 +34,7 @@ class MyFacebookAdsApi(FacebookAdsApi): # see `_should_restore_page_size` method docstring for more info. # attribute to handle the reduced request limit request_record_limit_is_reduced: bool = False - # attribute to save the status of last successfull call + # attribute to save the status of the last successful call last_api_call_is_successful: bool = False @dataclass @@ -109,7 +108,10 @@ def _get_max_usage_pause_interval_from_batch(self, records): if "headers" not in record: continue headers = {header["name"].lower(): header["value"] for header in record["headers"]} - usage_from_response, pause_interval_from_response = self._parse_call_rate_header(headers) + ( + usage_from_response, + pause_interval_from_response, + ) = self._parse_call_rate_header(headers) usage = max(usage, usage_from_response) pause_interval = max(pause_interval_from_response, pause_interval) return usage, pause_interval @@ -144,7 +146,7 @@ def _update_insights_throttle_limit(self, response: FacebookResponse): def _should_restore_default_page_size(self, params): """ - Track the state of the `request_record_limit_is_reduced` and `last_api_call_is_successfull`, + Track the state of the `request_record_limit_is_reduced` and `last_api_call_is_successful`, based on the logic from `@backoff_policy` (common.py > `reduce_request_record_limit` and `revert_request_record_limit`) """ params = True if params else False @@ -173,8 +175,8 @@ def call( class API: """Simple wrapper around Facebook API""" - def __init__(self, account_id: str, access_token: str, page_size: int = 100): - self._account_id = account_id + def __init__(self, access_token: str, page_size: int = 100): + self._accounts = {} # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) # adding the default page size from config to the api base class @@ -183,10 +185,12 @@ def __init__(self, account_id: str, access_token: str, page_size: int = 100): # set the default API client to Facebook lib. FacebookAdsApi.set_default_api(self.api) - @cached_property - def account(self) -> AdAccount: - """Find current account""" - return self._find_account(self._account_id) + def get_account(self, account_id: str) -> AdAccount: + """Get AdAccount object by id""" + if account_id in self._accounts: + return self._accounts[account_id] + self._accounts[account_id] = self._find_account(account_id) + return self._accounts[account_id] @staticmethod def _find_account(account_id: str) -> AdAccount: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py new file mode 100644 index 000000000000..f63b98ebd5b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses + +logger = logging.getLogger("airbyte_logger") + + +class MigrateAccountIdToArray: + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `account_id`. + + Starting from `1.3.0`, the `account_id` property is replaced with `account_ids` property, which is a list of strings. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "account_id" + migrate_to_key: str = "account_ids" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + return False if config.get(cls.migrate_to_key) else True + + @classmethod + def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # transform the config + config[cls.migrate_to_key] = [config[cls.migrate_from_key]] + # return transformed config + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) + + +class MigrateIncludeDeletedToStatusFilters(MigrateAccountIdToArray): + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `include_deleted`. + + Starting from `1.4.0`, the `include_deleted` property is replaced with `ad_statuses`, + `ad_statuses` and `campaign_statuses` which represent status filters. + """ + + migrate_from_key: str = "include_deleted" + migrate_to_key: str = "ad_statuses" + stream_filter_to_statuses: Mapping[str, List[str]] = { + "ad_statuses": [status.value for status in ValidAdStatuses], + "adset_statuses": [status.value for status in ValidAdSetStatuses], + "campaign_statuses": [status.value for status in ValidCampaignStatuses], + } + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new property for filters. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + config_is_updated = config.get(cls.migrate_to_key) + no_include_deleted = not config.get(cls.migrate_from_key) + return False if config_is_updated or no_include_deleted else True + + @classmethod + def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # transform the config + for stream_filter, statuses in cls.stream_filter_to_statuses.items(): + config[stream_filter] = statuses + # return transformed config + return config diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py new file mode 100644 index 000000000000..b070561488f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters +from .source import SourceFacebookMarketing + + +def run(): + source = SourceFacebookMarketing() + MigrateAccountIdToArray.migrate(sys.argv[1:], source) + MigrateIncludeDeletedToStatusFilters.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 4b5c729e0686..69a31b5f8b55 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "actor_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index cd0b5ec85f1c..52b81979e3d4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -60,6 +60,9 @@ }, "multi_share_end_card": { "type": ["null", "boolean"] + }, + "is_click_to_message": { + "type": ["null", "boolean"] } } }, @@ -884,6 +887,9 @@ }, "show_multiple_images": { "type": ["null", "boolean"] + }, + "use_flexible_image_aspect_ratio": { + "type": ["null", "boolean"] } }, "type": ["null", "object"] @@ -1099,6 +1105,17 @@ }, "type": ["null", "object"] }, + "place_data": { + "properties": { + "location_source_id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, "caption": { "type": ["null", "string"] }, @@ -1123,6 +1140,9 @@ }, "type": ["null", "array"] }, + "format_option": { + "type": ["null", "string"] + }, "multi_share_optimized": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json index b17fd1c819fc..99279e65f2f4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json @@ -94,6 +94,15 @@ }, "type": ["null", "object"], "properties": { + "brand_safety_content_filter_levels": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "targeting_relaxation_types": { + "type": ["null", "object"] + }, "messenger_positions": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 9aa51674dd10..3a146978ada6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "id": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 785a51afb23e..0e4744be48f2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -14,7 +14,6 @@ DestinationSyncMode, FailureType, OAuthConfigSpecification, - SyncMode, ) from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -76,6 +75,9 @@ def _validate_and_transform(self, config: Mapping[str, Any]): if config.end_date: config.end_date = pendulum.instance(config.end_date) + + config.account_ids = list(config.account_ids) + return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -93,11 +95,20 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) + api = API(access_token=config.access_token, page_size=config.page_size) + + for account_id in config.account_ids: + # Get Ad Account to check creds + logger.info(f"Attempting to retrieve information for account with ID: {account_id}") + ad_account = api.get_account(account_id=account_id) + logger.info(f"Successfully retrieved account information for account: {ad_account}") + + # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters + for stream in self.get_custom_insights_streams(api, config): + stream.check_breakdowns(account_id=account_id) - # Get Ad Account to check creds - ad_account = api.account - logger.info(f"Select account {ad_account}") + except facebook_business.exceptions.FacebookRequestError as e: + return False, e._api_error_message except AirbyteTracedException as e: return False, f"{e.message}. Full error: {e.internal_message}" @@ -105,12 +116,6 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> except Exception as e: return False, f"Unexpected error: {repr(e)}" - # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters - for stream in self.get_custom_insights_streams(api, config): - try: - stream.check_breakdowns() - except facebook_business.exceptions.FacebookRequestError as e: - return False, e._api_error_message return True, None def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: @@ -124,32 +129,40 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) + api = API(access_token=config.access_token, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) insights_args = dict( - api=api, start_date=report_start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window + api=api, + account_ids=config.account_ids, + start_date=report_start_date, + end_date=config.end_date, + insights_lookback_window=config.insights_lookback_window, + insights_job_timeout=config.insights_job_timeout, ) streams = [ - AdAccount(api=api), + AdAccount(api=api, account_ids=config.account_ids), AdSets( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, + filter_statuses=config.adset_statuses, page_size=config.page_size, ), Ads( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, + filter_statuses=config.ad_statuses, page_size=config.page_size, ), AdCreatives( api=api, + account_ids=config.account_ids, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size, ), @@ -175,40 +188,41 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: AdsInsightsDemographicsGender(page_size=config.page_size, **insights_args), Campaigns( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, + filter_statuses=config.campaign_statuses, page_size=config.page_size, ), CustomConversions( api=api, - include_deleted=config.include_deleted, + account_ids=config.account_ids, page_size=config.page_size, ), CustomAudiences( api=api, - include_deleted=config.include_deleted, + account_ids=config.account_ids, page_size=config.page_size, ), Images( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, page_size=config.page_size, ), Videos( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, page_size=config.page_size, ), Activities( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, - include_deleted=config.include_deleted, page_size=config.page_size, ), ] @@ -240,14 +254,23 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: }, complete_oauth_server_input_specification={ "type": "object", - "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + }, }, complete_oauth_server_output_specification={ "type": "object", "additionalProperties": True, "properties": { - "client_id": {"type": "string", "path_in_connector_config": ["client_id"]}, - "client_secret": {"type": "string", "path_in_connector_config": ["client_secret"]}, + "client_id": { + "type": "string", + "path_in_connector_config": ["client_id"], + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"], + }, }, }, ), @@ -271,6 +294,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List ) stream = AdsInsights( api=api, + account_ids=config.account_ids, name=f"Custom{insight.name}", fields=list(insight_fields), breakdowns=list(set(insight.breakdowns)), @@ -281,6 +305,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List start_date=insight.start_date or config.start_date or pendulum.now().add(years=-2), end_date=insight.end_date or config.end_date, insights_lookback_window=insight.insights_lookback_window or config.insights_lookback_window, + insights_job_timeout=insight.insights_job_timeout or config.insights_job_timeout, level=insight.level, ) streams.append(stream) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 0d12ca0a5af6..4b1b7a8a51e4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -5,11 +5,14 @@ import logging from datetime import datetime, timezone from enum import Enum -from typing import List, Optional +from typing import List, Optional, Set from airbyte_cdk.sources.config import BaseConfig +from facebook_business.adobjects.ad import Ad +from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.adsinsights import AdsInsights -from pydantic import BaseModel, Field, PositiveInt +from facebook_business.adobjects.campaign import Campaign +from pydantic import BaseModel, Field, PositiveInt, constr logger = logging.getLogger("airbyte") @@ -17,6 +20,9 @@ ValidFields = Enum("ValidEnums", AdsInsights.Field.__dict__) ValidBreakdowns = Enum("ValidBreakdowns", AdsInsights.Breakdowns.__dict__) ValidActionBreakdowns = Enum("ValidActionBreakdowns", AdsInsights.ActionBreakdowns.__dict__) +ValidCampaignStatuses = Enum("ValidCampaignStatuses", Campaign.EffectiveStatus.__dict__) +ValidAdSetStatuses = Enum("ValidAdSetStatuses", AdSet.EffectiveStatus.__dict__) +ValidAdStatuses = Enum("ValidAdStatuses", Ad.EffectiveStatus.__dict__) DATE_TIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" EMPTY_PATTERN = "^$" @@ -32,7 +38,12 @@ class Config: description="The name value of insight", ) - level: str = Field(title="Level", description="Chosen level for API", default="ad", enum=["ad", "adset", "campaign", "account"]) + level: str = Field( + title="Level", + description="Chosen level for API", + default="ad", + enum=["ad", "adset", "campaign", "account"], + ) fields: Optional[List[ValidFields]] = Field( title="Fields", @@ -97,6 +108,13 @@ class Config: mininum=1, default=28, ) + insights_job_timeout: Optional[PositiveInt] = Field( + title="Custom Insights Job Timeout", + description="The insights job timeout", + maximum=60, + mininum=10, + default=60, + ) class ConnectorConfig(BaseConfig): @@ -104,19 +122,20 @@ class ConnectorConfig(BaseConfig): class Config: title = "Source Facebook Marketing" + use_enum_values = True - account_id: str = Field( - title="Ad Account ID", + account_ids: Set[constr(regex="^[0-9]+$")] = Field( + title="Ad Account ID(s)", order=0, description=( - "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. " + "The Facebook Ad account ID(s) to pull data from. " "The Ad account ID number is in the account dropdown menu or in your browser's address " 'bar of your Meta Ads Manager. ' 'See the docs for more information.' ), - pattern="^[0-9]+$", - pattern_descriptor="1234567890", + pattern_descriptor="The Ad Account ID must be a number.", examples=["111111111111111"], + min_items=1, ) access_token: str = Field( @@ -155,23 +174,37 @@ class Config: default_factory=lambda: datetime.now(tz=timezone.utc), ) - include_deleted: bool = Field( - title="Include Deleted Campaigns, Ads, and AdSets", + campaign_statuses: Optional[List[ValidCampaignStatuses]] = Field( + title="Campaign Statuses", order=4, - default=False, - description="Set to active if you want to include data from deleted Campaigns, Ads, and AdSets.", + description="Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + default=[], + ) + + adset_statuses: Optional[List[ValidAdSetStatuses]] = Field( + title="AdSet Statuses", + order=5, + description="Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + default=[], + ) + + ad_statuses: Optional[List[ValidAdStatuses]] = Field( + title="Ad Statuses", + order=6, + description="Select the statuses you want to be loaded in the stream. If no specific statuses are selected, the API's default behavior applies, and some statuses may be filtered out.", + default=[], ) fetch_thumbnail_images: bool = Field( title="Fetch Thumbnail Images from Ad Creative", - order=5, + order=7, default=False, description="Set to active if you want to fetch the thumbnail_url and store the result in thumbnail_data_url for each Ad Creative.", ) custom_insights: Optional[List[InsightConfig]] = Field( title="Custom Insights", - order=6, + order=8, description=( "A list which contains ad statistics entries, each entry must have a name and can contains fields, " 'breakdowns or action_breakdowns. Click on "add" to fill this field.' @@ -180,7 +213,7 @@ class Config: page_size: Optional[PositiveInt] = Field( title="Page Size of Requests", - order=7, + order=10, default=100, description=( "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. " @@ -190,7 +223,7 @@ class Config: insights_lookback_window: Optional[PositiveInt] = Field( title="Insights Lookback Window", - order=8, + order=11, description=( "The attribution window. Facebook freezes insight data 28 days after it was generated, " "which means that all data from the past 28 days may have changed since we last emitted it, " @@ -202,6 +235,20 @@ class Config: default=28, ) + insights_job_timeout: Optional[PositiveInt] = Field( + title="Insights Job Timeout", + order=12, + description=( + "Insights Job Timeout establishes the maximum amount of time (in minutes) of waiting for the report job to complete. " + "When timeout is reached the job is considered failed and we are trying to request smaller amount of data by breaking the job to few smaller ones. " + "If you definitely know that 60 minutes is not enough for your report to be processed then you can decrease the timeout value, " + "so we start breaking job to smaller parts faster." + ), + maximum=60, + mininum=10, + default=60, + ) + action_breakdowns_allow_empty: bool = Field( description="Allows action_breakdowns to be an empty list", default=True, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index fe44c18223e8..e8f1038c2bb9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -17,6 +17,7 @@ from facebook_business.adobjects.campaign import Campaign from facebook_business.adobjects.objectparser import ObjectParser from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse +from pendulum.duration import Duration from source_facebook_marketing.streams.common import retry_pattern from ..utils import validate_start_date @@ -189,10 +190,15 @@ def __str__(self) -> str: class InsightAsyncJob(AsyncJob): """AsyncJob wraps FB AdReport class and provides interface to restart/retry the async job""" - job_timeout = pendulum.duration(hours=1) page_size = 100 - def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: Mapping[str, Any], **kwargs): + def __init__( + self, + edge_object: Union[AdAccount, Campaign, AdSet, Ad], + params: Mapping[str, Any], + job_timeout: Duration, + **kwargs, + ): """Initialize :param api: FB API @@ -205,6 +211,7 @@ def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: M "since": self._interval.start.to_date_string(), "until": self._interval.end.to_date_string(), } + self._job_timeout = job_timeout self._edge_object = edge_object self._job: Optional[AdReportRun] = None @@ -251,7 +258,16 @@ def _split_by_edge_class(self, edge_class: Union[Type[Campaign], Type[AdSet], Ty ids = set(row[pk_name] for row in result) logger.info(f"Got {len(ids)} {pk_name}s for period {self._interval}: {ids}") - jobs = [InsightAsyncJob(api=self._api, edge_object=edge_class(pk), params=self._params, interval=self._interval) for pk in ids] + jobs = [ + InsightAsyncJob( + api=self._api, + edge_object=edge_class(pk), + params=self._params, + interval=self._interval, + job_timeout=self._job_timeout, + ) + for pk in ids + ] return jobs def start(self): @@ -321,7 +337,11 @@ def update_job(self, batch: Optional[FacebookAdsApiBatch] = None): return if batch is not None: - self._job.api_get(batch=batch, success=self._batch_success_handler, failure=self._batch_failure_handler) + self._job.api_get( + batch=batch, + success=self._batch_success_handler, + failure=self._batch_failure_handler, + ) else: self._job = self._job.api_get() self._check_status() @@ -335,8 +355,8 @@ def _check_status(self) -> bool: percent = self._job["async_percent_completion"] logger.info(f"{self}: is {percent} complete ({job_status})") - if self.elapsed_time > self.job_timeout: - logger.info(f"{self}: run more than maximum allowed time {self.job_timeout}.") + if self.elapsed_time > self._job_timeout: + logger.info(f"{self}: run more than maximum allowed time {self._job_timeout}.") self._finish_time = pendulum.now() self._failed = True return True diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index 8bfcc6fe74af..dc01cd228412 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -32,13 +32,14 @@ class InsightAsyncJobManager: # limit is not reliable indicator of async workload capability we still have to use this parameter. MAX_JOBS_IN_QUEUE = 100 - def __init__(self, api: "API", jobs: Iterator[AsyncJob]): + def __init__(self, api: "API", jobs: Iterator[AsyncJob], account_id: str): """Init :param api: :param jobs: """ self._api = api + self._account_id = account_id self._jobs = iter(jobs) self._running_jobs = [] @@ -103,7 +104,10 @@ def _check_jobs_status_and_restart(self) -> List[AsyncJob]: if job.attempt_number >= self.MAX_NUMBER_OF_ATTEMPTS: raise JobException(f"{job}: failed more than {self.MAX_NUMBER_OF_ATTEMPTS} times. Terminating...") elif job.attempt_number == 2: - logger.info("%s: failed second time, trying to split job into smaller jobs.", job) + logger.info( + "%s: failed second time, trying to split job into smaller jobs.", + job, + ) smaller_jobs = job.split_job() grouped_jobs = ParentAsyncJob(api=self._api.api, jobs=smaller_jobs, interval=job.interval) running_jobs.append(grouped_jobs) @@ -133,7 +137,7 @@ def _get_current_throttle_value(self) -> float: """ Get current ads insights throttle value based on app id and account id. It evaluated as minimum of those numbers cause when account id throttle - hit 100 it cool down very slowly (i.e. it still says 100 despite no jobs + hit 100 it cools down very slowly (i.e. it still says 100 despite no jobs running and it capable serve new requests). Because of this behaviour facebook throttle limit is not reliable metric to estimate async workload. """ @@ -143,8 +147,8 @@ def _get_current_throttle_value(self) -> float: def _update_api_throttle_limit(self): """ - Sends /insights GET request with no parameters so it would + Sends /insights GET request with no parameters, so it would respond with empty list of data so api use "x-fb-ads-insights-throttle" header to update current insights throttle limit. """ - self._api.account.get_insights() + self._api.get_account(account_id=self._account_id).get_insights() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 0372057c645f..af36033599c3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -11,7 +11,6 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.utils import AirbyteTracedException -from cached_property import cached_property from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob from source_facebook_marketing.streams.async_job_manager import InsightAsyncJobManager @@ -26,7 +25,6 @@ class AdsInsights(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/insights""" cursor_field = "date_start" - enable_deleted = False ALL_ACTION_ATTRIBUTION_WINDOWS = [ "1d_click", @@ -45,8 +43,7 @@ class AdsInsights(FBMarketingIncrementalStream): ] # Facebook store metrics maximum of 37 months old. Any time range that - # older that 37 months from current date would result in 400 Bad request - # HTTP response. + # older than 37 months from current date would result in 400 Bad request HTTP response. # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview INSIGHTS_RETENTION_PERIOD = pendulum.duration(months=37) @@ -63,13 +60,14 @@ def __init__( action_report_time: str = "mixed", time_increment: Optional[int] = None, insights_lookback_window: int = None, + insights_job_timeout: int = 60, level: str = "ad", **kwargs, ): super().__init__(**kwargs) self._start_date = self._start_date.date() self._end_date = self._end_date.date() - self._fields = fields + self._custom_fields = fields if action_breakdowns_allow_empty: if action_breakdowns is not None: self.action_breakdowns = action_breakdowns @@ -82,12 +80,13 @@ def __init__( self.action_report_time = action_report_time self._new_class_name = name self._insights_lookback_window = insights_lookback_window + self._insights_job_timeout = insights_job_timeout self.level = level # state - self._cursor_value: Optional[pendulum.Date] = None # latest period that was read - self._next_cursor_value = self._get_start_date() - self._completed_slices = set() + self._cursor_values: Optional[Mapping[str, pendulum.Date]] = None # latest period that was read for each account + self._next_cursor_values = self._get_start_date() + self._completed_slices = {account_id: set() for account_id in self._account_ids} @property def name(self) -> str: @@ -105,11 +104,15 @@ def insights_lookback_period(self): """ Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so we retrieve it again. - But in some cases users my have define their own lookback window, thats - why the value for `insights_lookback_window` is set throught config. + But in some cases users my have define their own lookback window, that's + why the value for `insights_lookback_window` is set through the config. """ return pendulum.duration(days=self._insights_lookback_window) + @property + def insights_job_timeout(self): + return pendulum.duration(minutes=self._insights_job_timeout) + def list_objects(self, params: Mapping[str, Any]) -> Iterable: """Because insights has very different read_records we don't need this method anymore""" @@ -122,6 +125,8 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: """Waits for current job to finish (slice) and yield its result""" job = stream_slice["insight_job"] + account_id = stream_slice["account_id"] + try: for obj in job.get_result(): data = obj.export_all_data() @@ -136,25 +141,30 @@ def read_records( except FacebookRequestError as exc: raise traced_exception(exc) - self._completed_slices.add(job.interval.start) - if job.interval.start == self._next_cursor_value: - self._advance_cursor() + self._completed_slices[account_id].add(job.interval.start) + if job.interval.start == self._next_cursor_values[account_id]: + self._advance_cursor(account_id) @property def state(self) -> MutableMapping[str, Any]: """State getter, the result can be stored by the source""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value.isoformat(), - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + new_state = {account_id: {} for account_id in self._account_ids} + + if self._cursor_values: + for account_id in self._account_ids: + if account_id in self._cursor_values and self._cursor_values[account_id]: + new_state[account_id] = {self.cursor_field: self._cursor_values[account_id].isoformat()} + + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + new_state["time_increment"] = self.time_increment + return new_state if self._completed_slices: - return { - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + for account_id in self._account_ids: + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + + new_state["time_increment"] = self.time_increment + return new_state return {} @@ -162,17 +172,31 @@ def state(self) -> MutableMapping[str, Any]: def state(self, value: Mapping[str, Any]): """State setter, will ignore saved state if time_increment is different from previous.""" # if the time increment configured for this stream is different from the one in the previous state - # then the previous state object is invalid and we should start replicating data from scratch + # then the previous state object is invalid, and we should start replicating data from scratch # to achieve this, we skip setting the state - if value.get("time_increment", 1) != self.time_increment: + transformed_state = self._transform_state_from_one_account_format(value, ["time_increment"]) + if transformed_state.get("time_increment", 1) != self.time_increment: logger.info(f"Ignoring bookmark for {self.name} because of different `time_increment` option.") return - self._cursor_value = pendulum.parse(value[self.cursor_field]).date() if value.get(self.cursor_field) else None - self._completed_slices = set(pendulum.parse(v).date() for v in value.get("slices", [])) - self._next_cursor_value = self._get_start_date() + self._cursor_values = { + account_id: pendulum.parse(transformed_state[account_id][self.cursor_field]).date() + if transformed_state.get(account_id, {}).get(self.cursor_field) + else None + for account_id in self._account_ids + } + self._completed_slices = { + account_id: set(pendulum.parse(v).date() for v in transformed_state.get(account_id, {}).get("slices", [])) + for account_id in self._account_ids + } + + self._next_cursor_values = self._get_start_date() - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ): """Update stream state from latest record :param current_stream_state: latest state returned @@ -180,38 +204,47 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return self.state - def _date_intervals(self) -> Iterator[pendulum.Date]: + def _date_intervals(self, account_id: str) -> Iterator[pendulum.Date]: """Get date period to sync""" - if self._end_date < self._next_cursor_value: + if self._end_date < self._next_cursor_values[account_id]: return - date_range = self._end_date - self._next_cursor_value + date_range = self._end_date - self._next_cursor_values[account_id] yield from date_range.range("days", self.time_increment) - def _advance_cursor(self): + def _advance_cursor(self, account_id: str): """Iterate over state, find continuing sequence of slices. Get last value, advance cursor there and remove slices from state""" - for ts_start in self._date_intervals(): - if ts_start not in self._completed_slices: - self._next_cursor_value = ts_start + for ts_start in self._date_intervals(account_id): + if ts_start not in self._completed_slices[account_id]: + self._next_cursor_values[account_id] = ts_start break - self._completed_slices.remove(ts_start) - self._cursor_value = ts_start + self._completed_slices[account_id].remove(ts_start) + if self._cursor_values: + self._cursor_values[account_id] = ts_start + else: + self._cursor_values = {account_id: ts_start} - def _generate_async_jobs(self, params: Mapping) -> Iterator[AsyncJob]: + def _generate_async_jobs(self, params: Mapping, account_id: str) -> Iterator[AsyncJob]: """Generator of async jobs :param params: :return: """ - self._next_cursor_value = self._get_start_date() - for ts_start in self._date_intervals(): - if ts_start in self._completed_slices: + self._next_cursor_values = self._get_start_date() + for ts_start in self._date_intervals(account_id): + if ts_start in self._completed_slices.get(account_id, []): continue ts_end = ts_start + pendulum.duration(days=self.time_increment - 1) interval = pendulum.Period(ts_start, ts_end) - yield InsightAsyncJob(api=self._api.api, edge_object=self._api.account, interval=interval, params=params) - - def check_breakdowns(self): + yield InsightAsyncJob( + api=self._api.api, + edge_object=self._api.get_account(account_id=account_id), + interval=interval, + params=params, + job_timeout=self.insights_job_timeout, + ) + + def check_breakdowns(self, account_id: str): """ Making call to check "action_breakdowns" and "breakdowns" combinations https://developers.facebook.com/docs/marketing-api/insights/breakdowns#combiningbreakdowns @@ -221,7 +254,7 @@ def check_breakdowns(self): "breakdowns": self.breakdowns, "fields": ["account_id"], } - self._api.account.get_insights(params=params, is_async=False) + self._api.get_account(account_id=account_id).get_insights(params=params, is_async=False) def _response_data_is_valid(self, data: Iterable[Mapping[str, Any]]) -> bool: """ @@ -230,7 +263,10 @@ def _response_data_is_valid(self, data: Iterable[Mapping[str, Any]]) -> bool: return all([breakdown in data for breakdown in self.breakdowns]) def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[Mapping[str, Any]]]: """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time. This solution for Async was chosen because: @@ -247,14 +283,19 @@ def stream_slices( if stream_state: self.state = stream_state - try: - manager = InsightAsyncJobManager(api=self._api, jobs=self._generate_async_jobs(params=self.request_params())) - for job in manager.completed_jobs(): - yield {"insight_job": job} - except FacebookRequestError as exc: - raise traced_exception(exc) + for account_id in self._account_ids: + try: + manager = InsightAsyncJobManager( + api=self._api, + jobs=self._generate_async_jobs(params=self.request_params(), account_id=account_id), + account_id=account_id, + ) + for job in manager.completed_jobs(): + yield {"insight_job": job, "account_id": account_id} + except FacebookRequestError as exc: + raise traced_exception(exc) - def _get_start_date(self) -> pendulum.Date: + def _get_start_date(self) -> Mapping[str, pendulum.Date]: """Get start date to begin sync with. It is not that trivial as it might seem. There are few rules: - don't read data older than start_date @@ -269,33 +310,42 @@ def _get_start_date(self) -> pendulum.Date: today = pendulum.today().date() oldest_date = today - self.INSIGHTS_RETENTION_PERIOD refresh_date = today - self.insights_lookback_period - if self._cursor_value: - start_date = self._cursor_value + pendulum.duration(days=self.time_increment) - if start_date > refresh_date: - logger.info( - f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + + start_dates_for_account = {} + for account_id in self._account_ids: + cursor_value = self._cursor_values.get(account_id) if self._cursor_values else None + if cursor_value: + start_date = cursor_value + pendulum.duration(days=self.time_increment) + if start_date > refresh_date: + logger.info( + f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + ) + start_date = min(start_date, refresh_date) + + if start_date < self._start_date: + logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") + start_date = max(start_date, self._start_date) + else: + start_date = self._start_date + if start_date < oldest_date: + logger.warning( + f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}." ) - start_date = min(start_date, refresh_date) + start_dates_for_account[account_id] = max(oldest_date, start_date) - if start_date < self._start_date: - logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") - start_date = max(start_date, self._start_date) - else: - start_date = self._start_date - if start_date < oldest_date: - logger.warning(f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}.") - return max(oldest_date, start_date) + return start_dates_for_account def request_params(self, **kwargs) -> MutableMapping[str, Any]: - return { + req_params = { "level": self.level, "action_breakdowns": self.action_breakdowns, "action_report_time": self.action_report_time, "breakdowns": self.breakdowns, - "fields": self.fields, + "fields": self.fields(), "time_increment": self.time_increment, "action_attribution_windows": self.action_attribution_windows, } + return req_params def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Works differently for insights, so remove it""" @@ -307,19 +357,23 @@ def get_json_schema(self) -> Mapping[str, Any]: """ loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) schema = loader.get_schema("ads_insights") - if self._fields: + if self._custom_fields: # 'date_stop' and 'account_id' are also returned by default, even if they are not requested - custom_fields = set(self._fields + [self.cursor_field, "date_stop", "account_id", "ad_id"]) + custom_fields = set(self._custom_fields + [self.cursor_field, "date_stop", "account_id", "ad_id"]) schema["properties"] = {k: v for k, v in schema["properties"].items() if k in custom_fields} if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) return schema - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" + if self._custom_fields: + return self._custom_fields + if self._fields: return self._fields + schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights") - return list(schema.get("properties", {}).keys()) + self._fields = list(schema.get("properties", {}).keys()) + return self._fields diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 01b8488ca4ba..895456feb9c5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -12,7 +12,6 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams.common import traced_exception @@ -22,7 +21,6 @@ if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API - logger = logging.getLogger("airbyte") @@ -32,9 +30,9 @@ class FBMarketingStream(Stream, ABC): primary_key = "id" transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - # this flag will override `include_deleted` option for streams that does not support it - enable_deleted = True - # entity prefix for `include_deleted` filter, it usually matches singular version of stream name + valid_statuses = [] + status_field = "" + # entity prefix for statuses filter, it usually matches singular version of stream name entity_prefix = None # In case of Error 'Too much data was requested in batch' some fields should be removed from request fields_exceptions = [] @@ -43,16 +41,27 @@ class FBMarketingStream(Stream, ABC): def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None - def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, **kwargs): + def __init__( + self, + api: "API", + account_ids: List[str], + filter_statuses: list = [], + page_size: int = 100, + **kwargs, + ): super().__init__(**kwargs) self._api = api + self._account_ids = account_ids self.page_size = page_size if page_size is not None else 100 - self._include_deleted = include_deleted if self.enable_deleted else False + self._filter_statuses = filter_statuses + self._fields = None - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" - return list(self.get_json_schema().get("properties", {}).keys()) + if self._fields: + return self._fields + self._saved_fields = list(self.get_json_schema().get("properties", {}).keys()) + return self._saved_fields @classmethod def fix_date_time(cls, record): @@ -78,6 +87,83 @@ def fix_date_time(cls, record): for entry in record: cls.fix_date_time(entry) + @staticmethod + def add_account_id(record, account_id: str): + if "account_id" not in record: + record["account_id"] = account_id + + def get_account_state(self, account_id: str, stream_state: Mapping[str, Any] = None) -> MutableMapping[str, Any]: + """ + Retrieve the state for a specific account. + + If multiple account IDs are present, the state for the specific account ID + is returned if it exists in the stream state. If only one account ID is + present, the entire stream state is returned. + + :param account_id: The account ID for which to retrieve the state. + :param stream_state: The current stream state, optional. + :return: The state information for the specified account as a MutableMapping. + """ + if stream_state and account_id and account_id in stream_state: + account_state = stream_state.get(account_id) + + # copy `include_deleted` from general stream state + if "include_deleted" in stream_state: + account_state["include_deleted"] = stream_state["include_deleted"] + return account_state + elif len(self._account_ids) == 1: + return stream_state + else: + return {} + + def _transform_state_from_one_account_format(self, state: Mapping[str, Any], move_fields: List[str] = None) -> Mapping[str, Any]: + """ + Transforms the state from an old format to a new format based on account IDs. + + This method transforms the old state to be a dictionary where the keys are account IDs. + If the state is in the old format (not keyed by account IDs), it will transform the state + by nesting it under the account ID. + + :param state: The original state dictionary to transform. + :param move_fields: A list of field names whose values should be moved to the top level of the new state dictionary. + :return: The transformed state dictionary. + """ + + # If the state already contains any of the account IDs, return the state as is. + for account_id in self._account_ids: + if account_id in state: + return state + + # Handle the case where there is only one account ID. + # Transform the state by nesting it under the account ID. + if state and len(self._account_ids) == 1: + account_id = self._account_ids[0] + new_state = {account_id: state} + + # Move specified fields to the top level of the new state. + if move_fields: + for move_field in move_fields: + if move_field in state: + new_state[move_field] = state.pop(move_field) + + return new_state + + # If the state is empty or there are multiple account IDs, return an empty dictionary. + return {} + + def _transform_state_from_old_deleted_format(self, state: Mapping[str, Any]): + # transform from the old format with `include_deleted` + for account_id in self._account_ids: + account_state = state.get(account_id, {}) + # check if the state for this account id is in the old format + if "filter_statuses" not in account_state and "include_deleted" in account_state: + if account_state["include_deleted"]: + account_state["filter_statuses"] = self.valid_statuses + else: + account_state["filter_statuses"] = [] + state[account_id] = account_state + return state + def read_records( self, sync_mode: SyncMode, @@ -86,15 +172,31 @@ def read_records( stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: """Main read method used by CDK""" + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state", {}) + try: - for record in self.list_objects(params=self.request_params(stream_state=stream_state)): + for record in self.list_objects( + params=self.request_params(stream_state=account_state), + account_id=account_id, + ): if isinstance(record, AbstractObject): record = record.export_all_data() # convert FB object to dict self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) yield record except FacebookRequestError as exc: raise traced_exception(exc) + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + if stream_state: + stream_state = self._transform_state_from_one_account_format(stream_state, ["include_deleted"]) + stream_state = self._transform_state_from_old_deleted_format(stream_state) + + for account_id in self._account_ids: + account_state = self.get_account_state(account_id, stream_state) + yield {"account_id": account_id, "stream_state": account_state} + @abstractmethod def list_objects(self, params: Mapping[str, Any]) -> Iterable: """List FB objects, these objects will be loaded in read_records later with their details. @@ -106,36 +208,26 @@ def list_objects(self, params: Mapping[str, Any]) -> Iterable: def request_params(self, **kwargs) -> MutableMapping[str, Any]: """Parameters that should be passed to query_records method""" params = {"limit": self.page_size} - - if self._include_deleted: - params.update(self._filter_all_statuses()) + params.update(self._filter_all_statuses()) return params def _filter_all_statuses(self) -> MutableMapping[str, Any]: - """Filter that covers all possible statuses thus including deleted/archived records""" - filt_values = [ - "active", - "archived", - "completed", - "limited", - "not_delivering", - "deleted", - "not_published", - "pending_review", - "permanently_deleted", - "recently_completed", - "recently_rejected", - "rejected", - "scheduled", - "inactive", - ] - - return { - "filtering": [ - {"field": f"{self.entity_prefix}.delivery_info", "operator": "IN", "value": filt_values}, - ], - } + """Filter records by statuses""" + + return ( + { + "filtering": [ + { + "field": f"{self.entity_prefix}.{self.status_field}", + "operator": "IN", + "value": self._filter_statuses, + }, + ], + } + if self._filter_statuses and self.status_field + else {} + ) class FBMarketingIncrementalStream(FBMarketingStream, ABC): @@ -148,19 +240,30 @@ def __init__(self, start_date: Optional[datetime], end_date: Optional[datetime], self._start_date = pendulum.instance(start_date) if start_date else None self._end_date = pendulum.instance(end_date) if end_date else None - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ): """Update stream state from latest record""" - potentially_new_records_in_the_past = self._include_deleted and not current_stream_state.get("include_deleted", False) + account_id = latest_record["account_id"] + state_for_accounts = self._transform_state_from_one_account_format(current_stream_state, ["include_deleted"]) + state_for_accounts = self._transform_state_from_old_deleted_format(state_for_accounts) + account_state = self.get_account_state(account_id, state_for_accounts) + + potentially_new_records_in_the_past = self._filter_statuses and ( + set(self._filter_statuses) - set(account_state.get("filter_statuses", [])) + ) record_value = latest_record[self.cursor_field] - state_value = current_stream_state.get(self.cursor_field) or record_value + state_value = account_state.get(self.cursor_field) or record_value max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) if potentially_new_records_in_the_past: max_cursor = record_value - return { - self.cursor_field: str(max_cursor), - "include_deleted": self._include_deleted, - } + state_for_accounts.setdefault(account_id, {})[self.cursor_field] = str(max_cursor) + state_for_accounts[account_id]["filter_statuses"] = self._filter_statuses + + return state_for_accounts def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: """Include state filter""" @@ -180,9 +283,10 @@ def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: # if start_date is not specified then do not use date filters return {} - potentially_new_records_in_the_past = self._include_deleted and not stream_state.get("include_deleted", False) + potentially_new_records_in_the_past = set(self._filter_statuses) - set(stream_state.get("filter_statuses", [])) + if potentially_new_records_in_the_past: - self.logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") + self.logger.info(f"Ignoring bookmark for {self.name} because `filter_statuses` were changed.") if self._start_date: filter_value = self._start_date else: @@ -203,40 +307,40 @@ def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: class FBMarketingReversedIncrementalStream(FBMarketingIncrementalStream, ABC): """The base class for streams that don't support filtering and return records sorted desc by cursor_value""" - enable_deleted = False # API don't have any filtering, so implement include_deleted in code - def __init__(self, **kwargs): super().__init__(**kwargs) - self._cursor_value = None - self._max_cursor_value = None + self._cursor_values = {} @property def state(self) -> Mapping[str, Any]: """State getter, get current state and serialize it to emmit Airbyte STATE message""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value, - "include_deleted": self._include_deleted, - } + if self._cursor_values: + result_state = {account_id: {self.cursor_field: cursor_value} for account_id, cursor_value in self._cursor_values.items()} + result_state["filter_statuses"] = self._filter_statuses + return result_state return {} @state.setter def state(self, value: Mapping[str, Any]): """State setter, ignore state if current settings mismatch saved state""" - if self._include_deleted and not value.get("include_deleted"): - logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") + transformed_state = self._transform_state_from_one_account_format(value, ["include_deleted"]) + transformed_state = self._transform_state_from_old_deleted_format(transformed_state) + + if set(self._filter_statuses) - set(transformed_state.get("filter_statuses", [])): + logger.info(f"Ignoring bookmark for {self.name} because of enabled `filter_statuses` option") return - self._cursor_value = pendulum.parse(value[self.cursor_field]) + self._cursor_values = {} + for account_id in self._account_ids: + cursor_value = transformed_state.get(account_id, {}).get(self.cursor_field) + if cursor_value is not None: + self._cursor_values[account_id] = pendulum.parse(cursor_value) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Don't have classic cursor filtering""" return {} - def get_record_deleted_status(self, record) -> bool: - return False - def read_records( self, sync_mode: SyncMode, @@ -250,20 +354,28 @@ def read_records( - update state only when we reach the end - stop reading when we reached the end """ + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state") + try: - records_iter = self.list_objects(params=self.request_params(stream_state=stream_state)) + records_iter = self.list_objects( + params=self.request_params(stream_state=account_state), + account_id=account_id, + ) + account_cursor = self._cursor_values.get(account_id) + + max_cursor_value = None for record in records_iter: record_cursor_value = pendulum.parse(record[self.cursor_field]) - if self._cursor_value and record_cursor_value < self._cursor_value: + if account_cursor and record_cursor_value < account_cursor: break - if not self._include_deleted and self.get_record_deleted_status(record): - continue - self._max_cursor_value = max(self._max_cursor_value, record_cursor_value) if self._max_cursor_value else record_cursor_value + max_cursor_value = max(max_cursor_value, record_cursor_value) if max_cursor_value else record_cursor_value record = record.export_all_data() self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) yield record - self._cursor_value = self._max_cursor_value + self._cursor_values[account_id] = max_cursor_value except FacebookRequestError as exc: raise traced_exception(exc) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index 3947689761b3..b6e20f2c60a7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -15,7 +15,20 @@ # The Facebook API error codes indicating rate-limiting are listed at # https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ -FACEBOOK_RATE_LIMIT_ERROR_CODES = (4, 17, 32, 613, 80000, 80001, 80002, 80003, 80004, 80005, 80006, 80008) +FACEBOOK_RATE_LIMIT_ERROR_CODES = ( + 4, + 17, + 32, + 613, + 80000, + 80001, + 80002, + 80003, + 80004, + 80005, + 80006, + 80008, +) FACEBOOK_TEMPORARY_OAUTH_ERROR_CODE = 2 FACEBOOK_BATCH_ERROR_CODE = 960 FACEBOOK_UNKNOWN_ERROR_CODE = 99 @@ -61,12 +74,12 @@ def reduce_request_record_limit(details): def revert_request_record_limit(details): """ - This method is triggered `on_success` after successfull retry, + This method is triggered `on_success` after successful retry, sets the internal class flags to provide the logic to restore the previously reduced `limit` param. """ # reference issue: https://github.com/airbytehq/airbyte/issues/25383 - # set the flag to the api class that the last api call was ssuccessfull + # set the flag to the api class that the last api call was successful details.get("args")[0].last_api_call_is_successfull = True # set the flag to the api class that the `limit` param is restored details.get("args")[0].request_record_limit_is_reduced = False @@ -158,4 +171,9 @@ def traced_exception(fb_exception: FacebookRequestError): failure_type = FailureType.system_error friendly_msg = f"Error: {fb_exception.api_error_code()}, {fb_exception.api_error_message()}." - return AirbyteTracedException(message=friendly_msg or msg, internal_message=msg, failure_type=failure_type, exception=fb_exception) + return AirbyteTracedException( + message=friendly_msg or msg, + internal_message=msg, + failure_type=failure_type, + exception=fb_exception, + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 23fd4b565bdf..d33e202a637b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -9,11 +9,11 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from cached_property import cached_property from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount from facebook_business.adobjects.adimage import AdImage from facebook_business.adobjects.user import User from facebook_business.exceptions import FacebookRequestError +from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses from .base_insight_streams import AdsInsights from .base_streams import FBMarketingIncrementalStream, FBMarketingReversedIncrementalStream, FBMarketingStream @@ -37,21 +37,23 @@ def fetch_thumbnail_data_url(url: str) -> Optional[str]: class AdCreatives(FBMarketingStream): - """AdCreative is append only stream + """AdCreative is append-only stream doc: https://developers.facebook.com/docs/marketing-api/reference/ad-creative """ entity_prefix = "adcreative" - enable_deleted = False def __init__(self, fetch_thumbnail_images: bool = False, **kwargs): super().__init__(**kwargs) self._fetch_thumbnail_images = fetch_thumbnail_images - @cached_property - def fields(self) -> List[str]: - """Remove "thumbnail_data_url" field because it is computed field and it's not a field that we can request from Facebook""" - return [f for f in super().fields if f != "thumbnail_data_url"] + def fields(self, **kwargs) -> List[str]: + """Remove "thumbnail_data_url" field because it is a computed field, and it's not a field that we can request from Facebook""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields(**kwargs) if f != "thumbnail_data_url"] + return self._fields def read_records( self, @@ -68,58 +70,62 @@ def read_records( record["thumbnail_data_url"] = fetch_thumbnail_data_url(thumbnail_url) yield record - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_creatives(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_creatives(params=params, fields=self.fields()) class CustomConversions(FBMarketingStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-conversion""" entity_prefix = "customconversion" - enable_deleted = False - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_conversions(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_conversions(params=params, fields=self.fields()) class CustomAudiences(FBMarketingStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-audience""" entity_prefix = "customaudience" - enable_deleted = False # The `rule` field is excluded from the list because it caused the error message "Please reduce the amount of data" for certain connections. # https://github.com/airbytehq/oncall/issues/2765 fields_exceptions = ["rule"] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_audiences(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_audiences(params=params, fields=self.fields()) class Ads(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup""" entity_prefix = "ad" + status_field = "effective_status" + valid_statuses = [status.value for status in ValidAdStatuses] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ads(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ads(params=params, fields=self.fields()) class AdSets(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/ad-campaign""" entity_prefix = "adset" + status_field = "effective_status" + valid_statuses = [status.value for status in ValidAdSetStatuses] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_sets(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_sets(params=params, fields=self.fields()) class Campaigns(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group""" entity_prefix = "campaign" + status_field = "effective_status" + valid_statuses = [status.value for status in ValidCampaignStatuses] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_campaigns(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_campaigns(params=params, fields=self.fields()) class Activities(FBMarketingIncrementalStream): @@ -129,8 +135,16 @@ class Activities(FBMarketingIncrementalStream): cursor_field = "event_time" primary_key = None - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_activities(fields=self.fields, params=params) + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields(**kwargs) if f != "account_id"] + return self._fields + + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_activities(fields=self.fields(), params=params) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Additional filters associated with state if any set""" @@ -143,9 +157,11 @@ def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: # if start_date is not specified then do not use date filters return {} - potentially_new_records_in_the_past = self._include_deleted and not stream_state.get("include_deleted", False) + potentially_new_records_in_the_past = self._filter_statuses and ( + set(self._filter_statuses) - set(stream_state.get("filter_statuses", [])) + ) if potentially_new_records_in_the_past: - self.logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") + self.logger.info(f"Ignoring bookmark for {self.name} because of enabled `filter_statuses` option") if self._start_date: since = self._start_date else: @@ -160,66 +176,83 @@ class Videos(FBMarketingReversedIncrementalStream): entity_prefix = "video" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields() if f != "account_id"] + return self._fields + + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: # Remove filtering as it is not working for this stream since 2023-01-13 - return self._api.account.get_ad_videos(params=params, fields=self.fields) + return self._api.get_account(account_id=account_id).get_ad_videos(params=params, fields=self.fields()) class AdAccount(FBMarketingStream): """See: https://developers.facebook.com/docs/marketing-api/reference/ad-account""" use_batch = False - enable_deleted = False - def get_task_permissions(self) -> Set[str]: + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._fields_dict = {} + + def get_task_permissions(self, account_id: str) -> Set[str]: """https://developers.facebook.com/docs/marketing-api/reference/ad-account/assigned_users/""" res = set() me = User(fbid="me", api=self._api.api) for business_user in me.get_business_users(): - assigned_users = self._api.account.get_assigned_users(params={"business": business_user["business"].get_id()}) + assigned_users = self._api.get_account(account_id=account_id).get_assigned_users( + params={"business": business_user["business"].get_id()} + ) for assigned_user in assigned_users: if business_user.get_id() == assigned_user.get_id(): res.update(set(assigned_user["tasks"])) return res - @cached_property - def fields(self) -> List[str]: - properties = super().fields + def fields(self, account_id: str, **kwargs) -> List[str]: + if self._fields_dict.get(account_id): + return self._fields_dict.get(account_id) + + properties = super().fields(**kwargs) # https://developers.facebook.com/docs/marketing-apis/guides/javascript-ads-dialog-for-payments/ # To access "funding_source_details", the user making the API call must have a MANAGE task permission for # that specific ad account. - permissions = self.get_task_permissions() + permissions = self.get_task_permissions(account_id=account_id) if "funding_source_details" in properties and "MANAGE" not in permissions: properties.remove("funding_source_details") if "is_prepay_account" in properties and "MANAGE" not in permissions: properties.remove("is_prepay_account") + + self._fields_dict[account_id] = properties return properties - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: """noop in case of AdAccount""" - fields = self.fields + fields = self.fields(account_id=account_id) try: - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] except FacebookRequestError as e: # This is a workaround for cases when account seem to have all the required permissions - # but despite of that is not allowed to get `owner` field. See (https://github.com/airbytehq/oncall/issues/3167) + # but despite that is not allowed to get `owner` field. See (https://github.com/airbytehq/oncall/issues/3167) if e.api_error_code() == 200 and e.api_error_message() == "(#200) Requires business_management permission to manage the object": fields.remove("owner") - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] # FB api returns a non-obvious error when accessing the `funding_source_details` field # even though user is granted all the required permissions (`MANAGE`) # https://github.com/airbytehq/oncall/issues/3031 if e.api_error_code() == 100 and e.api_error_message() == "Unsupported request - method type: get": fields.remove("funding_source_details") - return [FBAdAccount(self._api.account.get_id()).api_get(fields=fields)] + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] raise e class Images(FBMarketingReversedIncrementalStream): """See: https://developers.facebook.com/docs/marketing-api/reference/ad-image""" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_images(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_images(params=params, fields=self.fields(account_id=account_id)) def get_record_deleted_status(self, record) -> bool: return record[AdImage.Field.status] == AdImage.Status.deleted diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py index d3550ae6d1c8..f7e54467f1e1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py @@ -11,7 +11,7 @@ logger = logging.getLogger("airbyte") # Facebook store metrics maximum of 37 months old. Any time range that -# older that 37 months from current date would result in 400 Bad request +# older than 37 months from current date would result in 400 Bad request # HTTP response. # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview DATA_RETENTION_PERIOD = 37 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index ad2454b02ea4..7c0d34ae8139 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -23,12 +23,19 @@ def account_id_fixture(): @fixture(scope="session", name="some_config") def some_config_fixture(account_id): - return {"start_date": "2021-01-23T00:00:00Z", "account_id": f"{account_id}", "access_token": "unknown_token"} + return { + "start_date": "2021-01-23T00:00:00Z", + "account_ids": [f"{account_id}"], + "access_token": "unknown_token", + } @fixture(autouse=True) def mock_default_sleep_interval(mocker): - mocker.patch("source_facebook_marketing.streams.common.DEFAULT_SLEEP_INTERVAL", return_value=pendulum.duration(seconds=5)) + mocker.patch( + "source_facebook_marketing.streams.common.DEFAULT_SLEEP_INTERVAL", + return_value=pendulum.duration(seconds=5), + ) @fixture(name="fb_account_response") @@ -41,7 +48,12 @@ def fb_account_response_fixture(account_id): "id": f"act_{account_id}", } ], - "paging": {"cursors": {"before": "MjM4NDYzMDYyMTcyNTAwNzEZD", "after": "MjM4NDYzMDYyMTcyNTAwNzEZD"}}, + "paging": { + "cursors": { + "before": "MjM4NDYzMDYyMTcyNTAwNzEZD", + "after": "MjM4NDYzMDYyMTcyNTAwNzEZD", + } + }, }, "status_code": 200, } @@ -49,8 +61,17 @@ def fb_account_response_fixture(account_id): @fixture(name="api") def api_fixture(some_config, requests_mock, fb_account_response): - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/adaccounts", [fb_account_response]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_id']}/", [fb_account_response]) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/adaccounts", + [fb_account_response], + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + + f"/{FB_API_VERSION}/act_{some_config['account_ids'][0]}/", + [fb_account_response], + ) return api diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py new file mode 100644 index 000000000000..ef0591147cc7 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +from datetime import datetime +from typing import Any, List, MutableMapping + +import pendulum + +ACCESS_TOKEN = "test_access_token" +ACCOUNT_ID = "111111111111111" +CLIENT_ID = "test_client_id" +CLIENT_SECRET = "test_client_secret" +DATE_FORMAT = "%Y-%m-%d" +DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +END_DATE = "2023-01-01T23:59:59Z" +NOW = pendulum.now(tz="utc") +START_DATE = "2023-01-01T00:00:00Z" + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: MutableMapping[str, Any] = { + "account_ids": [ACCOUNT_ID], + "access_token": ACCESS_TOKEN, + "start_date": START_DATE, + "end_date": END_DATE, + "include_deleted": True, + "fetch_thumbnail_images": True, + "custom_insights": [], + "page_size": 100, + "insights_lookback_window": 28, + "insights_job_timeout": 60, + "action_breakdowns_allow_empty": True, + "client_id": CLIENT_ID, + "client_secret": CLIENT_SECRET, + } + + def with_account_ids(self, account_ids: List[str]) -> ConfigBuilder: + self._config["account_ids"] = account_ids + return self + + def with_start_date(self, start_date: datetime) -> ConfigBuilder: + self._config["start_date"] = start_date.strftime(DATE_TIME_FORMAT) + return self + + def with_end_date(self, end_date: datetime) -> ConfigBuilder: + self._config["end_date"] = end_date.strftime(DATE_TIME_FORMAT) + return self + + def build(self) -> MutableMapping[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py new file mode 100644 index 000000000000..69b284d6d308 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict +from urllib.parse import urlunparse + +from airbyte_cdk.test.mock_http.request import HttpRequest +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + +NEXT_PAGE_TOKEN = "QVFIUlhOX3Rnbm5Y" + + +class FacebookMarketingPaginationStrategy(PaginationStrategy): + def __init__(self, request: HttpRequest, next_page_token: str) -> None: + self._next_page_token = next_page_token + self._next_page_url = f"{urlunparse(request._parsed_url)}&after={self._next_page_token}" + + def update(self, response: Dict[str, Any]) -> None: + # set a constant value for paging.cursors.after so we know how the 'next' link is built + # https://developers.facebook.com/docs/graph-api/results + response["paging"]["cursors"]["after"] = self._next_page_token + response["paging"]["next"] = self._next_page_url diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py new file mode 100644 index 000000000000..a07c81b13448 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.test.mock_http.request import HttpRequest + +from .config import ACCESS_TOKEN, ACCOUNT_ID + + +def get_account_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: + return RequestBuilder.get_account_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + + +class RequestBuilder: + + @classmethod + def get_account_endpoint(cls, access_token: str, account_id: str) -> RequestBuilder: + return cls(access_token=access_token).with_account_id(account_id) + + @classmethod + def get_videos_endpoint(cls, access_token: str, account_id: str) -> RequestBuilder: + return cls(access_token=access_token, resource="advideos").with_account_id(account_id) + + @classmethod + def get_insights_endpoint(cls, access_token: str, account_id: str) -> RequestBuilder: + return cls(access_token=access_token, resource="insights").with_account_id(account_id) + + @classmethod + def get_execute_batch_endpoint(cls, access_token: str) -> RequestBuilder: + return cls(access_token=access_token) + + @classmethod + def get_insights_download_endpoint(cls, access_token: str, job_id: str) -> RequestBuilder: + return cls(access_token=access_token, resource=f"{job_id}/insights") + + def __init__(self, access_token: str, resource: Optional[str] = "") -> None: + self._account_id = None + self._resource = resource + self._query_params = {"access_token": access_token} + self._body = None + + def with_account_id(self, account_id: str) -> RequestBuilder: + self._account_id = account_id + return self + + def with_limit(self, limit: int) -> RequestBuilder: + self._query_params["limit"] = limit + return self + + def with_summary(self) -> RequestBuilder: + self._query_params["summary"] = "true" + return self + + def with_fields(self, fields: List[str]) -> RequestBuilder: + self._query_params["fields"] = self._get_formatted_fields(fields) + return self + + def with_next_page_token(self, next_page_token: str) -> RequestBuilder: + self._query_params["after"] = next_page_token + return self + + def with_body(self, body: Union[str, bytes, Mapping[str, Any]]) -> RequestBuilder: + self._body = body + return self + + def build(self) -> HttpRequest: + return HttpRequest( + url=f"https://graph.facebook.com/v17.0/{self._account_sub_path()}{self._resource}", + query_params=self._query_params, + body=self._body, + ) + + def _account_sub_path(self) -> str: + return f"act_{self._account_id}/" if self._account_id else "" + + @staticmethod + def _get_formatted_fields(fields: List[str]) -> str: + return ",".join(fields) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/response_builder.py new file mode 100644 index 000000000000..836ab1a41e3a --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/response_builder.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.test.mock_http import HttpResponse + +from .config import ACCOUNT_ID + + +def build_response( + body: Union[Mapping[str, Any], List[Mapping[str, Any]]], + status_code: HTTPStatus, + headers: Optional[Mapping[str, str]] = None, +) -> HttpResponse: + headers = headers or {} + return HttpResponse(body=json.dumps(body), status_code=status_code.value, headers=headers) + + +def get_account_response(account_id: Optional[str] = ACCOUNT_ID) -> HttpResponse: + response = {"account_id": account_id, "id": f"act_{account_id}"} + return build_response(body=response, status_code=HTTPStatus.OK) + + +def error_reduce_amount_of_data_response() -> HttpResponse: + response = { + "error": {"code": 1, "message": "Please reduce the amount of data you're asking for, then retry your request"}, + } + return build_response(body=response, status_code=HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py new file mode 100644 index 000000000000..2fe71e37f271 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -0,0 +1,533 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import json +from datetime import datetime, timedelta +from http import HTTPStatus +from typing import List, Optional, Union +from unittest import TestCase + +import freezegun +import pendulum +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponse, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import AirbyteStateMessage, SyncMode +from source_facebook_marketing.streams.async_job import Status + +from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder +from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy +from .request_builder import RequestBuilder, get_account_request +from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response +from .utils import config, encode_request_body, read_output + +_STREAM_NAME = "ads_insights_action_product_id" +_CURSOR_FIELD = "date_start" +_REPORT_RUN_ID = "1571860060019548" +_JOB_ID = "1049937379601625" + + +def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: + return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + + +def _job_start_request( + account_id: Optional[str] = ACCOUNT_ID, since: Optional[datetime] = None, until: Optional[datetime] = None +) -> RequestBuilder: + since = since.strftime(DATE_FORMAT) if since else START_DATE[:10] + until = until.strftime(DATE_FORMAT) if until else END_DATE[:10] + body = { + "level": "ad", + "action_breakdowns": [], + "action_report_time": "mixed", + "breakdowns": ["product_id"], + "fields": [ + "account_currency", + "account_id", + "account_name", + "action_values", + "actions", + "ad_click_actions", + "ad_id", + "ad_impression_actions", + "ad_name", + "adset_id", + "adset_name", + "age_targeting", + "attribution_setting", + "auction_bid", + "auction_competitiveness", + "auction_max_competitor_bid", + "buying_type", + "campaign_id", + "campaign_name", + "canvas_avg_view_percent", + "canvas_avg_view_time", + "catalog_segment_actions", + "catalog_segment_value", + "catalog_segment_value_mobile_purchase_roas", + "catalog_segment_value_omni_purchase_roas", + "catalog_segment_value_website_purchase_roas", + "clicks", + "conversion_rate_ranking", + "conversion_values", + "conversions", + "converted_product_quantity", + "converted_product_value", + "cost_per_15_sec_video_view", + "cost_per_2_sec_continuous_video_view", + "cost_per_action_type", + "cost_per_ad_click", + "cost_per_conversion", + "cost_per_estimated_ad_recallers", + "cost_per_inline_link_click", + "cost_per_inline_post_engagement", + "cost_per_outbound_click", + "cost_per_thruplay", + "cost_per_unique_action_type", + "cost_per_unique_click", + "cost_per_unique_inline_link_click", + "cost_per_unique_outbound_click", + "cpc", + "cpm", + "cpp", + "created_time", + "ctr", + "date_start", + "date_stop", + "engagement_rate_ranking", + "estimated_ad_recall_rate", + "estimated_ad_recall_rate_lower_bound", + "estimated_ad_recall_rate_upper_bound", + "estimated_ad_recallers", + "estimated_ad_recallers_lower_bound", + "estimated_ad_recallers_upper_bound", + "frequency", + "full_view_impressions", + "full_view_reach", + "gender_targeting", + "impressions", + "inline_link_click_ctr", + "inline_link_clicks", + "inline_post_engagement", + "instant_experience_clicks_to_open", + "instant_experience_clicks_to_start", + "instant_experience_outbound_clicks", + "labels", + "location", + "mobile_app_purchase_roas", + "objective", + "optimization_goal", + "outbound_clicks", + "outbound_clicks_ctr", + "purchase_roas", + "qualifying_question_qualify_answer_rate", + "quality_ranking", + "reach", + "social_spend", + "spend", + "unique_actions", + "unique_clicks", + "unique_ctr", + "unique_inline_link_click_ctr", + "unique_inline_link_clicks", + "unique_link_clicks_ctr", + "unique_outbound_clicks", + "unique_outbound_clicks_ctr", + "updated_time", + "video_15_sec_watched_actions", + "video_30_sec_watched_actions", + "video_avg_time_watched_actions", + "video_continuous_2_sec_watched_actions", + "video_p100_watched_actions", + "video_p25_watched_actions", + "video_p50_watched_actions", + "video_p75_watched_actions", + "video_p95_watched_actions", + "video_play_actions", + "video_play_curve_actions", + "video_play_retention_0_to_15s_actions", + "video_play_retention_20_to_60s_actions", + "video_play_retention_graph_actions", + "video_time_watched_actions", + "website_ctr", + "website_purchase_roas", + "wish_bid", + ], + "time_increment": 1, + "action_attribution_windows": ["1d_click", "7d_click", "28d_click", "1d_view", "7d_view", "28d_view"], + "time_range": {"since": since, "until": until}, + } + return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id).with_body( + encode_request_body(body) + ) + + +def _job_status_request(report_run_ids: Union[str, List[str]]) -> RequestBuilder: + if isinstance(report_run_ids, str): + report_run_ids = [report_run_ids] + body = {"batch": [{"method": "GET", "relative_url": f"{report_run_id}/"} for report_run_id in report_run_ids]} + return RequestBuilder.get_execute_batch_endpoint(access_token=ACCESS_TOKEN).with_body(encode_request_body(body)) + + +def _get_insights_request(job_id: str) -> RequestBuilder: + return RequestBuilder.get_insights_download_endpoint(access_token=ACCESS_TOKEN, job_id=job_id).with_limit(100) + + +def _update_api_throttle_limit_response(api_throttle: Optional[int] = 0) -> HttpResponse: + body = {} + headers = { + "x-fb-ads-insights-throttle": json.dumps( + {"app_id_util_pct": api_throttle, "acc_id_util_pct": api_throttle, "ads_api_access_tier": "standard_access"} + ), + } + return build_response(body=body, status_code=HTTPStatus.OK, headers=headers) + + +def _job_start_response(report_run_id: str) -> HttpResponse: + body = {"report_run_id": report_run_id} + return build_response(body=body, status_code=HTTPStatus.OK) + + +def _job_status_response( + job_ids: Union[str, List[str]], status: Optional[Status] = Status.COMPLETED, account_id: Optional[str] = ACCOUNT_ID +) -> HttpResponse: + if isinstance(job_ids, str): + job_ids = [job_ids] + body = [ + { + "body": json.dumps( + { + "id": job_id, "account_id": account_id, "async_status": status, "async_percent_completion": 100 + } + ), + } for job_id in job_ids + ] + return build_response(body=body, status_code=HTTPStatus.OK) + + +def _insights_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=FieldPath("data"), + pagination_strategy=FacebookMarketingPaginationStrategy( + request=_get_insights_request(_JOB_ID).with_limit(100).build(), next_page_token=NEXT_PAGE_TOKEN + ), + ) + + +def _ads_insights_action_product_id_record() -> RecordBuilder: + return create_record_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=FieldPath("data"), + record_cursor_path=FieldPath(_CURSOR_FIELD), + ) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh(TestCase): + + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + client_side_account_id = "123123123" + server_side_account_id = "321321321" + + start_date = pendulum.parse(START_DATE) + end_date = start_date + timedelta(hours=23) + + http_mocker.get( + get_account_request(account_id=client_side_account_id).build(), + get_account_response(account_id=server_side_account_id), + ) + http_mocker.get( + _update_api_throttle_limit_request(account_id=server_side_account_id).build(), + _update_api_throttle_limit_response(), + ) + http_mocker.post( + _job_start_request(account_id=server_side_account_id, since=start_date, until=end_date).build(), + _job_start_response(_REPORT_RUN_ID), + ) + http_mocker.post(_job_status_request(_REPORT_RUN_ID).build(), _job_status_response(_JOB_ID)) + http_mocker.get( + _get_insights_request(_JOB_ID).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read( + config().with_account_ids([client_side_account_id]).with_start_date(start_date).with_end_date(end_date) + ) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get(_update_api_throttle_limit_request().build(), _update_api_throttle_limit_response()) + http_mocker.post(_job_start_request().build(), _job_start_response(_REPORT_RUN_ID)) + http_mocker.post(_job_status_request(_REPORT_RUN_ID).build(), _job_status_response(_JOB_ID)) + http_mocker.get( + _get_insights_request(_JOB_ID).build(), + _insights_response().with_pagination().with_record(_ads_insights_action_product_id_record()).build(), + ) + http_mocker.get( + _get_insights_request(_JOB_ID).with_next_page_token(NEXT_PAGE_TOKEN).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).with_record( + _ads_insights_action_product_id_record() + ).build(), + ) + + output = self._read(config()) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_api_throttle_exceeds_limit_on_first_check_when_read_then_wait_throttle_down_and_return_records( + self, http_mocker: HttpMocker + ) -> None: + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get( + _update_api_throttle_limit_request().build(), + [ + _update_api_throttle_limit_response(api_throttle=100), + _update_api_throttle_limit_response(api_throttle=0), + ], + ) + http_mocker.post(_job_start_request().build(), _job_start_response(_REPORT_RUN_ID)) + http_mocker.post(_job_status_request(_REPORT_RUN_ID).build(), _job_status_response(_JOB_ID)) + http_mocker.get( + _get_insights_request(_JOB_ID).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read(config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_days_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + start_date = NOW.subtract(days=1) + end_date = NOW + report_run_id_1 = "1571860060019500" + report_run_id_2 = "4571860060019599" + job_id_1 = "1049937379601600" + job_id_2 = "1049937379601699" + + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get(_update_api_throttle_limit_request().build(), _update_api_throttle_limit_response()) + http_mocker.post( + _job_start_request(since=start_date, until=start_date).build(), _job_start_response(report_run_id_1) + ) + http_mocker.post( + _job_start_request(since=end_date, until=end_date).build(), _job_start_response(report_run_id_2) + ) + http_mocker.post( + _job_status_request([report_run_id_1, report_run_id_2]).build(), _job_status_response([job_id_1, job_id_2]) + ) + http_mocker.get( + _get_insights_request(job_id_1).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + http_mocker.get( + _get_insights_request(job_id_2).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read(config().with_start_date(start_date).with_end_date(end_date)) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_multiple_account_ids_when_read_then_return_records_from_all_accounts( + self, http_mocker: HttpMocker + ) -> None: + account_id_1 = "123123123" + account_id_2 = "321321321" + report_run_id_1 = "1571860060019500" + report_run_id_2 = "4571860060019599" + job_id_1 = "1049937379601600" + job_id_2 = "1049937379601699" + + api_throttle_limit_response = _update_api_throttle_limit_response() + + http_mocker.get( + get_account_request().with_account_id(account_id_1).build(), get_account_response(account_id=account_id_1) + ) + http_mocker.get( + _update_api_throttle_limit_request().with_account_id(account_id_1).build(), api_throttle_limit_response + ) + http_mocker.post( + _job_start_request().with_account_id(account_id_1).build(), _job_start_response(report_run_id_1) + ) + http_mocker.post( + _job_status_request(report_run_id_1).build(), _job_status_response(job_id_1, account_id=account_id_1) + ) + http_mocker.get( + _get_insights_request(job_id_1).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + http_mocker.get( + get_account_request().with_account_id(account_id_2).build(), get_account_response(account_id=account_id_2) + ) + http_mocker.get( + _update_api_throttle_limit_request().with_account_id(account_id_2).build(), api_throttle_limit_response + ) + http_mocker.post( + _job_start_request().with_account_id(account_id_2).build(), _job_start_response(report_run_id_2) + ) + http_mocker.post( + _job_status_request(report_run_id_2).build(), _job_status_response(job_id_2, account_id=account_id_2) + ) + http_mocker.get( + _get_insights_request(job_id_2).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read(config().with_account_ids([account_id_1, account_id_2])) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_status_500_reduce_amount_of_data_when_read_then_limit_reduced(self, http_mocker: HttpMocker) -> None: + limit = 100 + + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get(_update_api_throttle_limit_request().build(), _update_api_throttle_limit_response()) + http_mocker.post(_job_start_request().build(), _job_start_response(_REPORT_RUN_ID)) + http_mocker.post(_job_status_request(_REPORT_RUN_ID).build(), _job_status_response(_JOB_ID)) + http_mocker.get( + _get_insights_request(_JOB_ID).with_limit(limit).build(), + error_reduce_amount_of_data_response(), + ) + http_mocker.get( + _get_insights_request(_JOB_ID).with_limit(int(limit / 2)).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + self._read(config()) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental(TestCase): + @staticmethod + def _read( + config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.incremental, + state=state, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_when_read_then_state_message_produced_and_state_match_start_interval( + self, http_mocker: HttpMocker + ) -> None: + account_id = "123123123" + start_date = NOW.set(hour=0, minute=0, second=0) + end_date = NOW.set(hour=23, minute=59, second=59) + + http_mocker.get( + get_account_request().with_account_id(account_id).build(), get_account_response(account_id=account_id) + ) + http_mocker.get( + _update_api_throttle_limit_request().with_account_id(account_id).build(), + _update_api_throttle_limit_response(), + ) + http_mocker.post( + _job_start_request(since=start_date, until=end_date).with_account_id(account_id).build(), + _job_start_response(_REPORT_RUN_ID), + ) + http_mocker.post( + _job_status_request(_REPORT_RUN_ID).build(), _job_status_response(_JOB_ID, account_id=account_id) + ) + http_mocker.get( + _get_insights_request(_JOB_ID).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read(config().with_account_ids([account_id]).with_start_date(start_date).with_end_date(end_date)) + cursor_value_from_state_message = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id, {}).get( + _CURSOR_FIELD + ) + assert cursor_value_from_state_message == start_date.strftime(DATE_FORMAT) + + @HttpMocker() + def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_and_state_match_start_interval( + self, http_mocker: HttpMocker + ) -> None: + account_id_1 = "123123123" + account_id_2 = "321321321" + start_date = NOW.set(hour=0, minute=0, second=0) + end_date = NOW.set(hour=23, minute=59, second=59) + report_run_id_1 = "1571860060019500" + report_run_id_2 = "4571860060019599" + job_id_1 = "1049937379601600" + job_id_2 = "1049937379601699" + + api_throttle_limit_response = _update_api_throttle_limit_response() + + http_mocker.get( + get_account_request().with_account_id(account_id_1).build(), get_account_response(account_id=account_id_1) + ) + http_mocker.get( + _update_api_throttle_limit_request().with_account_id(account_id_1).build(), api_throttle_limit_response + ) + http_mocker.post( + _job_start_request(since=start_date, until=end_date).with_account_id(account_id_1).build(), + _job_start_response(report_run_id_1), + ) + http_mocker.post( + _job_status_request(report_run_id_1).build(), _job_status_response(job_id_1, account_id=account_id_1) + ) + http_mocker.get( + _get_insights_request(job_id_1).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + http_mocker.get( + get_account_request().with_account_id(account_id_2).build(), get_account_response(account_id=account_id_2) + ) + http_mocker.get( + _update_api_throttle_limit_request().with_account_id(account_id_2).build(), api_throttle_limit_response + ) + http_mocker.post( + _job_start_request(since=start_date, until=end_date).with_account_id(account_id_2).build(), + _job_start_response(report_run_id_2), + ) + http_mocker.post( + _job_status_request(report_run_id_2).build(), _job_status_response(job_id_2, account_id=account_id_2) + ) + http_mocker.get( + _get_insights_request(job_id_2).build(), + _insights_response().with_record(_ads_insights_action_product_id_record()).build(), + ) + + output = self._read( + config().with_account_ids([account_id_1, account_id_2]).with_start_date(start_date).with_end_date(end_date) + ) + cursor_value_from_state_account_1 = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id_1, {}).get( + _CURSOR_FIELD + ) + cursor_value_from_state_account_2 = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id_2, {}).get( + _CURSOR_FIELD + ) + expected_cursor_value = start_date.strftime(DATE_FORMAT) + assert cursor_value_from_state_account_1 == expected_cursor_value + assert cursor_value_from_state_account_2 == expected_cursor_value diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py new file mode 100644 index 000000000000..f2aa2990f3fb --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -0,0 +1,348 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import List, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStateMessage, SyncMode + +from .config import ACCESS_TOKEN, ACCOUNT_ID, NOW, ConfigBuilder +from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy +from .request_builder import RequestBuilder, get_account_request +from .response_builder import error_reduce_amount_of_data_response, get_account_response +from .utils import config, read_output + +_STREAM_NAME = "videos" +_CURSOR_FIELD = "updated_time" +_FIELDS = [ + "id", + "ad_breaks", + "backdated_time", + "backdated_time_granularity", + "content_category", + "content_tags", + "created_time", + "custom_labels", + "description", + "embed_html", + "embeddable", + "format", + "icon", + "is_crosspost_video", + "is_crossposting_eligible", + "is_episode", + "is_instagram_eligible", + "length", + "live_status", + "permalink_url", + "post_views", + "premiere_living_room_status", + "published", + "scheduled_publish_time", + "source", + "title", + "universal_video_id", + "updated_time", + "views", +] + + +def _get_videos_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: + return RequestBuilder.get_videos_endpoint( + access_token=ACCESS_TOKEN, account_id=account_id + ).with_limit(100).with_fields(_FIELDS).with_summary() + + +def _get_videos_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=FieldPath("data"), + pagination_strategy=FacebookMarketingPaginationStrategy( + request=_get_videos_request().build(), next_page_token=NEXT_PAGE_TOKEN + ), + ) + + +def _video_record() -> RecordBuilder: + return create_record_builder( + response_template=find_template(_STREAM_NAME, __file__), + records_path=FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath(_CURSOR_FIELD), + ) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh(TestCase): + + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + client_side_account_id = ACCOUNT_ID + server_side_account_id = ACCOUNT_ID + + http_mocker.get( + get_account_request(account_id=client_side_account_id).build(), + get_account_response(account_id=server_side_account_id), + ) + http_mocker.get( + _get_videos_request(account_id=server_side_account_id).build(), + _get_videos_response().with_record(_video_record()).build(), + ) + + output = self._read(config().with_account_ids([client_side_account_id])) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get( + _get_videos_request().build(), + _get_videos_response().with_pagination().with_record(_video_record()).build(), + ) + http_mocker.get( + _get_videos_request().with_next_page_token(NEXT_PAGE_TOKEN).build(), + _get_videos_response().with_record(_video_record()).with_record(_video_record()).build(), + ) + + output = self._read(config()) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_multiple_account_ids_when_read_then_return_records_from_all_accounts( + self, http_mocker: HttpMocker + ) -> None: + account_id_1 = "123123123" + account_id_2 = "321321321" + + http_mocker.get( + get_account_request().with_account_id(account_id_1).build(), get_account_response(account_id=account_id_1) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_1).build(), + _get_videos_response().with_record(_video_record()).build(), + ) + http_mocker.get( + get_account_request().with_account_id(account_id_2).build(), get_account_response(account_id=account_id_2) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_2).build(), + _get_videos_response().with_record(_video_record()).build(), + ) + + output = self._read(config().with_account_ids([account_id_1, account_id_2])) + assert len(output.records) == 2 + + @HttpMocker() + def test_when_read_then_add_account_id_field(self, http_mocker: HttpMocker) -> None: + account_id = "123123123" + + http_mocker.get( + get_account_request().with_account_id(account_id).build(), get_account_response(account_id=account_id) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id).build(), + _get_videos_response().with_record(_video_record()).build(), + ) + + output = self._read(config().with_account_ids([account_id])) + assert output.records[0].record.data["account_id"] == account_id + + @HttpMocker() + def test_when_read_then_datetime_fields_transformed(self, http_mocker: HttpMocker) -> None: + created_time_field = "created_time" + input_datetime_value = "2024-01-01t00:00:00 0000" + expected_datetime_value = "2024-01-01T00:00:00+0000" + + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get( + _get_videos_request().with_fields(_FIELDS).with_summary().build(), + _get_videos_response().with_record( + _video_record().with_field(FieldPath(created_time_field), input_datetime_value) + ).build(), + ) + + output = self._read(config()) + assert output.records[0].record.data[created_time_field] == expected_datetime_value + + @HttpMocker() + def test_given_status_500_reduce_amount_of_data_when_read_then_limit_reduced(self, http_mocker: HttpMocker) -> None: + limit = 100 + + http_mocker.get(get_account_request().build(), get_account_response()) + http_mocker.get( + _get_videos_request().with_limit(limit).with_fields(_FIELDS).with_summary().build(), + error_reduce_amount_of_data_response(), + ) + http_mocker.get( + _get_videos_request().with_limit(int(limit / 2)).with_fields(_FIELDS).with_summary().build(), + _get_videos_response().with_record(_video_record()).build(), + ) + + self._read(config()) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental(TestCase): + @staticmethod + def _read( + config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False + ) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.incremental, + state=state, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + account_id = "123123123" + + http_mocker.get( + get_account_request().with_account_id(account_id).build(), get_account_response(account_id=account_id) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id).build(), + _get_videos_response().with_record(_video_record().with_cursor(max_cursor_value)).with_record( + _video_record().with_cursor(min_cursor_value) + ).build(), + ) + + output = self._read(config().with_account_ids([account_id])) + cursor_value_from_state_message = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id, {}).get( + _CURSOR_FIELD + ) + assert cursor_value_from_state_message == max_cursor_value + + @HttpMocker() + def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_and_state_match_latest_record( + self, http_mocker: HttpMocker + ) -> None: + account_id_1 = "123123123" + account_id_2 = "321321321" + min_cursor_value_account_id_1 = "2024-01-01T00:00:00+00:00" + max_cursor_value_account_id_1 = "2024-02-01T00:00:00+00:00" + min_cursor_value_account_id_2 = "2024-03-01T00:00:00+00:00" + max_cursor_value_account_id_2 = "2024-04-01T00:00:00+00:00" + + http_mocker.get( + get_account_request().with_account_id(account_id_1).build(), get_account_response(account_id=account_id_1) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_1).build(), + _get_videos_response().with_record(_video_record().with_cursor(max_cursor_value_account_id_1)).with_record( + _video_record().with_cursor(min_cursor_value_account_id_1) + ).build(), + ) + http_mocker.get( + get_account_request().with_account_id(account_id_2).build(), get_account_response(account_id=account_id_2) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_2).build(), + _get_videos_response().with_record(_video_record().with_cursor(max_cursor_value_account_id_2)).with_record( + _video_record().with_cursor(min_cursor_value_account_id_2) + ).build(), + ) + + output = self._read(config().with_account_ids([account_id_1, account_id_2])) + cursor_value_from_state_account_1 = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id_1, {}).get( + _CURSOR_FIELD + ) + cursor_value_from_state_account_2 = output.most_recent_state.get(_STREAM_NAME, {}).get(account_id_2, {}).get( + _CURSOR_FIELD + ) + assert cursor_value_from_state_account_1 == max_cursor_value_account_id_1 + assert cursor_value_from_state_account_2 == max_cursor_value_account_id_2 + + @HttpMocker() + def test_given_state_when_read_then_records_with_cursor_value_less_than_state_filtered( + self, http_mocker: HttpMocker + ) -> None: + account_id = "123123123" + cursor_value_1 = "2024-01-01T00:00:00+00:00" + cursor_value_2 = "2024-01-02T00:00:00+00:00" + cursor_value_3 = "2024-01-03T00:00:00+00:00" + + http_mocker.get( + get_account_request().with_account_id(account_id).build(), get_account_response(account_id=account_id) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id).build(), + _get_videos_response().with_record(_video_record().with_cursor(cursor_value_3)).with_record( + _video_record().with_cursor(cursor_value_2) + ).with_record( + _video_record().with_cursor(cursor_value_1) + ).build(), + ) + + output = self._read( + config().with_account_ids([account_id]), + state=StateBuilder().with_stream_state(_STREAM_NAME, {account_id: {_CURSOR_FIELD: cursor_value_2}}).build(), + ) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_multiple_account_ids_when_read_then_records_with_cursor_value_less_than_state_filtered( + self, http_mocker: HttpMocker + ) -> None: + account_id_1 = "123123123" + account_id_2 = "321321321" + cursor_value_1 = "2024-01-01T00:00:00+00:00" + cursor_value_2 = "2024-01-02T00:00:00+00:00" + cursor_value_3 = "2024-01-03T00:00:00+00:00" + + http_mocker.get( + get_account_request().with_account_id(account_id_1).build(), get_account_response(account_id=account_id_1) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_1).build(), + _get_videos_response().with_record(_video_record().with_cursor(cursor_value_3)).with_record( + _video_record().with_cursor(cursor_value_2) + ).with_record( + _video_record().with_cursor(cursor_value_1) + ).build(), + ) + http_mocker.get( + get_account_request().with_account_id(account_id_2).build(), get_account_response(account_id=account_id_2) + ) + http_mocker.get( + _get_videos_request().with_account_id(account_id_2).build(), + _get_videos_response().with_record(_video_record().with_cursor(cursor_value_3)).with_record( + _video_record().with_cursor(cursor_value_2) + ).with_record( + _video_record().with_cursor(cursor_value_1) + ).build(), + ) + + stream_state = {account_id_1: {_CURSOR_FIELD: cursor_value_2}, account_id_2: {_CURSOR_FIELD: cursor_value_2}} + output = self._read( + config().with_account_ids([account_id_1, account_id_2]), + state=StateBuilder().with_stream_state(_STREAM_NAME, stream_state).build(), + ) + assert len(output.records) == 4 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py new file mode 100644 index 000000000000..14d184412d9f --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode +from facebook_business.api import _top_level_param_json_encode +from source_facebook_marketing import SourceFacebookMarketing + +from .config import ConfigBuilder + + +def config() -> ConfigBuilder: + return ConfigBuilder() + + +def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(stream_name, sync_mode).build() + + +def source() -> SourceFacebookMarketing: + return SourceFacebookMarketing() + + +def read_output( + config_builder: ConfigBuilder, + stream_name: str, + sync_mode: SyncMode, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: Optional[bool] = False, +) -> EntrypointOutput: + _catalog = catalog(stream_name, sync_mode) + _config = config_builder.build() + return read(source(), _config, _catalog, state, expecting_exception) + + +def encode_request_body(body: Dict[str, Any]) -> str: + body = body.copy() + return urlencode(_top_level_param_json_encode(body)) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/ads_insights_action_product_id.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/ads_insights_action_product_id.json new file mode 100644 index 000000000000..2645a16c4eec --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/ads_insights_action_product_id.json @@ -0,0 +1,203 @@ +{ + "data": [ + { + "account_currency": "USD", + "account_id": "798085168510957", + "account_name": "Porsche Riverside", + "actions": [ + { + "action_type": "page_engagement", + "value": "10", + "1d_click": "10", + "7d_click": "10", + "28d_click": "10" + }, + { + "action_type": "post_engagement", + "value": "10", + "1d_click": "10", + "7d_click": "10", + "28d_click": "10" + } + ], + "ad_id": "23854695759200548", + "ad_name": "New - AIA - Advertised Offers - Cayenne", + "adset_id": "23854404706320548", + "adset_name": "New Cayenne", + "buying_type": "AUCTION", + "campaign_id": "23854404676180548", + "campaign_name": "zzzzzNew - AIA - Advertised Offers", + "clicks": "9", + "cost_per_action_type": [ + { + "action_type": "link_click", + "value": "1.594444", + "1d_click": "1.594444", + "7d_click": "1.594444", + "28d_click": "1.594444" + }, + { + "action_type": "post_engagement", + "value": "1.435", + "1d_click": "1.435", + "7d_click": "1.435", + "28d_click": "1.435" + }, + { + "action_type": "page_engagement", + "value": "1.435", + "1d_click": "1.435", + "7d_click": "1.435", + "28d_click": "1.435" + } + ], + "cost_per_inline_link_click": "1.594444", + "cost_per_inline_post_engagement": "1.435", + "cost_per_outbound_click": [ + { + "action_type": "outbound_click", + "value": "1.594444" + } + ], + "cpc": "1.594444", + "cpm": "77.567568", + "created_time": "2023-06-01", + "ctr": "4.864865", + "date_start": "2023-06-01", + "date_stop": "2023-06-01", + "impressions": "185", + "inline_link_click_ctr": "4.864865", + "inline_link_clicks": "9", + "inline_post_engagement": "10", + "objective": "PRODUCT_CATALOG_SALES", + "optimization_goal": "OFFSITE_CONVERSIONS", + "outbound_clicks": [ + { + "action_type": "outbound_click", + "value": "9" + } + ], + "outbound_clicks_ctr": [ + { + "action_type": "outbound_click", + "value": "4.864865" + } + ], + "spend": "14.35", + "updated_time": "2023-06-01", + "website_ctr": [ + { + "action_type": "link_click", + "value": "4.864865" + } + ], + "product_id": "5983858725075630" + }, + { + "account_currency": "USD", + "account_id": "798085168510957", + "account_name": "Porsche Riverside", + "actions": [ + { + "action_type": "page_engagement", + "value": "4", + "1d_click": "4", + "7d_click": "4", + "28d_click": "4" + }, + { + "action_type": "post_engagement", + "value": "4", + "1d_click": "4", + "7d_click": "4", + "28d_click": "4" + }, + { + "action_type": "link_click", + "value": "4", + "1d_click": "4", + "7d_click": "4", + "28d_click": "4" + } + ], + "ad_id": "23854695759200548", + "ad_name": "New - AIA - Advertised Offers - Cayenne", + "adset_id": "23854404706320548", + "adset_name": "New Cayenne", + "buying_type": "AUCTION", + "campaign_id": "23854404676180548", + "campaign_name": "zzzzzNew - AIA - Advertised Offers", + "clicks": "7", + "cost_per_action_type": [ + { + "action_type": "link_click", + "value": "2.1325", + "1d_click": "2.1325", + "7d_click": "2.1325", + "28d_click": "2.1325" + }, + { + "action_type": "post_engagement", + "value": "2.1325", + "1d_click": "2.1325", + "7d_click": "2.1325", + "28d_click": "2.1325" + }, + { + "action_type": "page_engagement", + "value": "2.1325", + "1d_click": "2.1325", + "7d_click": "2.1325", + "28d_click": "2.1325" + } + ], + "cost_per_inline_link_click": "2.1325", + "cost_per_inline_post_engagement": "2.1325", + "cost_per_outbound_click": [ + { + "action_type": "outbound_click", + "value": "2.1325" + } + ], + "cpc": "1.218571", + "cpm": "49.883041", + "created_time": "2023-06-01", + "ctr": "4.093567", + "date_start": "2023-06-01", + "date_stop": "2023-06-01", + "impressions": "171", + "inline_link_click_ctr": "2.339181", + "inline_link_clicks": "4", + "inline_post_engagement": "4", + "objective": "PRODUCT_CATALOG_SALES", + "optimization_goal": "OFFSITE_CONVERSIONS", + "outbound_clicks": [ + { + "action_type": "outbound_click", + "value": "4" + } + ], + "outbound_clicks_ctr": [ + { + "action_type": "outbound_click", + "value": "2.339181" + } + ], + "spend": "8.53", + "updated_time": "2023-06-01", + "website_ctr": [ + { + "action_type": "link_click", + "value": "2.339181" + } + ], + "product_id": "6023934044385671" + } + ], + "paging": { + "cursors": { + "before": "MAZDZD", + "after": "NzIZD" + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/videos.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/videos.json new file mode 100644 index 000000000000..b939df637bbc --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/resource/http/response/videos.json @@ -0,0 +1,111 @@ +{ + "data": [ + { + "id": "925443935443492", + "content_category": "OTHER", + "created_time": "2024-02-01T21:40:03+0000", + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"1080\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F925443935443492\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=1080\" style=\"border:none;overflow:hidden\" width=\"1080\">\u003C/iframe>", + "embeddable": true, + "format": [ + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"130\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F925443935443492\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=130\" style=\"border:none;overflow:hidden\" width=\"130\">\u003C/iframe>", + "filter": "130x130", + "height": 130, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/423069644_877218477743074_5429676119391867799_n.jpg?stp=dst-jpg_p130x130&_nc_cat=100&ccb=1-7&_nc_sid=1a7029&_nc_ohc=NA4oxYdIA_cAX-0Bvx_&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfBwNsaOjVl78jAV6jaMdmkBp54ZV6stgb8BBf5spWzRVw&oe=65CA7A97", + "width": 130 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"480\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F925443935443492\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=480\" style=\"border:none;overflow:hidden\" width=\"480\">\u003C/iframe>", + "filter": "480x480", + "height": 480, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/423069644_877218477743074_5429676119391867799_n.jpg?stp=dst-jpg_p480x480&_nc_cat=100&ccb=1-7&_nc_sid=1a7029&_nc_ohc=NA4oxYdIA_cAX-0Bvx_&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfCSSRmwbclnqnKAlmrAMK8Q8T6covE1azUyk-YQ7DGfAA&oe=65CA7A97", + "width": 480 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"720\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F925443935443492\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=720\" style=\"border:none;overflow:hidden\" width=\"720\">\u003C/iframe>", + "filter": "720x720", + "height": 720, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/423069644_877218477743074_5429676119391867799_n.jpg?stp=dst-jpg_p720x720&_nc_cat=100&ccb=1-7&_nc_sid=1a7029&_nc_ohc=NA4oxYdIA_cAX-0Bvx_&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfCiwt-R1NWb4qQaxws6AD0Sk1sp99cfyDzKdRPWII1ZRw&oe=65CA7A97", + "width": 720 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"1080\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F925443935443492\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=1080\" style=\"border:none;overflow:hidden\" width=\"1080\">\u003C/iframe>", + "filter": "native", + "height": 1080, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/423069644_877218477743074_5429676119391867799_n.jpg?stp=dst-jpg&_nc_cat=100&ccb=1-7&_nc_sid=1a7029&_nc_ohc=NA4oxYdIA_cAX-0Bvx_&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfDcziC4hL0U904kPv8GBKuhH4CP6gDqGagDX2xWVTvTnQ&oe=65CA7A97", + "width": 1080 + } + ], + "icon": "https://static.xx.fbcdn.net/rsrc.php/v3/yD/r/DggDhA4z4tO.gif", + "is_crosspost_video": false, + "is_crossposting_eligible": false, + "is_episode": false, + "is_instagram_eligible": true, + "length": 24.024, + "permalink_url": "/23854402720800548/videos/925443935443492/?idorvanity=23854402720800548", + "post_views": 0, + "published": true, + "source": "https://video.fiev6-1.fna.fbcdn.net/o1/v/t2/f1/m69/GFFjPxkyAKs6DSwMAENL3WReuCxVbmdjAAAF.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6Im9lcF9oZCJ9&_nc_ht=video.fiev6-1.fna.fbcdn.net&_nc_cat=101&strext=1&vs=3c5762ad873c98da&_nc_vs=HBksFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HRkZqUHhreUFLczZEU3dNQUVOTDNXUmV1Q3hWYm1kakFBQUYVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dFXzhOQmw4RGRxOXlGSUJBR3Z6MmtUWkdXUmJidjRHQUFBRhUCAsgBAEsHiBJwcm9ncmVzc2l2ZV9yZWNpcGUBMQ1zdWJzYW1wbGVfZnBzABB2bWFmX2VuYWJsZV9uc3ViACBtZWFzdXJlX29yaWdpbmFsX3Jlc29sdXRpb25fc3NpbQAoY29tcHV0ZV9zc2ltX29ubHlfYXRfb3JpZ2luYWxfcmVzb2x1dGlvbgAddXNlX2xhbmN6b3NfZm9yX3ZxbV91cHNjYWxpbmcAEWRpc2FibGVfcG9zdF9wdnFzABUAJQAcjBdAAAAAAAAAABERAAAAJsbx\u00252BO61t7kBFQIoAkMzGAt2dHNfcHJldmlldxwXQDgGJN0vGqAYIWRhc2hfZ2VuMmh3YmFzaWNfaHEyX2ZyYWdfMl92aWRlbxIAGBh2aWRlb3MudnRzLmNhbGxiYWNrLnByb2Q4ElZJREVPX1ZJRVdfUkVRVUVTVBsKiBVvZW1fdGFyZ2V0X2VuY29kZV90YWcGb2VwX2hkE29lbV9yZXF1ZXN0X3RpbWVfbXMBMAxvZW1fY2ZnX3J1bGUHdW5tdXRlZBNvZW1fcm9pX3JlYWNoX2NvdW50ATARb2VtX2lzX2V4cGVyaW1lbnQADG9lbV92aWRlb19pZA85MjU0NDM5MzU0NDM0OTISb2VtX3ZpZGVvX2Fzc2V0X2lkEDE3OTAxNjM5MDgxNjU3NjIVb2VtX3ZpZGVvX3Jlc291cmNlX2lkDzQwNzc3MTQyNDk1NTQ5MRxvZW1fc291cmNlX3ZpZGVvX2VuY29kaW5nX2lkDzM4NTQxMjEyNDE4MDIxNw52dHNfcmVxdWVzdF9pZAAlAhwAJY4CGweIAXMENjEyMQJjZAoyMDI0LTAyLTAxA2FwcANEREgCY3QGTEVHQUNZE29yaWdpbmFsX2R1cmF0aW9uX3MGMjQuMDY0AWYCYWQCdHMVcHJvZ3Jlc3NpdmVfZW5jb2RpbmdzAA\u00253D\u00253D&ccb=9-4&oh=00_AfA8BOCXei12URMU4g6BguZ4qCS2cTAwpMrLLW4g55t_Wg&oe=65C6D242&_nc_sid=1d576d&_nc_rid=530335006251005&_nc_store_type=1", + "updated_time": "2024-02-01T21:40:24+0000", + "views": 0 + }, + { + "id": "916735743507460", + "content_category": "OTHER", + "created_time": "2024-01-29T23:09:08+0000", + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"1080\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F916735743507460\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=1080\" style=\"border:none;overflow:hidden\" width=\"1080\">\u003C/iframe>", + "embeddable": true, + "format": [ + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"130\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F916735743507460\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=130\" style=\"border:none;overflow:hidden\" width=\"130\">\u003C/iframe>", + "filter": "130x130", + "height": 130, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/421466838_710585507916561_3872490001217946972_n.jpg?stp=dst-jpg_p130x130&_nc_cat=102&ccb=1-7&_nc_sid=1a7029&_nc_ohc=Bf3lDDsxlp8AX8WEBcN&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfCr0UPnWf53Mhxnn45ZBNRJ5C6jbMcNRpeg5jQwDvjgiQ&oe=65C8F32D", + "width": 130 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"480\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F916735743507460\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=480\" style=\"border:none;overflow:hidden\" width=\"480\">\u003C/iframe>", + "filter": "480x480", + "height": 480, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/421466838_710585507916561_3872490001217946972_n.jpg?stp=dst-jpg_p480x480&_nc_cat=102&ccb=1-7&_nc_sid=1a7029&_nc_ohc=Bf3lDDsxlp8AX8WEBcN&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfBfyCdBovsAE5hSpP9K3ADKnJJAHjtEIebTMn0F4vH2eA&oe=65C8F32D", + "width": 480 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"720\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F916735743507460\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=720\" style=\"border:none;overflow:hidden\" width=\"720\">\u003C/iframe>", + "filter": "720x720", + "height": 720, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/421466838_710585507916561_3872490001217946972_n.jpg?stp=dst-jpg_p720x720&_nc_cat=102&ccb=1-7&_nc_sid=1a7029&_nc_ohc=Bf3lDDsxlp8AX8WEBcN&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfBr2pAfGPEdFO66MpPC-NdJX3ANHWKXXdMzZzcN0cfp-g&oe=65C8F32D", + "width": 720 + }, + { + "embed_html": "\u003Ciframe allow=\"autoplay; clipboard-write; encrypted-media; picture-in-picture; web-share\" allowfullscreen=\"true\" frameborder=\"0\" height=\"1080\" scrolling=\"no\" src=\"https://www.facebook.com/plugins/video.php?href=https\u00253A\u00252F\u00252Fwww.facebook.com\u00252F23854402720800548\u00252Fvideos\u00252F916735743507460\u00252F\u00253Fidorvanity\u00253D23854402720800548&width=1080\" style=\"border:none;overflow:hidden\" width=\"1080\">\u003C/iframe>", + "filter": "native", + "height": 1080, + "picture": "https://scontent.fiev6-1.fna.fbcdn.net/v/t15.13418-10/421466838_710585507916561_3872490001217946972_n.jpg?stp=dst-jpg&_nc_cat=102&ccb=1-7&_nc_sid=1a7029&_nc_ohc=Bf3lDDsxlp8AX8WEBcN&_nc_ht=scontent.fiev6-1.fna&edm=AM_bLsMEAAAA&oh=00_AfBYXRKMdH0ZQ8QLOycwQh74ipObGO1IhrntUoNxAwwGPw&oe=65C8F32D", + "width": 1080 + } + ], + "icon": "https://static.xx.fbcdn.net/rsrc.php/v3/yD/r/DggDhA4z4tO.gif", + "is_crosspost_video": false, + "is_crossposting_eligible": false, + "is_episode": false, + "is_instagram_eligible": true, + "length": 24.024, + "permalink_url": "/23854402720800548/videos/916735743507460/?idorvanity=23854402720800548", + "post_views": 0, + "published": true, + "source": "https://video.fiev6-1.fna.fbcdn.net/o1/v/t2/f1/m69/GJwhThkQRrIkM7QCAEDVozMuFWg8bmdjAAAF.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6Im9lcF9oZCJ9&_nc_ht=video.fiev6-1.fna.fbcdn.net&_nc_cat=103&strext=1&vs=9578d47bdeedff3e&_nc_vs=HBksFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HSndoVGhrUVJySWtNN1FDQUVEVm96TXVGV2c4Ym1kakFBQUYVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dLVVdKeG15a183ODZZc0NBRWdXRFFQRGk1TlNidjRHQUFBRhUCAsgBAEsHiBJwcm9ncmVzc2l2ZV9yZWNpcGUBMQ1zdWJzYW1wbGVfZnBzABB2bWFmX2VuYWJsZV9uc3ViACBtZWFzdXJlX29yaWdpbmFsX3Jlc29sdXRpb25fc3NpbQAoY29tcHV0ZV9zc2ltX29ubHlfYXRfb3JpZ2luYWxfcmVzb2x1dGlvbgAddXNlX2xhbmN6b3NfZm9yX3ZxbV91cHNjYWxpbmcAEWRpc2FibGVfcG9zdF9wdnFzABUAJQAcjBdAAAAAAAAAABERAAAAJqal7crZj6oBFQIoAkMzGAt2dHNfcHJldmlldxwXQDgGJN0vGqAYIWRhc2hfZ2VuMmh3YmFzaWNfaHEyX2ZyYWdfMl92aWRlbxIAGBh2aWRlb3MudnRzLmNhbGxiYWNrLnByb2Q4ElZJREVPX1ZJRVdfUkVRVUVTVBsKiBVvZW1fdGFyZ2V0X2VuY29kZV90YWcGb2VwX2hkE29lbV9yZXF1ZXN0X3RpbWVfbXMBMAxvZW1fY2ZnX3J1bGUHdW5tdXRlZBNvZW1fcm9pX3JlYWNoX2NvdW50ATARb2VtX2lzX2V4cGVyaW1lbnQADG9lbV92aWRlb19pZA85MTY3MzU3NDM1MDc0NjASb2VtX3ZpZGVvX2Fzc2V0X2lkDzM3NzMyNzM0ODI2NDE5NxVvZW1fdmlkZW9fcmVzb3VyY2VfaWQPMzc0MTAzNjc1MzQ5MzMxHG9lbV9zb3VyY2VfdmlkZW9fZW5jb2RpbmdfaWQQMTExMTczNTcwOTg0MDIwNA52dHNfcmVxdWVzdF9pZAAlAhwAJY4CGweIAXMEODc4MwJjZAoyMDI0LTAxLTI5A2FwcANEREgCY3QGTEVHQUNZE29yaWdpbmFsX2R1cmF0aW9uX3MGMjQuMDY0AWYCYWQCdHMVcHJvZ3Jlc3NpdmVfZW5jb2RpbmdzAA\u00253D\u00253D&ccb=9-4&oh=00_AfC-KNsd9MXPG2i2rHKk40YOhDGQY0stsUS5wvZrwKAXoA&oe=65C6A407&_nc_sid=1d576d&_nc_rid=776303965488972&_nc_store_type=1", + "updated_time": "2024-01-29T23:09:30+0000", + "views": 0 + } + ], + "paging": { + "cursors": { + "before": "QVFIUlJBUTRfNU5HQUJ4c1V", + "after": "QVFIUlhOX3Rnbm5YNmxOWjBC" + } + }, + "summary": { + "total_count": 2 + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py index 3bc8a37c2db8..d8aae90765b7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py @@ -14,7 +14,9 @@ class TestMyFacebookAdsApi: @pytest.fixture def fb_api(self): - return source_facebook_marketing.api.MyFacebookAdsApi.init(access_token="foo", crash_log=False) + return source_facebook_marketing.api.MyFacebookAdsApi.init( + access_token="foo", crash_log=False + ) @pytest.mark.parametrize( "max_rate,max_pause_interval,min_pause_interval,usage,pause_interval,expected_pause_interval", @@ -46,7 +48,15 @@ def fb_api(self): ], ) def test__compute_pause_interval( - self, mocker, fb_api, max_rate, max_pause_interval, min_pause_interval, usage, pause_interval, expected_pause_interval + self, + mocker, + fb_api, + max_rate, + max_pause_interval, + min_pause_interval, + usage, + pause_interval, + expected_pause_interval, ): mocker.patch.object(fb_api, "MAX_RATE", max_rate) mocker.patch.object(fb_api, "MAX_PAUSE_INTERVAL", max_pause_interval) @@ -59,12 +69,18 @@ def test__compute_pause_interval( [ ( pendulum.duration(minutes=1), # min_pause_interval - [(5, pendulum.duration(minutes=6)), (7, pendulum.duration(minutes=5))], # usages_pause_intervals + [ + (5, pendulum.duration(minutes=6)), + (7, pendulum.duration(minutes=5)), + ], # usages_pause_intervals (7, pendulum.duration(minutes=6)), # expected_output ), ( pendulum.duration(minutes=10), # min_pause_interval - [(5, pendulum.duration(minutes=6)), (7, pendulum.duration(minutes=5))], # usages_pause_intervals + [ + (5, pendulum.duration(minutes=6)), + (7, pendulum.duration(minutes=5)), + ], # usages_pause_intervals (7, pendulum.duration(minutes=10)), # expected_output ), ( @@ -85,19 +101,36 @@ def test__compute_pause_interval( ), ], ) - def test__get_max_usage_pause_interval_from_batch(self, mocker, fb_api, min_pause_interval, usages_pause_intervals, expected_output): + def test__get_max_usage_pause_interval_from_batch( + self, + mocker, + fb_api, + min_pause_interval, + usages_pause_intervals, + expected_output, + ): records = [ - {"headers": [{"name": "USAGE", "value": usage}, {"name": "PAUSE_INTERVAL", "value": pause_interval}]} + { + "headers": [ + {"name": "USAGE", "value": usage}, + {"name": "PAUSE_INTERVAL", "value": pause_interval}, + ] + } for usage, pause_interval in usages_pause_intervals ] mock_parse_call_rate_header = mocker.Mock(side_effect=usages_pause_intervals) - mocker.patch.object(fb_api, "_parse_call_rate_header", mock_parse_call_rate_header) + mocker.patch.object( + fb_api, "_parse_call_rate_header", mock_parse_call_rate_header + ) mocker.patch.object(fb_api, "MIN_PAUSE_INTERVAL", min_pause_interval) output = fb_api._get_max_usage_pause_interval_from_batch(records) fb_api._parse_call_rate_header.assert_called_with( - {"usage": usages_pause_intervals[-1][0], "pause_interval": usages_pause_intervals[-1][1]} + { + "usage": usages_pause_intervals[-1][0], + "pause_interval": usages_pause_intervals[-1][1], + } ) assert output == expected_output @@ -112,30 +145,52 @@ def test__get_max_usage_pause_interval_from_batch(self, mocker, fb_api, min_paus (["not_batch"], 2, 1, False), ], ) - def test__handle_call_rate_limit(self, mocker, fb_api, params, min_rate, usage, expect_sleep): + def test__handle_call_rate_limit( + self, mocker, fb_api, params, min_rate, usage, expect_sleep + ): pause_interval = 1 mock_response = mocker.Mock() mocker.patch.object(fb_api, "MIN_RATE", min_rate) - mocker.patch.object(fb_api, "_get_max_usage_pause_interval_from_batch", mocker.Mock(return_value=(usage, pause_interval))) - mocker.patch.object(fb_api, "_parse_call_rate_header", mocker.Mock(return_value=(usage, pause_interval))) + mocker.patch.object( + fb_api, + "_get_max_usage_pause_interval_from_batch", + mocker.Mock(return_value=(usage, pause_interval)), + ) + mocker.patch.object( + fb_api, + "_parse_call_rate_header", + mocker.Mock(return_value=(usage, pause_interval)), + ) mocker.patch.object(fb_api, "_compute_pause_interval") mocker.patch.object(source_facebook_marketing.api, "logger") mocker.patch.object(source_facebook_marketing.api, "sleep") assert fb_api._handle_call_rate_limit(mock_response, params) is None if "batch" in params: - fb_api._get_max_usage_pause_interval_from_batch.assert_called_with(mock_response.json.return_value) + fb_api._get_max_usage_pause_interval_from_batch.assert_called_with( + mock_response.json.return_value + ) else: - fb_api._parse_call_rate_header.assert_called_with(mock_response.headers.return_value) + fb_api._parse_call_rate_header.assert_called_with( + mock_response.headers.return_value + ) if expect_sleep: - fb_api._compute_pause_interval.assert_called_with(usage=usage, pause_interval=pause_interval) - source_facebook_marketing.api.sleep.assert_called_with(fb_api._compute_pause_interval.return_value.total_seconds()) + fb_api._compute_pause_interval.assert_called_with( + usage=usage, pause_interval=pause_interval + ) + source_facebook_marketing.api.sleep.assert_called_with( + fb_api._compute_pause_interval.return_value.total_seconds() + ) source_facebook_marketing.api.logger.warning.assert_called_with( f"Utilization is too high ({usage})%, pausing for {fb_api._compute_pause_interval.return_value}" ) def test_find_account(self, api, account_id, requests_mock): - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", [{"json": {"id": "act_test"}}]) - account = api._find_account(account_id) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", + [{"json": {"id": "act_test"}}], + ) + account = api.get_account(account_id) assert isinstance(account, AdAccount) assert account.get_id() == "act_test" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index 55073068ce1d..1c4bb0f67c37 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -49,12 +49,27 @@ def job_fixture(api, account): } interval = pendulum.Period(pendulum.Date(2019, 1, 1), pendulum.Date(2019, 1, 1)) - return InsightAsyncJob(edge_object=account, api=api, interval=interval, params=params) + return InsightAsyncJob( + edge_object=account, + api=api, + interval=interval, + params=params, + job_timeout=pendulum.duration(minutes=60), + ) @pytest.fixture(name="grouped_jobs") def grouped_jobs_fixture(mocker): - return [mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False, elapsed_time=None) for _ in range(10)] + return [ + mocker.Mock( + spec=InsightAsyncJob, + attempt_number=1, + failed=False, + completed=False, + elapsed_time=None, + ) + for _ in range(10) + ] @pytest.fixture(name="parent_job") @@ -170,7 +185,10 @@ def test_start(self, job): def test_start_already_started(self, job): job.start() - with pytest.raises(RuntimeError, match=r": Incorrect usage of start - the job already started, use restart instead"): + with pytest.raises( + RuntimeError, + match=r": Incorrect usage of start - the job already started, use restart instead", + ): job.start() def test_restart(self, failed_job, api, adreport): @@ -185,15 +203,24 @@ def test_restart_when_job_not_failed(self, job, api): job.start() assert not job.failed - with pytest.raises(RuntimeError, match=r": Incorrect usage of restart - only failed jobs can be restarted"): + with pytest.raises( + RuntimeError, + match=r": Incorrect usage of restart - only failed jobs can be restarted", + ): job.restart() def test_restart_when_job_not_started(self, job): - with pytest.raises(RuntimeError, match=r": Incorrect usage of restart - only failed jobs can be restarted"): + with pytest.raises( + RuntimeError, + match=r": Incorrect usage of restart - only failed jobs can be restarted", + ): job.restart() def test_update_job_not_started(self, job): - with pytest.raises(RuntimeError, match=r": Incorrect usage of the method - the job is not started"): + with pytest.raises( + RuntimeError, + match=r": Incorrect usage of the method - the job is not started", + ): job.update_job() def test_update_job_on_completed_job(self, completed_job, adreport): @@ -207,7 +234,7 @@ def test_update_job(self, started_job, adreport): adreport.api_get.assert_called_once() def test_update_job_expired(self, started_job, adreport, mocker): - mocker.patch.object(started_job, "job_timeout", new=pendulum.Duration()) + mocker.patch.object(started_job, "_job_timeout", new=pendulum.Duration()) started_job.update_job() assert started_job.failed @@ -240,7 +267,9 @@ def test_update_job_with_batch(self, started_job, adreport, mocker): kwargs["failure"](response) def test_elapsed_time(self, job, api, adreport): - assert job.elapsed_time is None, "should be None for the job that is not started" + assert ( + job.elapsed_time is None + ), "should be None for the job that is not started" job.start() adreport["async_status"] = Status.COMPLETED.value @@ -285,9 +314,13 @@ def test_str(self, api, account): api=api, params={"breakdowns": [10, 20]}, interval=interval, + job_timeout=pendulum.duration(minutes=60), ) - assert str(job) == f"InsightAsyncJob(id=, {account}, time_range= 2011-01-01]>, breakdowns=[10, 20])" + assert ( + str(job) + == f"InsightAsyncJob(id=, {account}, time_range= 2011-01-01]>, breakdowns=[10, 20])" + ) def test_get_result(self, job, adreport, api): job.start() @@ -308,17 +341,26 @@ def test_get_result_retried(self, mocker, job, api): ads_insights._set_data({"items": [{"some_data": 123}, {"some_data": 77}]}) with mocker.patch( "facebook_business.adobjects.objectparser.ObjectParser.parse_multiple", - side_effect=[FacebookBadObjectError("Bad data to set object data"), ads_insights], + side_effect=[ + FacebookBadObjectError("Bad data to set object data"), + ads_insights, + ], ): # in case this is not retried, an error will be raised job.get_result() def test_get_result_when_job_is_not_started(self, job): - with pytest.raises(RuntimeError, match=r"Incorrect usage of get_result - the job is not started or failed"): + with pytest.raises( + RuntimeError, + match=r"Incorrect usage of get_result - the job is not started or failed", + ): job.get_result() def test_get_result_when_job_is_failed(self, failed_job): - with pytest.raises(RuntimeError, match=r"Incorrect usage of get_result - the job is not started or failed"): + with pytest.raises( + RuntimeError, + match=r"Incorrect usage of get_result - the job is not started or failed", + ): failed_job.get_result() @pytest.mark.parametrize( @@ -333,10 +375,22 @@ def test_get_result_when_job_is_failed(self, failed_job): def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): """Test that split will correctly downsize edge_object""" today = pendulum.today().date() - start, end = today - pendulum.duration(days=365 * 3 + 20), today - pendulum.duration(days=365 * 3 + 10) + start, end = today - pendulum.duration( + days=365 * 3 + 20 + ), today - pendulum.duration(days=365 * 3 + 10) params = {"time_increment": 1, "breakdowns": []} - job = InsightAsyncJob(api=api, edge_object=edge_class(1), interval=pendulum.Period(start, end), params=params) - mocker.patch.object(edge_class, "get_insights", return_value=[{id_field: 1}, {id_field: 2}, {id_field: 3}]) + job = InsightAsyncJob( + api=api, + edge_object=edge_class(1), + interval=pendulum.Period(start, end), + params=params, + job_timeout=pendulum.duration(minutes=60), + ) + mocker.patch.object( + edge_class, + "get_insights", + return_value=[{id_field: 1}, {id_field: 2}, {id_field: 3}], + ) small_jobs = job.split_job() @@ -350,7 +404,9 @@ def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): # with the one 37 months ago, that's why current date is frozen. # For a different date the since date would be also different. # See facebook_marketing.utils.validate_start_date for reference - "since": (today - pendulum.duration(months=37) + pendulum.duration(days=1)).to_date_string(), + "since": ( + today - pendulum.duration(months=37) + pendulum.duration(days=1) + ).to_date_string(), "until": end.to_date_string(), }, } @@ -359,15 +415,28 @@ def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): assert all(j.interval == job.interval for j in small_jobs) for i, small_job in enumerate(small_jobs, start=1): assert small_job._params["time_range"] == job._params["time_range"] - assert str(small_job) == f"InsightAsyncJob(id=, {next_edge_class(i)}, time_range={job.interval}, breakdowns={[]})" + assert ( + str(small_job) + == f"InsightAsyncJob(id=, {next_edge_class(i)}, time_range={job.interval}, breakdowns={[]})" + ) def test_split_job_smallest(self, mocker, api): """Test that split will correctly downsize edge_object""" - interval = pendulum.Period(pendulum.Date(2010, 1, 1), pendulum.Date(2010, 1, 10)) + interval = pendulum.Period( + pendulum.Date(2010, 1, 1), pendulum.Date(2010, 1, 10) + ) params = {"time_increment": 1, "breakdowns": []} - job = InsightAsyncJob(api=api, edge_object=Ad(1), interval=interval, params=params) + job = InsightAsyncJob( + api=api, + edge_object=Ad(1), + interval=interval, + params=params, + job_timeout=pendulum.duration(minutes=60), + ) - with pytest.raises(ValueError, match="The job is already splitted to the smallest size."): + with pytest.raises( + ValueError, match="The job is already splitted to the smallest size." + ): job.split_job() @@ -429,7 +498,10 @@ def test_get_result(self, parent_job, grouped_jobs): def test_split_job(self, parent_job, grouped_jobs, mocker): grouped_jobs[0].failed = True - grouped_jobs[0].split_job.return_value = [mocker.Mock(spec=InsightAsyncJob), mocker.Mock(spec=InsightAsyncJob)] + grouped_jobs[0].split_job.return_value = [ + mocker.Mock(spec=InsightAsyncJob), + mocker.Mock(spec=InsightAsyncJob), + ] grouped_jobs[5].failed = True grouped_jobs[5].split_job.return_value = [ mocker.Mock(spec=InsightAsyncJob), @@ -439,7 +511,9 @@ def test_split_job(self, parent_job, grouped_jobs, mocker): small_jobs = parent_job.split_job() - assert len(small_jobs) == len(grouped_jobs) + 5 - 2, "each failed job must be replaced with its split" + assert ( + len(small_jobs) == len(grouped_jobs) + 5 - 2 + ), "each failed job must be replaced with its split" for i, job in enumerate(grouped_jobs): if i in (0, 5): job.split_job.assert_called_once() @@ -461,4 +535,7 @@ def test_split_job_smallest(self, parent_job, grouped_jobs): count += 1 def test_str(self, parent_job, grouped_jobs): - assert str(parent_job) == f"ParentAsyncJob({grouped_jobs[0]} ... {len(grouped_jobs) - 1} jobs more)" + assert ( + str(parent_job) + == f"ParentAsyncJob({grouped_jobs[0]} ... {len(grouped_jobs) - 1} jobs more)" + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py index a9234fc31465..77d38e96a19f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py @@ -25,28 +25,34 @@ def time_mock_fixture(mocker): @pytest.fixture(name="update_job_mock") def update_job_mock_fixture(mocker): - return mocker.patch("source_facebook_marketing.streams.async_job_manager.update_in_batch") + return mocker.patch( + "source_facebook_marketing.streams.async_job_manager.update_in_batch" + ) class TestInsightAsyncManager: - def test_jobs_empty(self, api): + def test_jobs_empty(self, api, some_config): """Should work event without jobs""" - manager = InsightAsyncJobManager(api=api, jobs=[]) + manager = InsightAsyncJobManager( + api=api, jobs=[], account_id=some_config["account_ids"][0] + ) jobs = list(manager.completed_jobs()) assert not jobs - def test_jobs_completed_immediately(self, api, mocker, time_mock): + def test_jobs_completed_immediately(self, api, mocker, time_mock, some_config): """Manager should emmit jobs without waiting if they completed""" jobs = [ mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) completed_jobs = list(manager.completed_jobs()) assert jobs == completed_jobs time_mock.sleep.assert_not_called() - def test_jobs_wait(self, api, mocker, time_mock, update_job_mock): + def test_jobs_wait(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should return completed jobs and wait for others""" def update_job_behaviour(): @@ -58,10 +64,16 @@ def update_job_behaviour(): update_job_mock.side_effect = update_job_behaviour() jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) job = next(manager.completed_jobs(), None) assert job == jobs[1] @@ -69,12 +81,14 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job == jobs[0] - time_mock.sleep.assert_called_with(InsightAsyncJobManager.JOB_STATUS_UPDATE_SLEEP_SECONDS) + time_mock.sleep.assert_called_with( + InsightAsyncJobManager.JOB_STATUS_UPDATE_SLEEP_SECONDS + ) job = next(manager.completed_jobs(), None) assert job is None - def test_job_restarted(self, api, mocker, time_mock, update_job_mock): + def test_job_restarted(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should restart failed jobs""" def update_job_behaviour(): @@ -86,10 +100,16 @@ def update_job_behaviour(): update_job_mock.side_effect = update_job_behaviour() jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -101,7 +121,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_split(self, api, mocker, time_mock, update_job_mock): + def test_job_split(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should split failed jobs when they fail second time""" def update_job_behaviour(): @@ -111,17 +131,27 @@ def update_job_behaviour(): update_job_mock.side_effect = update_job_behaviour() jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), ] sub_jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), ] sub_jobs[0].get_result.return_value = [1, 2] sub_jobs[1].get_result.return_value = [3, 4] jobs[1].split_job.return_value = sub_jobs - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -134,7 +164,9 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_job_failed_too_many_times( + self, api, mocker, time_mock, update_job_mock, some_config + ): """Manager should fail when job failed too many times""" def update_job_behaviour(): @@ -144,15 +176,26 @@ def update_job_behaviour(): update_job_mock.side_effect = update_job_behaviour() jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) - - with pytest.raises(JobException, match=f"{jobs[1]}: failed more than {InsightAsyncJobManager.MAX_NUMBER_OF_ATTEMPTS} times."): + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) + + with pytest.raises( + JobException, + match=f"{jobs[1]}: failed more than {InsightAsyncJobManager.MAX_NUMBER_OF_ATTEMPTS} times.", + ): next(manager.completed_jobs(), None) - def test_nested_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_nested_job_failed_too_many_times( + self, api, mocker, time_mock, update_job_mock, some_config + ): """Manager should fail when a nested job within a ParentAsyncJob failed too many times""" def update_job_behaviour(): @@ -163,14 +206,28 @@ def update_job_behaviour(): update_job_mock.side_effect = update_job_behaviour() sub_jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False + ), ] jobs = [ - mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), - mocker.Mock(spec=ParentAsyncJob, _jobs=sub_jobs, attempt_number=1, failed=False, completed=False), + mocker.Mock( + spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True + ), + mocker.Mock( + spec=ParentAsyncJob, + _jobs=sub_jobs, + attempt_number=1, + failed=False, + completed=False, + ), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager( + api=api, jobs=jobs, account_id=some_config["account_ids"][0] + ) with pytest.raises(JobException): next(manager.completed_jobs(), None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 6f98004bcfce..612ed22ef25a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -36,30 +36,45 @@ def start_date_fixture(): @pytest.fixture(name="async_manager_mock") def async_manager_mock_fixture(mocker): - mock = mocker.patch("source_facebook_marketing.streams.base_insight_streams.InsightAsyncJobManager") + mock = mocker.patch( + "source_facebook_marketing.streams.base_insight_streams.InsightAsyncJobManager" + ) mock.return_value = mock return mock @pytest.fixture(name="async_job_mock") def async_job_mock_fixture(mocker): - mock = mocker.patch("source_facebook_marketing.streams.base_insight_streams.InsightAsyncJob") + mock = mocker.patch( + "source_facebook_marketing.streams.base_insight_streams.InsightAsyncJob" + ) mock.side_effect = lambda api, **kwargs: {"api": api, **kwargs} class TestBaseInsightsStream: - def test_init(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_init(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) assert not stream.breakdowns - assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] + assert stream.action_breakdowns == [ + "action_type", + "action_target_id", + "action_destination", + ] assert stream.name == "ads_insights" assert stream.primary_key == ["date_start", "account_id", "ad_id"] assert stream.action_report_time == "mixed" - def test_init_override(self, api): + def test_init_override(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), name="CustomName", @@ -71,18 +86,27 @@ def test_init_override(self, api): assert stream.breakdowns == ["test1", "test2"] assert stream.action_breakdowns == ["field1", "field2"] assert stream.name == "custom_name" - assert stream.primary_key == ["date_start", "account_id", "ad_id", "test1", "test2"] - - def test_read_records_all(self, mocker, api): + assert stream.primary_key == [ + "date_start", + "account_id", + "ad_id", + "test1", + "test2", + ] + + def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 """ job = mocker.Mock(spec=InsightAsyncJob) job.get_result.return_value = [mocker.Mock(), mocker.Mock(), mocker.Mock()] - job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) + job.interval = pendulum.Period( + pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1) + ) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, @@ -91,76 +115,179 @@ def test_read_records_all(self, mocker, api): records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={ + "insight_job": job, + "account_id": some_config["account_ids"][0], + }, ) ) assert len(records) == 3 - def test_read_records_random_order(self, mocker, api): + def test_read_records_random_order(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 """ job = mocker.Mock(spec=AsyncJob) job.get_result.return_value = [mocker.Mock(), mocker.Mock(), mocker.Mock()] - job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + job.interval = pendulum.Period( + pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1) + ) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={ + "insight_job": job, + "account_id": some_config["account_ids"][0], + }, ) ) assert len(records) == 3 @pytest.mark.parametrize( - "state", + "state,result_state", [ - { - AdsInsights.cursor_field: "2010-10-03", - "slices": [ - "2010-01-01", - "2010-01-02", - ], - "time_increment": 1, - }, - { - AdsInsights.cursor_field: "2010-10-03", - }, - { - "slices": [ - "2010-01-01", - "2010-01-02", - ] - }, + # Old format + ( + { + AdsInsights.cursor_field: "2010-10-03", + "slices": [ + "2010-01-01", + "2010-01-02", + ], + "time_increment": 1, + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + ), + ( + { + AdsInsights.cursor_field: "2010-10-03", + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + ), + ( + { + "slices": [ + "2010-01-01", + "2010-01-02", + ] + }, + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + ), + # New format - nested with account_id + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + None, + ), + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + None, + ), + ( + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + None, + ), ], ) - def test_state(self, api, state): + def test_state(self, api, state, result_state, some_config): """State setter/getter should work with all combinations""" - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) - assert stream.state == {} + assert stream.state == { + "time_increment": 1, + "unknown_account": {"slices": set()}, + } stream.state = state actual_state = stream.state - actual_state["slices"] = sorted(actual_state.get("slices", [])) - state["slices"] = sorted(state.get("slices", [])) - state["time_increment"] = 1 - assert actual_state == state + result_state = state if not result_state else result_state + result_state[some_config["account_ids"][0]]["slices"] = result_state[ + some_config["account_ids"][0] + ].get("slices", set()) + result_state["time_increment"] = 1 + + assert actual_state == result_state - def test_stream_slices_no_state(self, api, async_manager_mock, start_date): + def test_stream_slices_no_state( + self, api, async_manager_mock, start_date, some_config + ): """Stream will use start_date when there is not state""" end_date = start_date + duration(weeks=2) - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=start_date, + end_date=end_date, + insights_lookback_window=28, + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] - slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) + slices = list( + stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental) + ) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -168,16 +295,30 @@ def test_stream_slices_no_state(self, api, async_manager_mock, start_date): assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_no_state_close_to_now( + self, api, async_manager_mock, recent_start_date, some_config + ): """Stream will use start_date when there is not state and start_date within 28d from now""" start_date = recent_start_date end_date = pendulum.now() - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=start_date, + end_date=end_date, + insights_lookback_window=28, + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] - slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) + slices = list( + stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental) + ) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -185,36 +326,68 @@ def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, rece assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state(self, api, async_manager_mock, start_date): + def test_stream_slices_with_state( + self, api, async_manager_mock, start_date, some_config + ): """Stream will use cursor_value from state when there is state""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=start_date, + end_date=end_date, + insights_lookback_window=28, + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] - slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) + slices = list( + stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental) + ) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) assert len(generated_jobs) == (end_date - cursor_value).days - assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=1) - assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=2) + assert generated_jobs[0].interval.start == cursor_value.date() + duration( + days=1 + ) + assert generated_jobs[1].interval.start == cursor_value.date() + duration( + days=2 + ) - def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_with_state_close_to_now( + self, api, async_manager_mock, recent_start_date, some_config + ): """Stream will use start_date when close to now and start_date close to now""" start_date = recent_start_date end_date = pendulum.now() cursor_value = end_date - duration(days=1) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=start_date, + end_date=end_date, + insights_lookback_window=28, + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] - slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) + slices = list( + stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental) + ) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -222,39 +395,84 @@ def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, re assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date): + @pytest.mark.parametrize("state_format", ["old_format", "new_format"]) + def test_stream_slices_with_state_and_slices( + self, api, async_manager_mock, start_date, some_config, state_format + ): """Stream will use cursor_value from state, but will skip saved slices""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) - state = { - AdsInsights.cursor_field: cursor_value.date().isoformat(), - "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], - } - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + + if state_format == "old_format": + state = { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [ + (cursor_value + duration(days=1)).date().isoformat(), + (cursor_value + duration(days=3)).date().isoformat(), + ], + } + else: + state = { + "unknown_account": { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [ + (cursor_value + duration(days=1)).date().isoformat(), + (cursor_value + duration(days=3)).date().isoformat(), + ], + } + } + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=start_date, + end_date=end_date, + insights_lookback_window=28, + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] - slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) + slices = list( + stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental) + ) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) - assert len(generated_jobs) == (end_date - cursor_value).days - 2, "should be 2 slices short because of state" - assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=2) - assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=4) + assert ( + len(generated_jobs) == (end_date - cursor_value).days - 2 + ), "should be 2 slices short because of state" + assert generated_jobs[0].interval.start == cursor_value.date() + duration( + days=2 + ) + assert generated_jobs[1].interval.start == cursor_value.date() + duration( + days=4 + ) - def test_get_json_schema(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_get_json_schema(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) schema = stream.get_json_schema() assert "device_platform" not in schema["properties"] assert "country" not in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not ( + set(stream.fields()) - set(schema["properties"].keys()) + ), "all fields present in schema" - def test_get_json_schema_custom(self, api): + def test_get_json_schema_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["device_platform", "country"], @@ -265,38 +483,51 @@ def test_get_json_schema_custom(self, api): assert "device_platform" in schema["properties"] assert "country" in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not ( + set(stream.fields()) - set(schema["properties"].keys()) + ), "all fields present in schema" - def test_fields(self, api): + def test_fields(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, ) - fields = stream.fields + fields = stream.fields() assert "account_id" in fields assert "account_currency" in fields assert "actions" in fields - def test_fields_custom(self, api): + def test_fields_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) - assert stream.fields == ["account_id", "account_currency"] + assert stream.fields() == ["account_id", "account_currency"] schema = stream.get_json_schema() - assert schema["properties"].keys() == set(["account_currency", "account_id", stream.cursor_field, "date_stop", "ad_id"]) + assert schema["properties"].keys() == set( + [ + "account_currency", + "account_id", + stream.cursor_field, + "date_stop", + "ad_id", + ] + ) - def test_level_custom(self, api): + def test_level_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], @@ -306,9 +537,10 @@ def test_level_custom(self, api): assert stream.level == "adset" - def test_breackdowns_fields_present_in_response_data(self, api): + def test_breackdowns_fields_present_in_response_data(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["age", "gender"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py index 1f035c5c878e..4ddd72eab91d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py @@ -9,12 +9,16 @@ from facebook_business import FacebookSession from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch from source_facebook_marketing.api import MyFacebookAdsApi -from source_facebook_marketing.streams.base_streams import FBMarketingStream +from source_facebook_marketing.streams.base_streams import FBMarketingIncrementalStream, FBMarketingStream @pytest.fixture(name="mock_batch_responses") def mock_batch_responses_fixture(requests_mock): - return partial(requests_mock.register_uri, "POST", f"{FacebookSession.GRAPH}/{FacebookAdsApi.API_VERSION}/") + return partial( + requests_mock.register_uri, + "POST", + f"{FacebookSession.GRAPH}/{FacebookAdsApi.API_VERSION}/", + ) @pytest.fixture(name="batch") @@ -96,3 +100,177 @@ def test_date_time_value(self): } }, } == record + + +class ConcreteFBMarketingIncrementalStream(FBMarketingIncrementalStream): + cursor_field = "date" + valid_statuses = ["ACTIVE", "PAUSED", "DELETED"] + + def list_objects(self, **kwargs): + return [] + + +@pytest.fixture +def incremental_class_instance(api): + return ConcreteFBMarketingIncrementalStream( + api=api, account_ids=["123", "456", "789"], start_date=None, end_date=None + ) + + +class TestFBMarketingIncrementalStreamSliceAndState: + def test_stream_slices_multiple_accounts_with_state( + self, incremental_class_instance + ): + stream_state = { + "123": {"state_key": "state_value"}, + "456": {"state_key": "another_state_value"}, + } + expected_slices = [ + {"account_id": "123", "stream_state": {"state_key": "state_value"}}, + {"account_id": "456", "stream_state": {"state_key": "another_state_value"}}, + {"account_id": "789", "stream_state": {}}, + ] + assert ( + list(incremental_class_instance.stream_slices(stream_state)) + == expected_slices + ) + + def test_stream_slices_multiple_accounts_empty_state( + self, incremental_class_instance + ): + expected_slices = [ + {"account_id": "123", "stream_state": {}}, + {"account_id": "456", "stream_state": {}}, + {"account_id": "789", "stream_state": {}}, + ] + assert list(incremental_class_instance.stream_slices()) == expected_slices + + def test_stream_slices_single_account_with_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + stream_state = {"state_key": "state_value"} + expected_slices = [{"account_id": "123", "stream_state": stream_state}] + assert ( + list(incremental_class_instance.stream_slices(stream_state)) + == expected_slices + ) + + def test_stream_slices_single_account_empty_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + expected_slices = [{"account_id": "123", "stream_state": None}] + assert list(incremental_class_instance.stream_slices()) == expected_slices + + @pytest.mark.parametrize( + "current_stream_state, latest_record, expected_state, instance_filter_statuses", + [ + # Test case 1: State date is used because fewer filters are used + ( + {"123": {"date": "2021-01-30T00:00:00+00:00", "include_deleted": True}, "include_deleted": True}, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2021-01-30T00:00:00+00:00", + "filter_statuses": ["ACTIVE"], + "include_deleted": True, + }, + "include_deleted": True, + }, + ["ACTIVE"], + ), + # Test case 2: State date is used because filter_statuses is the same as include_deleted + ( + {"123": {"date": "2021-01-30T00:00:00+00:00", "include_deleted": True}, "include_deleted": True}, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2021-01-30T00:00:00+00:00", + "filter_statuses": ["ACTIVE", "PAUSED", "DELETED"], + "include_deleted": True, + }, + "include_deleted": True, + }, + ["ACTIVE", "PAUSED", "DELETED"], + ), + # Test case 3: State date is used because filter_statuses is the same as include_deleted + ( + { + "123": { + "date": "2023-02-15T00:00:00+00:00", + "include_deleted": False, + } + }, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2023-02-15T00:00:00+00:00", + "filter_statuses": [], + "include_deleted": False, + } + }, + [], + ), + # Test case 4: State date is ignored because there are more filters in the new config + ( + { + "123": { + "date": "2023-02-15T00:00:00+00:00", + "include_deleted": False, + } + }, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2021-01-20T00:00:00+00:00", + "filter_statuses": ["ACTIVE", "PAUSED"], + "include_deleted": False, + } + }, + ["ACTIVE", "PAUSED"], + ), + # Test case 5: Mismatching filter_statuses with include_deleted false + ( + { + "123": { + "date": "2023-02-15T00:00:00+00:00", + "filter_statuses": ["PAUSED"], + "include_deleted": False, + } + }, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2021-01-20T00:00:00+00:00", + "filter_statuses": ["ACTIVE"], + "include_deleted": False, + } + }, + ["ACTIVE"], + ), + # Test case 6: No filter_statuses or include_deleted in state, instance has filter_statuses + ( + {"123": {"date": "2023-02-15T00:00:00+00:00"}}, + {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"}, + { + "123": { + "date": "2021-01-20T00:00:00+00:00", + "filter_statuses": ["ACTIVE"], + } + }, + ["ACTIVE"], + ), + ], + ) + def test_get_updated_state( + self, + incremental_class_instance, + current_stream_state, + latest_record, + expected_state, + instance_filter_statuses, + ): + # Set the instance's filter_statuses + incremental_class_instance._filter_statuses = instance_filter_statuses + + new_state = incremental_class_instance.get_updated_state( + current_stream_state, latest_record + ) + assert new_state == expected_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 0f18516db132..5395bab12a5c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -28,7 +28,11 @@ def fb_call_rate_response_fixture(): "fbtrace_id": "this_is_fake_response", } - headers = {"x-app-usage": json.dumps({"call_count": 28, "total_time": 25, "total_cputime": 25})} + headers = { + "x-app-usage": json.dumps( + {"call_count": 28, "total_time": 25, "total_cputime": 25} + ) + } return { "json": { @@ -41,7 +45,10 @@ def fb_call_rate_response_fixture(): @pytest.fixture(name="fb_call_amount_data_response") def fb_call_amount_data_response_fixture(): - error = {"message": "Please reduce the amount of data you're asking for, then retry your request", "code": 1} + error = { + "message": "Please reduce the amount of data you're asking for, then retry your request", + "code": 1, + } return { "json": { @@ -52,29 +59,61 @@ def fb_call_amount_data_response_fixture(): class TestBackoff: - def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id): + def test_limit_reached( + self, mocker, requests_mock, api, fb_call_rate_response, account_id, some_config + ): """Error once, check that we retry and not fail""" # turn Campaigns into non batch mode to test non batch logic campaign_responses = [ fb_call_rate_response, { - "json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]}, + "json": { + "data": [ + {"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, + {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}, + ] + }, "status_code": 200, }, ] - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", campaign_responses) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/1/", [{"status_code": 200}]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/2/", [{"status_code": 200}]) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", + campaign_responses, + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/1/", + [{"status_code": 200}], + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/2/", + [{"status_code": 200}], + ) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False) + stream = Campaigns( + api=api, + account_ids=[account_id], + start_date=pendulum.now(), + end_date=pendulum.now(), + ) try: - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + records = list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert records except FacebookRequestError: pytest.fail("Call rate error has not being handled") - def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, account_id): + def test_batch_limit_reached( + self, requests_mock, api, fb_call_rate_response, account_id + ): """Error once, check that we retry and not fail""" responses = [ fb_call_rate_response, @@ -101,22 +140,56 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac fb_call_rate_response, { "json": [ - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - {"body": json.dumps({"name": "creative 2"}), "code": 200, "headers": {}}, + { + "body": json.dumps({"name": "creative 1"}), + "code": 200, + "headers": {}, + }, + { + "body": json.dumps({"name": "creative 2"}), + "code": 200, + "headers": {}, + }, ] }, ] - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/adcreatives", responses) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) - requests_mock.register_uri("POST", FacebookSession.GRAPH + f"/{FB_API_VERSION}/", batch_responses) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/adcreatives", + responses, + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", + responses, + ) + requests_mock.register_uri( + "POST", FacebookSession.GRAPH + f"/{FB_API_VERSION}/", batch_responses + ) - stream = AdCreatives(api=api, include_deleted=False) - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdCreatives(api=api, account_ids=[account_id]) + records = list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert records == [ - {"id": "123", "object_type": "SHARE", "status": "ACTIVE"}, - {"id": "1234", "object_type": "SHARE", "status": "ACTIVE"}, + { + "account_id": "unknown_account", + "id": "123", + "object_type": "SHARE", + "status": "ACTIVE", + }, + { + "account_id": "unknown_account", + "id": "1234", + "object_type": "SHARE", + "status": "ACTIVE", + }, ] @pytest.mark.parametrize( @@ -130,7 +203,12 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac ) def test_common_error_retry(self, error_response, requests_mock, api, account_id): """Error once, check that we retry and not fail""" - account_data = {"id": 1, "updated_time": "2020-09-25T00:00:00Z", "name": "Some name"} + account_data = { + "account_id": "unknown_account", + "id": 1, + "updated_time": "2020-09-25T00:00:00Z", + "name": "Some name", + } responses = [ error_response, { @@ -139,27 +217,67 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id }, ] - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/business_users", json={"data": []}) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", responses) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/business_users", + json={"data": []}, + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", + responses, + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", + responses, + ) - stream = AdAccount(api=api) - accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdAccount(api=api, account_ids=[account_id]) + accounts = list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert accounts == [account_data] - def test_limit_error_retry(self, fb_call_amount_data_response, requests_mock, api, account_id): + def test_limit_error_retry( + self, fb_call_amount_data_response, requests_mock, api, account_id + ): """Error every time, check limit parameter decreases by 2 times every new call""" res = requests_mock.register_uri( - "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", [fb_call_amount_data_response] + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", + [fb_call_amount_data_response], ) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Campaigns( + api=api, + account_ids=[account_id], + start_date=pendulum.now(), + end_date=pendulum.now(), + page_size=100, + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) except AirbyteTracedException: - assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "25", "12", "6"] + assert [x.qs.get("limit")[0] for x in res.request_history] == [ + "100", + "50", + "25", + "12", + "6", + ] def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id): """Error every time, check limit parameter decreases by 2 times every new call""" @@ -192,11 +310,28 @@ def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id [error, success, error, success], ) - stream = Activities(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Activities( + api=api, + account_ids=[account_id], + start_date=pendulum.now(), + end_date=pendulum.now(), + page_size=100, + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) except FacebookRequestError: - assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "100", "50"] + assert [x.qs.get("limit")[0] for x in res.request_history] == [ + "100", + "50", + "100", + "50", + ] def test_start_date_not_provided(self, requests_mock, api, account_id): success = { @@ -218,12 +353,28 @@ def test_start_date_not_provided(self, requests_mock, api, account_id): [success], ) - stream = Activities(api=api, start_date=None, end_date=None, include_deleted=False, page_size=100) - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = Activities( + api=api, + account_ids=[account_id], + start_date=None, + end_date=None, + page_size=100, + ) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) - def test_limit_error_retry_next_page(self, fb_call_amount_data_response, requests_mock, api, account_id): + def test_limit_error_retry_next_page( + self, fb_call_amount_data_response, requests_mock, api, account_id + ): """Unlike the previous test, this one tests the API call fail on the second or more page of a request.""" - base_url = FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/advideos" + base_url = ( + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/advideos" + ) res = requests_mock.register_uri( "GET", @@ -231,7 +382,10 @@ def test_limit_error_retry_next_page(self, fb_call_amount_data_response, request [ { "json": { - "data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}], + "data": [ + {"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, + {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}, + ], "paging": {"next": f"{base_url}?after=after_page_1&limit=100"}, }, "status_code": 200, @@ -240,8 +394,27 @@ def test_limit_error_retry_next_page(self, fb_call_amount_data_response, request ], ) - stream = Videos(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Videos( + api=api, + account_ids=[account_id], + start_date=pendulum.now(), + end_date=pendulum.now(), + page_size=100, + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) except AirbyteTracedException: - assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "100", "50", "25", "12", "6"] + assert [x.qs.get("limit")[0] for x in res.request_history] == [ + "100", + "100", + "50", + "25", + "12", + "6", + ] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..d72b4ce6c3e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -0,0 +1,149 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +import pytest +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters +from source_facebook_marketing.source import SourceFacebookMarketing + +# BASE ARGS +CMD = "check" +SOURCE: Source = SourceFacebookMarketing() + + +# HELPERS +def load_config(config_path: str) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +class TestMigrateAccountIdToArray: + TEST_CONFIG_PATH = "unit_tests/test_migrations/account_id_to_array/test_old_config.json" + NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/account_id_to_array/test_new_config.json" + UPGRADED_TEST_CONFIG_PATH = "unit_tests/test_migrations/account_id_to_array/test_upgraded_config.json" + + @staticmethod + def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("account_ids") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + def test_migrate_config(self): + migration_instance = MigrateAccountIdToArray() + original_config = load_config(self.TEST_CONFIG_PATH) + # migrate the test_config + migration_instance.migrate([CMD, "--config", self.TEST_CONFIG_PATH], SOURCE) + # load the updated config + test_migrated_config = load_config(self.TEST_CONFIG_PATH) + # check migrated property + assert "account_ids" in test_migrated_config + assert isinstance(test_migrated_config["account_ids"], list) + # check the old property is in place + assert "account_id" in test_migrated_config + assert isinstance(test_migrated_config["account_id"], str) + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert [original_config["account_id"]] == test_migrated_config["account_ids"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # old custom_reports are stil type(str) + assert isinstance(control_msg.control.connectorConfig.config["account_id"], str) + # new custom_reports are type(list) + assert isinstance(control_msg.control.connectorConfig.config["account_ids"], list) + # check the migrated values + assert control_msg.control.connectorConfig.config["account_ids"] == ["01234567890"] + # revert the test_config to the starting point + self.revert_migration() + + def test_config_is_reverted(self): + # check the test_config state, it has to be the same as before tests + test_config = load_config(self.TEST_CONFIG_PATH) + # check the config no longer has the migarted property + assert "account_ids" not in test_config + # check the old property is still there + assert "account_id" in test_config + assert isinstance(test_config["account_id"], str) + + def test_should_not_migrate_new_config(self): + new_config = load_config(self.NEW_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) + + def test_should_not_migrate_upgraded_config(self): + new_config = load_config(self.UPGRADED_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) + + +class TestMigrateIncludeDeletedToStatusFilters: + OLD_TEST1_CONFIG_PATH = "unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_old_config.json" + NEW_TEST1_CONFIG_PATH = "unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_new_config.json" + OLD_TEST2_CONFIG_PATH = "unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json" + NEW_TEST2_CONFIG_PATH = "unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_new_config.json" + + UPGRADED_TEST_CONFIG_PATH = "unit_tests/test_migrations/account_id_to_array/test_upgraded_config.json" + + filter_properties = ["ad_statuses", "adset_statuses", "campaign_statuses"] + + def revert_migration(self, config_path: str) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + for filter in self.filter_properties: + config.pop(filter) + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + @pytest.mark.parametrize( + "old_config_path, new_config_path, include_deleted", + [(OLD_TEST1_CONFIG_PATH, NEW_TEST1_CONFIG_PATH, False), (OLD_TEST2_CONFIG_PATH, NEW_TEST2_CONFIG_PATH, True)], + ) + def test_migrate_config(self, old_config_path, new_config_path, include_deleted): + migration_instance = MigrateIncludeDeletedToStatusFilters() + original_config = load_config(old_config_path) + # migrate the test_config + migration_instance.migrate([CMD, "--config", old_config_path], SOURCE) + # load the updated config + test_migrated_config = load_config(old_config_path) + # load expected updated config + expected_new_config = load_config(new_config_path) + # compare expected with migrated + assert expected_new_config == test_migrated_config + # check migrated property + if include_deleted: + assert all([filter in test_migrated_config for filter in self.filter_properties]) + # check the old property is in place + assert "include_deleted" in test_migrated_config + assert test_migrated_config["include_deleted"] == include_deleted + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + if include_deleted: + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # revert the test_config to the starting point + self.revert_migration(old_config_path) + + @pytest.mark.parametrize("new_config_path", [NEW_TEST1_CONFIG_PATH, NEW_TEST2_CONFIG_PATH]) + def test_should_not_migrate_new_config(self, new_config_path): + new_config = load_config(new_config_path) + migration_instance = MigrateIncludeDeletedToStatusFilters() + assert not migration_instance.should_migrate(new_config) + + def test_should_not_migrate_upgraded_config(self): + new_config = load_config(self.UPGRADED_TEST_CONFIG_PATH) + migration_instance = MigrateIncludeDeletedToStatusFilters() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_deep_merge.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_deep_merge.py index ca0b43393fef..75214acff1ab 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_deep_merge.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_deep_merge.py @@ -17,7 +17,10 @@ def test_return_new_object(): "key_2": [1, 2], } right = {"key_1": {"two": "right_value", "three": [1, 2, 3]}, "key_2": [3]} - expected_result = {"key_1": {"one": {"a", "b"}, "two": "right_value", "three": [1, 2, 3]}, "key_2": [1, 2, 3]} + expected_result = { + "key_1": {"one": {"a", "b"}, "two": "right_value", "three": [1, 2, 3]}, + "key_2": [1, 2, 3], + } result = deep_merge(deepcopy(left), deepcopy(right)) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py index 372ca7c5cdd2..46bbbbccd0d5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -15,7 +15,11 @@ FB_API_VERSION = FacebookAdsApi.API_VERSION account_id = "unknown_account" -some_config = {"start_date": "2021-01-23T00:00:00Z", "account_id": account_id, "access_token": "unknown_token"} +some_config = { + "start_date": "2021-01-23T00:00:00Z", + "account_ids": [account_id], + "access_token": "unknown_token", +} base_url = f"{FacebookSession.GRAPH}/{FB_API_VERSION}/" act_url = f"{base_url}act_{account_id}/" @@ -26,8 +30,18 @@ } } ad_creative_data = [ - {"id": "111111", "name": "ad creative 1", "updated_time": "2023-03-21T22:33:56-0700"}, - {"id": "222222", "name": "ad creative 2", "updated_time": "2023-03-22T22:33:56-0700"}, + { + "account_id": account_id, + "id": "111111", + "name": "ad creative 1", + "updated_time": "2023-03-21T22:33:56-0700", + }, + { + "account_id": account_id, + "id": "222222", + "name": "ad creative 2", + "updated_time": "2023-03-22T22:33:56-0700", + }, ] ad_creative_response = { "json": { @@ -239,13 +253,23 @@ class TestRealErrors: }, }, "status_code": 400, - "headers": {"x-app-usage": json.dumps({"call_count": 28, "total_time": 25, "total_cputime": 25})}, + "headers": { + "x-app-usage": json.dumps( + {"call_count": 28, "total_time": 25, "total_cputime": 25} + ) + }, }, ), ( "error_500_unknown", { - "json": {"error": {"code": 1, "message": "An unknown error occurred", "error_subcode": 99}}, + "json": { + "error": { + "code": 1, + "message": "An unknown error occurred", + "error_subcode": 99, + } + }, "status_code": 500, }, ), @@ -282,31 +306,52 @@ class TestRealErrors: ), ], ) - def test_retryable_error(self, some_config, requests_mock, name, retryable_error_response): + def test_retryable_error( + self, some_config, requests_mock, name, retryable_error_response + ): """Error once, check that we retry and not fail""" requests_mock.reset_mock() - requests_mock.register_uri("GET", f"{act_url}", [retryable_error_response, ad_account_response]) - requests_mock.register_uri("GET", f"{act_url}adcreatives", [retryable_error_response, ad_creative_response]) + requests_mock.register_uri( + "GET", f"{act_url}", [retryable_error_response, ad_account_response] + ) + requests_mock.register_uri( + "GET", + f"{act_url}adcreatives", + [retryable_error_response, ad_creative_response], + ) - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) - ad_creative_records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"]) + ad_creative_records = list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert ad_creative_records == ad_creative_data - # requests_mock.register_uri("GET", f"{self.act_url}advideos", [error_400_service_temporarily_unavailable, ad_creative_response]) - # stream = Videos(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) - @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) - def test_config_error_during_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): + def test_config_error_during_account_info_read( + self, requests_mock, name, friendly_msg, config_error_response + ): """Error raised during account info read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"]) - requests_mock.register_uri("GET", f"{act_url}", [config_error_response, ad_account_response]) + requests_mock.register_uri( + "GET", f"{act_url}", [config_error_response, ad_account_response] + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) @@ -315,16 +360,28 @@ def test_config_error_during_account_info_read(self, requests_mock, name, friend # @pytest.mark.parametrize("name, friendly_msg, config_error_response", [CONFIG_ERRORS[-1]]) @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) - def test_config_error_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): + def test_config_error_during_actual_nodes_read( + self, requests_mock, name, friendly_msg, config_error_response + ): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdCreatives(api=api, include_deleted=False) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"]) requests_mock.register_uri("GET", f"{act_url}", [ad_account_response]) - requests_mock.register_uri("GET", f"{act_url}adcreatives", [config_error_response, ad_creative_response]) + requests_mock.register_uri( + "GET", + f"{act_url}adcreatives", + [config_error_response, ad_creative_response], + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_state={}, + stream_slice={"account_id": account_id}, + ) + ) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) @@ -332,45 +389,69 @@ def test_config_error_during_actual_nodes_read(self, requests_mock, name, friend assert friendly_msg in error.message @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) - def test_config_error_insights_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): + def test_config_error_insights_account_info_read( + self, requests_mock, name, friendly_msg, config_error_response + ): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) - requests_mock.register_uri("GET", f"{act_url}", [config_error_response, ad_account_response]) + requests_mock.register_uri( + "GET", f"{act_url}", [config_error_response, ad_account_response] + ) try: - slice = list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}))[0] - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={})) + slice = list( + stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}) + )[0] + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={} + ) + ) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) assert error.failure_type == FailureType.config_error assert friendly_msg in error.message - @pytest.mark.parametrize("name, friendly_msg, config_error_response", [CONFIG_ERRORS[0]]) - def test_config_error_insights_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): + @pytest.mark.parametrize( + "name, friendly_msg, config_error_response", [CONFIG_ERRORS[0]] + ) + def test_config_error_insights_during_actual_nodes_read( + self, requests_mock, name, friendly_msg, config_error_response + ): """Error raised during actual nodes read""" - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) requests_mock.register_uri("GET", f"{act_url}", [ad_account_response]) - requests_mock.register_uri("GET", f"{act_url}insights", [config_error_response, ad_creative_response]) + requests_mock.register_uri( + "GET", f"{act_url}insights", [config_error_response, ad_creative_response] + ) try: - slice = list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}))[0] - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={})) + slice = list( + stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}) + )[0] + list( + stream.read_records( + sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={} + ) + ) assert False except Exception as error: assert isinstance(error, AirbyteTracedException) @@ -411,17 +492,35 @@ def test_adaccount_list_objects_retry(self, requests_mock, failure_response): ] As a workaround for this case we can retry the API call excluding `owner` from `?fields=` GET query param. """ - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) - stream = AdAccount(api=api) + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdAccount( + api=api, + account_ids=some_config["account_ids"], + ) - business_user = {"account_id": account_id, "business": {"id": "1", "name": "TEST"}} - requests_mock.register_uri("GET", f"{base_url}me/business_users", status_code=200, json=business_user) + business_user = { + "account_id": account_id, + "business": {"id": "1", "name": "TEST"}, + } + requests_mock.register_uri( + "GET", f"{base_url}me/business_users", status_code=200, json=business_user + ) assigend_users = {"account_id": account_id, "tasks": ["TASK"]} - requests_mock.register_uri("GET", f"{act_url}assigned_users", status_code=200, json=assigend_users) + requests_mock.register_uri( + "GET", f"{act_url}assigned_users", status_code=200, json=assigend_users + ) success_response = {"status_code": 200, "json": {"account_id": account_id}} - requests_mock.register_uri("GET", f"{act_url}", [failure_response, success_response]) + requests_mock.register_uri( + "GET", f"{act_url}", [failure_response, success_response] + ) - record_gen = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=None, stream_state={}) - assert list(record_gen) == [{"account_id": "unknown_account", "id": "act_unknown_account"}] + record_gen = stream.read_records( + sync_mode=SyncMode.full_refresh, + stream_slice={"account_id": account_id}, + stream_state={}, + ) + assert list(record_gen) == [ + {"account_id": "unknown_account", "id": "act_unknown_account"} + ] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_new_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_new_config.json new file mode 100644 index 000000000000..489ff3fd68fb --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_new_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json new file mode 100644 index 000000000000..a04560eb7710 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_upgraded_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_upgraded_config.json new file mode 100644 index 000000000000..648b4e2c390b --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_upgraded_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_new_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_new_config.json new file mode 100644 index 000000000000..d054e1bae501 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_new_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "access_token": "access_token", + "include_deleted": false +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_old_config.json new file mode 100644 index 000000000000..72dcc27afbdf --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_false/test_old_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": false, + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_new_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_new_config.json new file mode 100644 index 000000000000..e579fa634de8 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_new_config.json @@ -0,0 +1,46 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": true, + "account_ids": ["01234567890"], + "access_token": "access_token", + "ad_statuses": [ + "ACTIVE", + "ADSET_PAUSED", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "DISAPPROVED", + "IN_PROCESS", + "PAUSED", + "PENDING_BILLING_INFO", + "PENDING_REVIEW", + "PREAPPROVED", + "WITH_ISSUES" + ], + "adset_statuses": [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ], + "campaign_statuses": [ + "ACTIVE", + "ARCHIVED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json new file mode 100644 index 000000000000..0cf00a31758d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": true, + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/test_upgraded_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/test_upgraded_config.json new file mode 100644 index 000000000000..e579fa634de8 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/test_upgraded_config.json @@ -0,0 +1,46 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": true, + "account_ids": ["01234567890"], + "access_token": "access_token", + "ad_statuses": [ + "ACTIVE", + "ADSET_PAUSED", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "DISAPPROVED", + "IN_PROCESS", + "PAUSED", + "PENDING_BILLING_INFO", + "PENDING_REVIEW", + "PREAPPROVED", + "WITH_ISSUES" + ], + "adset_statuses": [ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ], + "campaign_statuses": [ + "ACTIVE", + "ARCHIVED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES" + ] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 98bb41ad72f9..2ca1e4e6a822 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -4,6 +4,7 @@ from copy import deepcopy +from unittest.mock import call import pytest from airbyte_cdk.models import ( @@ -26,13 +27,21 @@ @pytest.fixture(name="config") def config_fixture(requests_mock): config = { - "account_id": "123", + "account_ids": ["123"], "access_token": "TOKEN", "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/me/business_users", json={"data": []}) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", json={"account": 123}) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/me/business_users", + json={"data": []}, + ) + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", + json={"account": 123}, + ) return config @@ -50,7 +59,7 @@ def inner(**kwargs): @pytest.fixture(name="api") def api_fixture(mocker): api_mock = mocker.patch("source_facebook_marketing.source.API") - api_mock.return_value = mocker.Mock(account=123) + api_mock.return_value = mocker.Mock(account=mocker.Mock(return_value=123)) return api_mock @@ -78,16 +87,25 @@ def test_check_connection_ok(self, config, logger_mock, fb_marketing): assert ok assert not error_msg - def test_check_connection_find_account_was_called(self, api_find_account, config, logger_mock, fb_marketing): + def test_check_connection_find_account_was_called( + self, api_find_account, config, logger_mock, fb_marketing + ): """Check if _find_account was called to validate credentials""" ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) - api_find_account.assert_called_once_with(config["account_id"]) - logger_mock.info.assert_called_once_with("Select account 1234") + api_find_account.assert_called_once_with(config["account_ids"][0]) + logger_mock.info.assert_has_calls( + [ + call("Attempting to retrieve information for account with ID: 123"), + call("Successfully retrieved account information for account: 1234"), + ] + ) assert ok assert not error_msg - def test_check_connection_future_date_range(self, api, config, logger_mock, fb_marketing): + def test_check_connection_future_date_range( + self, api, config, logger_mock, fb_marketing + ): config["start_date"] = "2219-10-10T00:00:00" config["end_date"] = "2219-10-11T00:00:00" assert fb_marketing.check_connection(logger_mock, config=config) == ( @@ -95,7 +113,9 @@ def test_check_connection_future_date_range(self, api, config, logger_mock, fb_m "Date range can not be in the future.", ) - def test_check_connection_end_date_before_start_date(self, api, config, logger_mock, fb_marketing): + def test_check_connection_end_date_before_start_date( + self, api, config, logger_mock, fb_marketing + ): config["start_date"] = "2019-10-10T00:00:00" config["end_date"] = "2019-10-09T00:00:00" assert fb_marketing.check_connection(logger_mock, config=config) == ( @@ -110,7 +130,9 @@ def test_check_connection_empty_config(self, api, logger_mock, fb_marketing): assert not ok assert error_msg - def test_check_connection_config_no_start_date(self, api, config, logger_mock, fb_marketing): + def test_check_connection_config_no_start_date( + self, api, config, logger_mock, fb_marketing + ): config.pop("start_date") ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) @@ -137,24 +159,44 @@ def test_spec(self, fb_marketing): def test_get_custom_insights_streams(self, api, config, fb_marketing): config["custom_insights"] = [ - {"name": "test", "fields": ["account_id"], "breakdowns": ["ad_format_asset"], "action_breakdowns": ["action_device"]}, + { + "name": "test", + "fields": ["account_id"], + "breakdowns": ["ad_format_asset"], + "action_breakdowns": ["action_device"], + }, ] config = ConnectorConfig.parse_obj(config) assert fb_marketing.get_custom_insights_streams(api, config) - def test_get_custom_insights_action_breakdowns_allow_empty(self, api, config, fb_marketing): + def test_get_custom_insights_action_breakdowns_allow_empty( + self, api, config, fb_marketing + ): config["custom_insights"] = [ - {"name": "test", "fields": ["account_id"], "breakdowns": ["ad_format_asset"], "action_breakdowns": []}, + { + "name": "test", + "fields": ["account_id"], + "breakdowns": ["ad_format_asset"], + "action_breakdowns": [], + }, ] config["action_breakdowns_allow_empty"] = False - streams = fb_marketing.get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) + streams = fb_marketing.get_custom_insights_streams( + api, ConnectorConfig.parse_obj(config) + ) assert len(streams) == 1 assert streams[0].breakdowns == ["ad_format_asset"] - assert streams[0].action_breakdowns == ["action_type", "action_target_id", "action_destination"] + assert streams[0].action_breakdowns == [ + "action_type", + "action_target_id", + "action_destination", + ] config["action_breakdowns_allow_empty"] = True - streams = fb_marketing.get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) + streams = fb_marketing.get_custom_insights_streams( + api, ConnectorConfig.parse_obj(config) + ) assert len(streams) == 1 assert streams[0].breakdowns == ["ad_format_asset"] assert streams[0].action_breakdowns == [] @@ -181,9 +223,13 @@ def test_read_missing_stream(self, config, api, logger_mock, fb_marketing): def test_check_config(config_gen, requests_mock, fb_marketing): - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {} + ) - assert command_check(fb_marketing, config_gen()) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check(fb_marketing, config_gen()) == AirbyteConnectionStatus( + status=Status.SUCCEEDED, message=None + ) status = command_check(fb_marketing, config_gen(start_date="2019-99-10T00:00:00Z")) assert status.status == Status.FAILED @@ -194,5 +240,9 @@ def test_check_config(config_gen, requests_mock, fb_marketing): status = command_check(fb_marketing, config_gen(start_date=...)) assert status.status == Status.SUCCEEDED - assert command_check(fb_marketing, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) - assert command_check(fb_marketing, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check( + fb_marketing, config_gen(end_date=...) + ) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check( + fb_marketing, config_gen(end_date="") + ) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py index 12f493ce37e5..d300af0571b7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -7,6 +7,7 @@ from pendulum import duration from source_facebook_marketing.api import MyFacebookAdsApi from source_facebook_marketing.streams import ( + AdSets, AdsInsights, AdsInsightsActionType, AdsInsightsAgeAndGender, @@ -19,40 +20,65 @@ from source_facebook_marketing.streams.streams import fetch_thumbnail_data_url -def test_filter_all_statuses(api, mocker): +def test_filter_all_statuses(api, mocker, some_config): mocker.patch.multiple(FBMarketingStream, __abstractmethods__=set()) + expected = {} + assert ( + FBMarketingStream( + api=api, account_ids=some_config["account_ids"] + )._filter_all_statuses() + == expected + ) + expected = { "filtering": [ { - "field": "None.delivery_info", + "field": "adset.effective_status", "operator": "IN", "value": [ - "active", - "archived", - "completed", - "limited", - "not_delivering", - "deleted", - "not_published", - "pending_review", - "permanently_deleted", - "recently_completed", - "recently_rejected", - "rejected", - "scheduled", - "inactive", + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", ], } ] } - assert FBMarketingStream(api=api)._filter_all_statuses() == expected + assert ( + AdSets( + account_ids=some_config["account_ids"], + start_date="", + end_date="", + api=api, + filter_statuses=[ + "ACTIVE", + "ARCHIVED", + "CAMPAIGN_PAUSED", + "DELETED", + "IN_PROCESS", + "PAUSED", + "WITH_ISSUES", + ], + )._filter_all_statuses() + == expected + ) @pytest.mark.parametrize( - "url", ["https://graph.facebook.com", "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] + "url", + [ + "https://graph.facebook.com", + "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", + "https://graph.facebook.com?", + ], ) def test_fetch_thumbnail_data_url(url, requests_mock): - requests_mock.get(url, status_code=200, headers={"content-type": "content-type"}, content=b"") + requests_mock.get( + url, status_code=200, headers={"content-type": "content-type"}, content=b"" + ) assert fetch_thumbnail_data_url(url) == "data:content-type;base64," @@ -69,38 +95,84 @@ def test_parse_call_rate_header(): [ [AdsInsights, [], ["action_type", "action_target_id", "action_destination"]], [AdsInsightsActionType, [], ["action_type"]], - [AdsInsightsAgeAndGender, ["age", "gender"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsCountry, ["country"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsDma, ["dma"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsPlatformAndDevice, ["publisher_platform", "platform_position", "impression_device"], ["action_type"]], - [AdsInsightsRegion, ["region"], ["action_type", "action_target_id", "action_destination"]], + [ + AdsInsightsAgeAndGender, + ["age", "gender"], + ["action_type", "action_target_id", "action_destination"], + ], + [ + AdsInsightsCountry, + ["country"], + ["action_type", "action_target_id", "action_destination"], + ], + [ + AdsInsightsDma, + ["dma"], + ["action_type", "action_target_id", "action_destination"], + ], + [ + AdsInsightsPlatformAndDevice, + ["publisher_platform", "platform_position", "impression_device"], + ["action_type"], + ], + [ + AdsInsightsRegion, + ["region"], + ["action_type", "action_target_id", "action_destination"], + ], ], ) -def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} +def test_ads_insights_breakdowns( + class_name, breakdowns, action_breakdowns, some_config +): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } stream = class_name(**kwargs) assert stream.breakdowns == breakdowns assert stream.action_breakdowns == action_breakdowns -def test_custom_ads_insights_breakdowns(): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} - stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs) +def test_custom_ads_insights_breakdowns(some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } + stream = AdsInsights( + breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs + ) assert stream.breakdowns == ["mmm"] assert stream.action_breakdowns == ["action_destination"] stream = AdsInsights(breakdowns=[], action_breakdowns=[], **kwargs) assert stream.breakdowns == [] - assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] - - stream = AdsInsights(breakdowns=[], action_breakdowns=[], action_breakdowns_allow_empty=True, **kwargs) + assert stream.action_breakdowns == [ + "action_type", + "action_target_id", + "action_destination", + ] + + stream = AdsInsights( + breakdowns=[], + action_breakdowns=[], + action_breakdowns_allow_empty=True, + **kwargs + ) assert stream.breakdowns == [] assert stream.action_breakdowns == [] -def test_custom_ads_insights_action_report_times(): +def test_custom_ads_insights_action_report_times(some_config): kwargs = { "api": None, + "account_ids": some_config["account_ids"], "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py index ccde2ee1fcba..652237fb8f91 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py @@ -23,13 +23,18 @@ "start_date", pendulum.local(2019, 1, 1), pendulum.local(2020, 3, 2), - [f"The start date cannot be beyond 37 months from the current date. " f"Set start date to {pendulum.local(2020, 3, 2)}."], + [ + f"The start date cannot be beyond 37 months from the current date. " + f"Set start date to {pendulum.local(2020, 3, 2)}." + ], ), ( "start_date", TODAY + pendulum.duration(months=1), TODAY, - [f"The start date cannot be in the future. Set start date to today's date - {TODAY}."], + [ + f"The start date cannot be in the future. Set start date to today's date - {TODAY}." + ], ), ( "end_date", diff --git a/airbyte-integrations/connectors/source-facebook-pages/main.py b/airbyte-integrations/connectors/source-facebook-pages/main.py index aba9ffd9f0ea..466fc2800442 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/main.py +++ b/airbyte-integrations/connectors/source-facebook-pages/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_facebook_pages import SourceFacebookPages +from source_facebook_pages.run import run if __name__ == "__main__": - source = SourceFacebookPages() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml index d8dd267fcc57..1101c253d8ce 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml @@ -15,6 +15,10 @@ data: icon: facebook.svg license: ELv2 name: Facebook Pages + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-facebook-pages registries: cloud: enabled: false # hide from cloud until https://github.com/airbytehq/airbyte/issues/25515 is finished diff --git a/airbyte-integrations/connectors/source-facebook-pages/setup.py b/airbyte-integrations/connectors/source-facebook-pages/setup.py index 7bce89eaea9d..808c6ffb27c7 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/setup.py +++ b/airbyte-integrations/connectors/source-facebook-pages/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( + entry_points={ + "console_scripts": [ + "source-facebook-pages=source_facebook_pages.run:run", + ], + }, name="source_facebook_pages", description="Source implementation for Facebook Pages.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py new file mode 100644 index 000000000000..3b70710fe59d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_facebook_pages import SourceFacebookPages + + +def run(): + source = SourceFacebookPages() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile deleted file mode 100644 index d0648a0212e1..000000000000 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_faker ./source_faker - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=5.0.0 -LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/README.md b/airbyte-integrations/connectors/source-faker/README.md index b8415263d9a6..e42d204e2dd7 100644 --- a/airbyte-integrations/connectors/source-faker/README.md +++ b/airbyte-integrations/connectors/source-faker/README.md @@ -1,78 +1,56 @@ -# Faker Source +# Faker source connector + This is the repository for the Faker source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/faker). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/faker). ## Local development ### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies - -From this connector directory, create a virtual environment: - -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: - -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/faker) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_faker/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -#### Create credentials - -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/faker) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_faker/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source faker test creds` -and place them into `secrets/config.json`. ### Locally running the connector - ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-faker spec +poetry run source-faker check --config secrets/config.json +poetry run source-faker discover --config secrets/config.json +poetry run source-faker read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-faker build ``` -An image will be built with the tag `airbyte/source-faker:dev`. - -**Via `docker build`:** -```bash -docker build -t airbyte/source-faker:dev . -``` +An image will be available on your host with the tag `airbyte/source-faker:dev`. -#### Run +### Running as a docker container Then run any of the connector commands as follows: - ``` docker run --rm airbyte/source-faker:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-faker:dev check --config /secrets/config.json @@ -80,32 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-faker:dev discover --c docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-faker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-faker test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management - -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -- required for the testing need to go to `TEST_REQUIREMENTS` list +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-faker test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/faker.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/faker.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py index 782659c7a6fb..9df2974ae7bd 100644 --- a/airbyte-integrations/connectors/source-faker/main.py +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_faker import SourceFaker +from source_faker.run import run if __name__ == "__main__": - source = SourceFaker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-faker/metadata.yaml b/airbyte-integrations/connectors/source-faker/metadata.yaml index 83aa3520b711..71bec9727379 100644 --- a/airbyte-integrations/connectors/source-faker/metadata.yaml +++ b/airbyte-integrations/connectors/source-faker/metadata.yaml @@ -4,10 +4,12 @@ data: sl: 100 allowedHosts: hosts: [] + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 - dockerImageTag: 5.0.0 + dockerImageTag: 6.0.2 dockerRepository: airbyte/source-faker documentationUrl: https://docs.airbyte.com/integrations/sources/faker githubIssueLabel: source-faker @@ -30,6 +32,13 @@ data: ID and products.year fields are changing to be integers instead of floats. upgradeDeadline: "2023-08-31" + 6.0.0: + message: Declare 'id' columns as primary keys. + upgradeDeadline: "2024-04-01" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-faker resourceRequirements: jobSpecific: - jobType: sync diff --git a/airbyte-integrations/connectors/source-faker/poetry.lock b/airbyte-integrations/connectors/source-faker/poetry.lock new file mode 100644 index 000000000000..e68b5b3b7c47 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/poetry.lock @@ -0,0 +1,1045 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mimesis" +version = "6.1.1" +description = "Mimesis: Fake Data Generator." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "mimesis-6.1.1-py3-none-any.whl", hash = "sha256:eabe41d7afa23b01dffb51ebd9e10837df6417fef02fa9841989ca886e479790"}, + {file = "mimesis-6.1.1.tar.gz", hash = "sha256:044ac378c61db0e06832ff722548fd6e604881d36bc938002e0bd5b85eeb6a98"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "2714d95fc3a63bfd137182625175cb88f55987b73bfad9cb2a2859c9be10ed44" diff --git a/airbyte-integrations/connectors/source-faker/pyproject.toml b/airbyte-integrations/connectors/source-faker/pyproject.toml new file mode 100644 index 000000000000..d0c56fe9b177 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "6.0.2" +name = "source-faker" +description = "Source implementation for fake but realistic looking data." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/faker" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_faker" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.1" +mimesis = "==6.1.1" + +[tool.poetry.scripts] +source-faker = "source_faker.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.2" diff --git a/airbyte-integrations/connectors/source-faker/requirements.txt b/airbyte-integrations/connectors/source-faker/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-faker/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-faker/setup.py b/airbyte-integrations/connectors/source-faker/setup.py deleted file mode 100644 index 1a16ba5ea485..000000000000 --- a/airbyte-integrations/connectors/source-faker/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "mimesis==6.1.1"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.2", -] - -setup( - name="source_faker", - description="Source implementation for fake but realistic looking data.", - author="Airbyte", - author_email="evan@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "record_data/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/run.py b/airbyte-integrations/connectors/source-faker/source_faker/run.py new file mode 100644 index 000000000000..5bf64ce0d724 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_faker import SourceFaker + + +def run(): + source = SourceFaker() + launch(source, sys.argv[1:]) + + +if __name__ == "__main__": + run() diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index ba7d70b7dd2c..9b015e94851b 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -15,7 +15,7 @@ class Products(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): @@ -65,7 +65,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class Users(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): @@ -119,13 +119,13 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: if records_remaining_this_loop == 0: break - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} class Purchases(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): @@ -180,6 +180,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: if records_remaining_this_loop == 0: break - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} diff --git a/airbyte-integrations/connectors/source-fastbill/main.py b/airbyte-integrations/connectors/source-fastbill/main.py index d807714bce50..acf657a8214d 100644 --- a/airbyte-integrations/connectors/source-fastbill/main.py +++ b/airbyte-integrations/connectors/source-fastbill/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fastbill import SourceFastbill +from source_fastbill.run import run if __name__ == "__main__": - source = SourceFastbill() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fastbill/metadata.yaml b/airbyte-integrations/connectors/source-fastbill/metadata.yaml index 805d8dd61b3c..faf0afa33809 100644 --- a/airbyte-integrations/connectors/source-fastbill/metadata.yaml +++ b/airbyte-integrations/connectors/source-fastbill/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fastbill registries: cloud: enabled: false @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/fastbill tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-fastbill/setup.py b/airbyte-integrations/connectors/source-fastbill/setup.py index 843e992da23a..a83e4e1d307a 100644 --- a/airbyte-integrations/connectors/source-fastbill/setup.py +++ b/airbyte-integrations/connectors/source-fastbill/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-fastbill=source_fastbill.run:run", + ], + }, name="source_fastbill", description="Source implementation for Fastbill.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py b/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py new file mode 100644 index 000000000000..eee32cd7dd6c --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fastbill import SourceFastbill + + +def run(): + source = SourceFastbill() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-fauna/main.py b/airbyte-integrations/connectors/source-fauna/main.py index 86b4f167af64..9e4bc25307ed 100644 --- a/airbyte-integrations/connectors/source-fauna/main.py +++ b/airbyte-integrations/connectors/source-fauna/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fauna import SourceFauna +from source_fauna.run import run if __name__ == "__main__": - source = SourceFauna() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fauna/metadata.yaml b/airbyte-integrations/connectors/source-fauna/metadata.yaml index 602dccd287e4..cf36ff0c13ec 100644 --- a/airbyte-integrations/connectors/source-fauna/metadata.yaml +++ b/airbyte-integrations/connectors/source-fauna/metadata.yaml @@ -8,6 +8,10 @@ data: icon: fauna.svg license: MIT name: Fauna + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fauna registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index 552523c1eb37..25c4e60b8647 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-fauna=source_fauna.run:run", + ], + }, name="source_fauna", description="Source implementation for Fauna.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/run.py b/airbyte-integrations/connectors/source-fauna/source_fauna/run.py new file mode 100644 index 000000000000..20d6ee8bb021 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fauna import SourceFauna + + +def run(): + source = SourceFauna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-file/.coveragerc b/airbyte-integrations/connectors/source-file/.coveragerc new file mode 100644 index 000000000000..b9ac751e624a --- /dev/null +++ b/airbyte-integrations/connectors/source-file/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_file/run.py diff --git a/airbyte-integrations/connectors/source-file/README.md b/airbyte-integrations/connectors/source-file/README.md index a48bdec6d9bb..c313bbb3a0f6 100644 --- a/airbyte-integrations/connectors/source-file/README.md +++ b/airbyte-integrations/connectors/source-file/README.md @@ -1,67 +1,91 @@ -# Mailerlite Source +# File source connector -This is the repository for the Mailerlite configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailerlite). + +This is the repository for the File source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/file). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailerlite) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailerlite/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailerlite test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name source-mailerlite build +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/file) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_file/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-file spec +poetry run source-file check --config secrets/config.json +poetry run source-file discover --config secrets/config.json +poetry run source-file read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -An image will be built with the tag `airbyte/source-mailerlite:dev`. +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -**Via `docker build`:** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-mailerlite:dev . +airbyte-ci connectors --name=source-file build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-file:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-mailerlite:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailerlite:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-file:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-file:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-file test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-file test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/file.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/file.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-file/acceptance-test-config.yml b/airbyte-integrations/connectors/source-file/acceptance-test-config.yml index 34d9327e1a9e..dde3f1091671 100644 --- a/airbyte-integrations/connectors/source-file/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-file/acceptance-test-config.yml @@ -28,6 +28,9 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes + file_types: + skip_test: yes + bypass_reason: "Source is not based on file based CDK" full_refresh: tests: - config_path: "integration_tests/config.json" diff --git a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py index a1d70ec923b5..758c1118eae6 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py @@ -25,6 +25,7 @@ def check_read(config, expected_columns=10, expected_rows=42): ("ssh", "files/test.csv", "csv"), ("scp", "files/test.csv", "csv"), ("sftp", "files/test.csv", "csv"), + ("ssh", "files/test.csv.zip", "csv"), ("ssh", "files/test.csv.gz", "csv"), # text in binary ("ssh", "files/test.pkl", "pickle"), # binary ("sftp", "files/test.pkl.gz", "pickle"), # binary in binary diff --git a/airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json index fc398565ac12..222f59dea2ab 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json +++ b/airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json @@ -20,6 +20,7 @@ "jsonl", "excel", "excel_binary", + "fwf", "feather", "parquet", "yaml" diff --git a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py index 2f9b195df3c7..66290c6512a9 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py @@ -30,6 +30,7 @@ def check_read(config, expected_columns=10, expected_rows=42): ("jsonl", "jsonl", 2, 6492, "jsonl"), ("excel", "xls", 8, 50, "demo"), ("excel", "xlsx", 8, 50, "demo"), + ("fwf", "txt", 4, 2, "demo"), ("feather", "feather", 9, 3, "demo"), ("parquet", "parquet", 9, 3, "demo"), ("yaml", "yaml", 8, 3, "demo"), diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/configured_catalog_fwf.json b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/configured_catalog_fwf.json new file mode 100644 index 000000000000..e2aa27a38298 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/configured_catalog_fwf.json @@ -0,0 +1,25 @@ +{ + "streams": [ + { + "stream": { + "name": "test", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "text": { "type": "string" }, + "num": { "type": "number" }, + "float": { "type": "number" }, + "bool": { "type": "string" } + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/demo.txt b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/demo.txt new file mode 100644 index 000000000000..ba1ebc76256e --- /dev/null +++ b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/formats/fwf/demo.txt @@ -0,0 +1,3 @@ +text num float bool +short 1 0.2 true +long_text 33 0.0 false diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.zip b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.zip new file mode 100644 index 000000000000..0c388a8f2b47 Binary files /dev/null and b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.csv.zip differ diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.xlsx b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.xlsx index 5c8fab657899..55730f641643 100644 Binary files a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.xlsx and b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test.xlsx differ diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_one_line.xlsx b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_one_line.xlsx new file mode 100644 index 000000000000..cf318f5ae36b Binary files /dev/null and b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_one_line.xlsx differ diff --git a/airbyte-integrations/connectors/source-file/main.py b/airbyte-integrations/connectors/source-file/main.py index 3ab698c087c0..3e7e82fd61d8 100644 --- a/airbyte-integrations/connectors/source-file/main.py +++ b/airbyte-integrations/connectors/source-file/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_file import SourceFile +from source_file.run import run if __name__ == "__main__": - source = SourceFile() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-file/metadata.yaml b/airbyte-integrations/connectors/source-file/metadata.yaml index a2c748e050d3..3f58d1298337 100644 --- a/airbyte-integrations/connectors/source-file/metadata.yaml +++ b/airbyte-integrations/connectors/source-file/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.15 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-file documentationUrl: https://docs.airbyte.com/integrations/sources/file githubIssueLabel: source-file icon: file.svg license: MIT name: File (CSV, JSON, Excel, Feather, Parquet) + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-file registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-file/poetry.lock b/airbyte-integrations/connectors/source-file/poetry.lock new file mode 100644 index 000000000000..976dee468500 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/poetry.lock @@ -0,0 +1,2641 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "aiobotocore" +version = "2.3.4" +description = "Async client for aws services using botocore and aiohttp" +optional = false +python-versions = ">=3.6" +files = [ + {file = "aiobotocore-2.3.4-py3-none-any.whl", hash = "sha256:eae059eb51726cee4de2027cfc72bfccc76cf0c229d6b2b08f640e53a568f657"}, + {file = "aiobotocore-2.3.4.tar.gz", hash = "sha256:6554ebea5764f66f4be544a4fcaa0953ee80e600dd7bd818ba4893d72bf12bfb"}, +] + +[package.dependencies] +aiohttp = ">=3.3.1" +aioitertools = ">=0.5.1" +botocore = ">=1.24.21,<1.24.22" +wrapt = ">=1.10.10" + +[package.extras] +awscli = ["awscli (>=1.22.76,<1.22.77)"] +boto3 = ["boto3 (>=1.21.21,<1.21.22)"] + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aioitertools" +version = "0.11.0" +description = "itertools and builtins for AsyncIO and mixed iterables" +optional = false +python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "airbyte-cdk" +version = "0.51.41" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.41.tar.gz", hash = "sha256:cce614d67872cf66a151e5b72d70f4bf26e2a1ce672c7abfc15a5cb4e45d8429"}, + {file = "airbyte_cdk-0.51.41-py3-none-any.whl", hash = "sha256:bbf82a45d9ec97c4a92b85e3312b327f8060fffec1f7c7ea7dfa720f9adcc13b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "azure-common" +version = "1.1.28" +description = "Microsoft Azure Client Library for Python (Common)" +optional = false +python-versions = "*" +files = [ + {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, + {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, +] + +[[package]] +name = "azure-core" +version = "1.30.0" +description = "Microsoft Azure Core Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"}, + {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"}, +] + +[package.dependencies] +requests = ">=2.21.0" +six = ">=1.11.0" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["aiohttp (>=3.0)"] + +[[package]] +name = "azure-storage-blob" +version = "12.19.0" +description = "Microsoft Azure Blob Storage Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "azure-storage-blob-12.19.0.tar.gz", hash = "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897"}, + {file = "azure_storage_blob-12.19.0-py3-none-any.whl", hash = "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b"}, +] + +[package.dependencies] +azure-core = ">=1.28.0,<2.0.0" +cryptography = ">=2.1.4" +isodate = ">=0.6.1" +typing-extensions = ">=4.3.0" + +[package.extras] +aio = ["azure-core[aio] (>=1.28.0,<2.0.0)"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bcrypt" +version = "4.1.2" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.7" +files = [ + {file = "bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0"}, + {file = "bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369"}, + {file = "bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551"}, + {file = "bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a"}, + {file = "bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f"}, + {file = "bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb"}, + {file = "bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "beautifulsoup4" +version = "4.11.1" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, + {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "boto3" +version = "1.21.21" +description = "The AWS SDK for Python" +optional = false +python-versions = ">= 3.6" +files = [ + {file = "boto3-1.21.21-py3-none-any.whl", hash = "sha256:8fa32fcc8be38327bd667237223d71e5e4b2475f39d6882aca4dbad19fff8c29"}, + {file = "boto3-1.21.21.tar.gz", hash = "sha256:6fa0622f308cfd1da758966fc98b52fbd74b80606d14586c8ad82c7a6c4f32d0"}, +] + +[package.dependencies] +botocore = ">=1.24.21,<1.25.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.5.0,<0.6.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.24.21" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">= 3.6" +files = [ + {file = "botocore-1.24.21-py3-none-any.whl", hash = "sha256:92daca8775e738a9db9b465d533019285f09d541e903233261299fd87c2f842c"}, + {file = "botocore-1.24.21.tar.gz", hash = "sha256:7e976cfd0a61601e74624ef8f5246b40a01f2cce73a011ef29cf80a6e371d0fa"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.13.5)"] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2"}, + {file = "cryptography-42.0.2-cp37-abi3-win32.whl", hash = "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee"}, + {file = "cryptography-42.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee"}, + {file = "cryptography-42.0.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33"}, + {file = "cryptography-42.0.2-cp39-abi3-win32.whl", hash = "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635"}, + {file = "cryptography-42.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65"}, + {file = "cryptography-42.0.2.tar.gz", hash = "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "et-xmlfile" +version = "1.1.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, + {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + +[[package]] +name = "fsspec" +version = "2022.7.1" +description = "File-system specification" +optional = false +python-versions = ">=3.7" +files = [ + {file = "fsspec-2022.7.1-py3-none-any.whl", hash = "sha256:36c5a8e7c4fc20cf32ef6934ac0a122accc8a593ddc8478d30c3ca4dbbd95500"}, + {file = "fsspec-2022.7.1.tar.gz", hash = "sha256:7f9fb19d811b027b97c4636c6073eb53bc4cbee2d3c4b33fa88b9f26906fd7d7"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "gcsfs" +version = "2022.7.1" +description = "Convenient Filesystem interface over GCS" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gcsfs-2022.7.1-py2.py3-none-any.whl", hash = "sha256:9bed362f06c677aea54c6991ce15b4ac6526d68bafef381c5be957de0fb9ef0c"}, + {file = "gcsfs-2022.7.1.tar.gz", hash = "sha256:4119a08473bfdb9c7e7dfa431d04fb9c130fb7e26c9c9d97fb486c0e5b25adfd"}, +] + +[package.dependencies] +aiohttp = "*" +decorator = ">4.1.2" +fsspec = "2022.7.1" +google-auth = ">=1.2" +google-auth-oauthlib = "*" +google-cloud-storage = "*" +requests = "*" + +[package.extras] +crc = ["crcmod"] +gcsfuse = ["fusepy"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-api-core" +version = "2.17.0" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.17.0.tar.gz", hash = "sha256:de7ef0450faec7c75e0aea313f29ac870fdc44cfaec9d6499a9a17305980ef66"}, + {file = "google_api_core-2.17.0-py3-none-any.whl", hash = "sha256:08ed79ed8e93e329de5e3e7452746b734e6bf8438d8d64dd3319d21d3164890c"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.27.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-oauthlib" +version = "1.2.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"}, + {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"}, +] + +[package.dependencies] +google-auth = ">=2.15.0" +requests-oauthlib = ">=0.7.0" + +[package.extras] +tool = ["click (>=6.0.0)"] + +[[package]] +name = "google-cloud-core" +version = "2.4.1" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] + +[[package]] +name = "google-cloud-storage" +version = "2.5.0" +description = "Google Cloud Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-storage-2.5.0.tar.gz", hash = "sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235"}, + {file = "google_cloud_storage-2.5.0-py2.py3-none-any.whl", hash = "sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-resumable-media = ">=2.3.2" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.7.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "html5lib" +version = "1.1" +description = "HTML parser based on the WHATWG HTML specification" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, + {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, +] + +[package.dependencies] +six = ">=1.9" +webencodings = "*" + +[package.extras] +all = ["chardet (>=2.2)", "genshi", "lxml"] +chardet = ["chardet (>=2.2)"] +genshi = ["genshi"] +lxml = ["lxml"] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "lxml" +version = "4.9.1" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" +files = [ + {file = "lxml-4.9.1-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed"}, + {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc"}, + {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc"}, + {file = "lxml-4.9.1-cp27-cp27m-win32.whl", hash = "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3"}, + {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"}, + {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"}, + {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"}, + {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8"}, + {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d"}, + {file = "lxml-4.9.1-cp310-cp310-win32.whl", hash = "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7"}, + {file = "lxml-4.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b"}, + {file = "lxml-4.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d"}, + {file = "lxml-4.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3"}, + {file = "lxml-4.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29"}, + {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d"}, + {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318"}, + {file = "lxml-4.9.1-cp35-cp35m-win32.whl", hash = "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7"}, + {file = "lxml-4.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4"}, + {file = "lxml-4.9.1-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf"}, + {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3"}, + {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391"}, + {file = "lxml-4.9.1-cp36-cp36m-win32.whl", hash = "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e"}, + {file = "lxml-4.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7"}, + {file = "lxml-4.9.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca"}, + {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785"}, + {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785"}, + {file = "lxml-4.9.1-cp37-cp37m-win32.whl", hash = "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a"}, + {file = "lxml-4.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e"}, + {file = "lxml-4.9.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715"}, + {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036"}, + {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387"}, + {file = "lxml-4.9.1-cp38-cp38-win32.whl", hash = "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94"}, + {file = "lxml-4.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345"}, + {file = "lxml-4.9.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000"}, + {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25"}, + {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd"}, + {file = "lxml-4.9.1-cp39-cp39-win32.whl", hash = "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb"}, + {file = "lxml-4.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73"}, + {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c"}, + {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9"}, + {file = "lxml-4.9.1.tar.gz", hash = "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=0.29.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "openpyxl" +version = "3.0.10" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.6" +files = [ + {file = "openpyxl-3.0.10-py2.py3-none-any.whl", hash = "sha256:0ab6d25d01799f97a9464630abacbb34aafecdcaa0ef3cba6d6b3499867d0355"}, + {file = "openpyxl-3.0.10.tar.gz", hash = "sha256:e47805627aebcf860edb4edf7987b1309c1b3632f3750538ed962bbcc3bd7449"}, +] + +[package.dependencies] +et-xmlfile = "*" + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "1.4.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas-1.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d51674ed8e2551ef7773820ef5dab9322be0828629f2cbf8d1fc31a0c4fed640"}, + {file = "pandas-1.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:16ad23db55efcc93fa878f7837267973b61ea85d244fc5ff0ccbcfa5638706c5"}, + {file = "pandas-1.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:958a0588149190c22cdebbc0797e01972950c927a11a900fe6c2296f207b1d6f"}, + {file = "pandas-1.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e48fbb64165cda451c06a0f9e4c7a16b534fcabd32546d531b3c240ce2844112"}, + {file = "pandas-1.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f803320c9da732cc79210d7e8cc5c8019aad512589c910c66529eb1b1818230"}, + {file = "pandas-1.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:2893e923472a5e090c2d5e8db83e8f907364ec048572084c7d10ef93546be6d1"}, + {file = "pandas-1.4.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:24ea75f47bbd5574675dae21d51779a4948715416413b30614c1e8b480909f81"}, + {file = "pandas-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d5ebc990bd34f4ac3c73a2724c2dcc9ee7bf1ce6cf08e87bb25c6ad33507e318"}, + {file = "pandas-1.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d6c0106415ff1a10c326c49bc5dd9ea8b9897a6ca0c8688eb9c30ddec49535ef"}, + {file = "pandas-1.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78b00429161ccb0da252229bcda8010b445c4bf924e721265bec5a6e96a92e92"}, + {file = "pandas-1.4.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dfbf16b1ea4f4d0ee11084d9c026340514d1d30270eaa82a9f1297b6c8ecbf0"}, + {file = "pandas-1.4.3-cp38-cp38-win32.whl", hash = "sha256:48350592665ea3cbcd07efc8c12ff12d89be09cd47231c7925e3b8afada9d50d"}, + {file = "pandas-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:605d572126eb4ab2eadf5c59d5d69f0608df2bf7bcad5c5880a47a20a0699e3e"}, + {file = "pandas-1.4.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a3924692160e3d847e18702bb048dc38e0e13411d2b503fecb1adf0fcf950ba4"}, + {file = "pandas-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07238a58d7cbc8a004855ade7b75bbd22c0db4b0ffccc721556bab8a095515f6"}, + {file = "pandas-1.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:755679c49460bd0d2f837ab99f0a26948e68fa0718b7e42afbabd074d945bf84"}, + {file = "pandas-1.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41fc406e374590a3d492325b889a2686b31e7a7780bec83db2512988550dadbf"}, + {file = "pandas-1.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d9382f72a4f0e93909feece6fef5500e838ce1c355a581b3d8f259839f2ea76"}, + {file = "pandas-1.4.3-cp39-cp39-win32.whl", hash = "sha256:0daf876dba6c622154b2e6741f29e87161f844e64f84801554f879d27ba63c0d"}, + {file = "pandas-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:721a3dd2f06ef942f83a819c0f3f6a648b2830b191a72bbe9451bcd49c3bd42e"}, + {file = "pandas-1.4.3.tar.gz", hash = "sha256:2ff7788468e75917574f080cd4681b27e1a7bf36461fe968b49a87b5a54d007c"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.18.5", markers = "(platform_machine != \"aarch64\" and platform_machine != \"arm64\") and python_version < \"3.10\""}, + {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, + {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "paramiko" +version = "2.11.0" +description = "SSH2 protocol library" +optional = false +python-versions = "*" +files = [ + {file = "paramiko-2.11.0-py2.py3-none-any.whl", hash = "sha256:655f25dc8baf763277b933dfcea101d636581df8d6b9774d1fb653426b72c270"}, + {file = "paramiko-2.11.0.tar.gz", hash = "sha256:003e6bee7c034c21fbb051bf83dc0a9ee4106204dd3c53054c71452cc4ec3938"}, +] + +[package.dependencies] +bcrypt = ">=3.1.3" +cryptography = ">=2.5" +pynacl = ">=1.0.1" +six = "*" + +[package.extras] +all = ["bcrypt (>=3.1.3)", "gssapi (>=1.4.1)", "invoke (>=1.3)", "pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "pywin32 (>=2.1.8)"] +ed25519 = ["bcrypt (>=3.1.3)", "pynacl (>=1.0.1)"] +gssapi = ["gssapi (>=1.4.1)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=1.3)"] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyarrow" +version = "9.0.0" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, + {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, + {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, + {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, + {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, + {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-docker" +version = "2.0.1" +description = "Simple pytest fixtures for Docker and Docker Compose based tests" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-docker-2.0.1.tar.gz", hash = "sha256:1c17e9202a566f85ed5ef269fe2815bd4899e90eb639622e5d14277372ca7524"}, + {file = "pytest_docker-2.0.1-py3-none-any.whl", hash = "sha256:7103f97b8c479c826b63d73cfb83383dc1970d35105ed1ce78a722c90c7fe650"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +pytest = ">=4.0,<8.0" + +[package.extras] +docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] +tests = ["pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pyxlsb" +version = "1.0.9" +description = "Excel 2007-2010 Binary Workbook (xlsb) parser" +optional = false +python-versions = "*" +files = [ + {file = "pyxlsb-1.0.9-py2.py3-none-any.whl", hash = "sha256:af2daeba799de62eaa05f434607569c1dc39268ad8a0efa5343e027e690289e6"}, + {file = "pyxlsb-1.0.9.tar.gz", hash = "sha256:286f08a55703338eac470fa7fecd6ab8b44dcb0eea8a3eb3ef503ba226e4966a"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "s3fs" +version = "2022.7.1" +description = "Convenient Filesystem interface over S3" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3fs-2022.7.1-py3-none-any.whl", hash = "sha256:97d91bfd78085f33d22c9697824126dbcd2438236951d8ff2374bb718af74e58"}, + {file = "s3fs-2022.7.1.tar.gz", hash = "sha256:4883d682cb96b769fca568dbbcaa33ff457ac8077a03af221e6f253adac05771"}, +] + +[package.dependencies] +aiobotocore = ">=2.3.4,<2.4.0" +aiohttp = "*" +fsspec = "2022.7.1" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.3.4,<2.4.0)"] +boto3 = ["aiobotocore[boto3] (>=2.3.4,<2.4.0)"] + +[[package]] +name = "s3transfer" +version = "0.5.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.6" +files = [ + {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"}, + {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smart-open" +version = "6.0.0" +description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "smart_open-6.0.0-py3-none-any.whl", hash = "sha256:94afbd5058a45d4fdc4f859ed158b46054cb5ca1c019d76f6f8a60495f662129"}, + {file = "smart_open-6.0.0.tar.gz", hash = "sha256:d60106b96f0bcaedf5f1cd46ff5524a1c3d02d5653425618bb0fa66e158d22b0"}, +] + +[package.dependencies] +azure-common = {version = "*", optional = true, markers = "extra == \"all\""} +azure-core = {version = "*", optional = true, markers = "extra == \"all\""} +azure-storage-blob = {version = "*", optional = true, markers = "extra == \"all\""} +boto3 = {version = "*", optional = true, markers = "extra == \"all\""} +google-cloud-storage = {version = ">=1.31.0", optional = true, markers = "extra == \"all\""} +requests = {version = "*", optional = true, markers = "extra == \"all\""} + +[package.extras] +all = ["azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage (>=1.31.0)", "requests"] +azure = ["azure-common", "azure-core", "azure-storage-blob"] +gcs = ["google-cloud-storage (>=1.31.0)"] +http = ["requests"] +s3 = ["boto3"] +test = ["azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage (>=1.31.0)", "moto[server]", "paramiko", "pathlib2", "pytest", "pytest-rerunfailures", "requests", "responses"] +webhdfs = ["requests"] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +optional = false +python-versions = "*" +files = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xlrd" +version = "2.0.1" +description = "Library for developers to extract data from Microsoft Excel (tm) .xls spreadsheet files" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "xlrd-2.0.1-py2.py3-none-any.whl", hash = "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd"}, + {file = "xlrd-2.0.1.tar.gz", hash = "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88"}, +] + +[package.extras] +build = ["twine", "wheel"] +docs = ["sphinx"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "0ffa93b245e7c41287cebbcfc45ceda1e8a715e878c7263f66b4c0a0dae33244" diff --git a/airbyte-integrations/connectors/source-file/pyproject.toml b/airbyte-integrations/connectors/source-file/pyproject.toml new file mode 100644 index 000000000000..3e1f83564b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.4.0" +name = "source-file" +description = "Source implementation for File" +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/file" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_file" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +html5lib = "==1.1" +beautifulsoup4 = "==4.11.1" +openpyxl = "==3.0.10" +google-cloud-storage = "==2.5.0" +pandas = "==1.4.3" +airbyte-cdk = "==0.51.41" +paramiko = "==2.11.0" +xlrd = "==2.0.1" +boto3 = "==1.21.21" +pyarrow = "==9.0.0" +s3fs = "==2022.7.1" +lxml = "==4.9.1" +gcsfs = "==2022.7.1" +pyxlsb = "==1.0.9" +genson = "==1.2.2" + +[tool.poetry.scripts] +source-file = "source_file.run:run" + +[tool.poetry.dependencies.smart-open] +extras = [ "all",] +version = "==6.0.0" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.6.1" +pytest = "==6.2.5" +requests-mock = "^1.9.3" +pytest-docker = "==2.0.1" diff --git a/airbyte-integrations/connectors/source-file/requirements.txt b/airbyte-integrations/connectors/source-file/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-file/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 87dd1fee2806..e69de29bb2d1 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.51.25", - "gcsfs==2022.7.1", - "genson==1.2.2", - "google-cloud-storage==2.5.0", - "pandas==1.4.3", - "paramiko==2.11.0", - "s3fs==2022.7.1", - "boto3==1.21.21", - "smart-open[all]==6.0.0", - "lxml==4.9.1", - "html5lib==1.1", - "beautifulsoup4==4.11.1", - "pyarrow==13.0.0", - "xlrd==2.0.1", - "openpyxl==3.0.10", - "pyxlsb==1.0.9", - "python-gnupg==0.5.2", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=2.0.1", "pytest-mock~=3.6.1"] - -setup( - name="source_file", - description="Source implementation for File", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index fb434d22e595..3709b756498c 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -11,6 +11,8 @@ import traceback import urllib from os import environ, getcwd +import zipfile +from os import environ from typing import Iterable from urllib.parse import urlparse from zipfile import BadZipFile @@ -194,6 +196,7 @@ def storage_scheme(self) -> str: """ storage_name = self._provider["storage"].upper() parse_result = urlparse(self._url) + if storage_name == "GCS": return "gs://" elif storage_name == "S3": @@ -213,7 +216,7 @@ def storage_scheme(self) -> str: elif parse_result.scheme: return parse_result.scheme - logger.error(f"Unknown Storage provider in: {self.full_url}") + logger.error(f"Unknown Storage provider in: {self._url}") return "" def _open_gcs_url(self) -> object: @@ -291,7 +294,8 @@ def __init__( self._provider = provider self._reader_format = format or "csv" self._reader_options = reader_options or {} - self.binary_source = self._reader_format in self.binary_formats or encryption_options + self._is_zip = url.endswith(".zip") + self.binary_source = self._reader_format in self.binary_formats or self._is_zip or encryption_options self.encoding = self._reader_options.get("encoding") self.encryption_options = encryption_options @@ -358,6 +362,7 @@ def load_dataframes(self, fp, skip_data=False, read_sample_chunk: bool = False) "html": pd.read_html, "excel": pd.read_excel, "excel_binary": pd.read_excel, + "fwf": pd.read_fwf, "feather": pd.read_feather, "parquet": pq.ParquetFile, "orc": pd.read_orc, @@ -385,9 +390,9 @@ def load_dataframes(self, fp, skip_data=False, read_sample_chunk: bool = False) yield record if read_sample_chunk and bytes_read >= self.CSV_CHUNK_SIZE: return - elif self._reader_options == "excel_binary": + elif self._reader_format == "excel_binary": reader_options["engine"] = "pyxlsb" - yield from reader(fp, **reader_options) + yield reader(fp, **reader_options) elif self._reader_format == "excel": # Use openpyxl to read new-style Excel (xlsx) file; return to pandas for others try: @@ -463,12 +468,14 @@ def read(self, fields: Iterable = None) -> Iterable[dict]: fields = frozenset(fields) if fields else None # if self.binary_source: # fp = self._cache_stream(fp) + # if self._is_zip: + # fp = self._unzip(fp) + for batch in self.load_dataframes(fp): df = batch.to_pandas() if self._reader_format == "parquet" else batch # for parquet files df['_ab_source_file_url'] = self._url - df_cols = list(df.columns) - columns = [x for x in df_cols if x in fields] if fields else df.columns + columns = fields.intersection(set(df.columns)) if fields else df.columns df.replace({np.nan: None}, inplace=True) yield from df[list(columns)].to_dict(orient="records") except ConnectionResetError: @@ -489,9 +496,21 @@ def read(self, fields: Iterable = None) -> Iterable[dict]: os.remove(file_path) logger.info(f"The file at {file_path} has been deleted.") + def _unzip(self, fp): + tmp_dir = tempfile.TemporaryDirectory() + with zipfile.ZipFile(str(fp.name), "r") as zip_ref: + zip_ref.extractall(tmp_dir.name) + + logger.info("Temp dir content: " + str(os.listdir(tmp_dir.name))) + final_file: str = os.path.join(tmp_dir.name, os.listdir(tmp_dir.name)[0]) + logger.info("Pick up first file: " + final_file) + fp_tmp = open(final_file, "r") + return fp_tmp + + def _cache_stream(self, fp): """cache stream to file""" - fp_tmp = tempfile.TemporaryFile(mode="w+b") + fp_tmp = tempfile.NamedTemporaryFile(mode="w+b") fp_tmp.write(fp.read()) fp_tmp.seek(0) fp.close() @@ -509,6 +528,9 @@ def _stream_properties(self, fp, empty_schema: bool = False, read_sample_chunk: # if self.binary_source: # fp = self._cache_stream(fp) # logger.info("Cache stream successs") + # if self._is_zip: + # fp = self._unzip(fp) + df_list = self.load_dataframes(fp, skip_data=False) fields = {} for df in df_list: @@ -562,7 +584,7 @@ def streams(self, empty_schema: bool = False) -> Iterable: def openpyxl_chunk_reader(self, file, **kwargs): """Use openpyxl lazy loading feature to read excel files (xlsx only) in chunks of 500 lines at a time""" - work_book = load_workbook(filename=file, read_only=True) + work_book = load_workbook(filename=file) user_provided_column_names = kwargs.get("names") for sheetname in work_book.sheetnames: work_sheet = work_book[sheetname] diff --git a/airbyte-integrations/connectors/source-file/source_file/run.py b/airbyte-integrations/connectors/source-file/source_file/run.py new file mode 100644 index 000000000000..646b35cb4c93 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/source_file/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_file import SourceFile + + +def run(): + source = SourceFile() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-file/source_file/source.py b/airbyte-integrations/connectors/source-file/source_file/source.py index ae637904d12b..2903bfb559a6 100644 --- a/airbyte-integrations/connectors/source-file/source_file/source.py +++ b/airbyte-integrations/connectors/source-file/source_file/source.py @@ -16,6 +16,7 @@ AirbyteConnectionStatus, AirbyteMessage, AirbyteRecordMessage, + AirbyteStreamStatus, ConfiguredAirbyteCatalog, ConnectorSpecification, FailureType, @@ -24,6 +25,7 @@ ) from airbyte_cdk.sources import Source from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment +from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message from .client import Client, ConfigurationError from .utils import LOCAL_STORAGE_NAME, dropbox_force_download @@ -61,6 +63,7 @@ class SourceFile(Source): - read_json - read_html - read_excel + - read_fwf - read_feather - read_parquet - read_orc @@ -172,15 +175,33 @@ def read( fields = self.selected_fields(catalog, config) name = client.stream_name - logger.info(f"Reading {name} ({client.reader.full_url})...") - logger.info(fields) + configured_stream = catalog.streams[0] + + logger.info(f"Syncing stream: {name} ({client.reader.full_url})...") + + yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED) + + record_counter = 0 try: for row in client.read(fields=fields): record = AirbyteRecordMessage(stream=name, data=row, emitted_at=int(datetime.now().timestamp()) * 1000) + + record_counter += 1 + if record_counter == 1: + logger.info(f"Marking stream {name} as RUNNING") + yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING) + yield AirbyteMessage(type=Type.RECORD, record=record) + + logger.info(f"Marking stream {name} as STOPPED") + yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.COMPLETE) + except Exception as err: reason = f"Failed to read data of {name} at {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) + logger.exception(f"Encountered an exception while reading stream {name}") + logger.info(f"Marking stream {name} as STOPPED") + yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) raise err @staticmethod diff --git a/airbyte-integrations/connectors/source-file/source_file/spec.json b/airbyte-integrations/connectors/source-file/source_file/spec.json index 4419676384a4..d288184994d8 100644 --- a/airbyte-integrations/connectors/source-file/source_file/spec.json +++ b/airbyte-integrations/connectors/source-file/source_file/spec.json @@ -20,6 +20,7 @@ "jsonl", "excel", "excel_binary", + "fwf", "feather", "parquet", "yaml" diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index f5b1e9d24246..c7a2ae011c36 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -7,9 +7,10 @@ import pytest from airbyte_cdk.utils import AirbyteTracedException -from pandas import read_csv, read_excel +from pandas import read_csv, read_excel, testing from paramiko import SSHException from source_file.client import Client, URLFile +from source_file.utils import backoff_handler from urllib3.exceptions import ProtocolError @@ -34,21 +35,22 @@ def csv_format_client(): @pytest.mark.parametrize( - "storage, expected_scheme", + "storage, expected_scheme, url", [ - ("GCS", "gs://"), - ("S3", "s3://"), - ("AZBLOB", "azure://"), - ("HTTPS", "https://"), - ("SSH", "scp://"), - ("SCP", "scp://"), - ("SFTP", "sftp://"), - ("WEBHDFS", "webhdfs://"), - ("LOCAL", "file://"), + ("GCS", "gs://", "http://localhost"), + ("S3", "s3://", "http://localhost"), + ("AZBLOB", "azure://", "http://localhost"), + ("HTTPS", "https://", "http://localhost"), + ("SSH", "scp://", "http://localhost"), + ("SCP", "scp://", "http://localhost"), + ("SFTP", "sftp://", "http://localhost"), + ("WEBHDFS", "webhdfs://", "http://localhost"), + ("LOCAL", "file://", "http://localhost"), + ("WRONG", "", ""), ], ) -def test_storage_scheme(storage, expected_scheme): - urlfile = URLFile(provider={"storage": storage}, url="http://localhost") +def test_storage_scheme(storage, expected_scheme, url): + urlfile = URLFile(provider={"storage": storage}, url=url) assert urlfile.storage_scheme == expected_scheme @@ -80,8 +82,27 @@ def test_load_dataframes_xlsb(config, absolute_path, test_files): assert read_file.equals(expected) -def test_load_nested_json(client, absolute_path, test_files): - f = f"{absolute_path}/{test_files}/formats/json/demo.json" +@pytest.mark.parametrize("file_name, should_raise_error", [("test.xlsx", False), ("test_one_line.xlsx", True)]) +def test_load_dataframes_xlsx(config, absolute_path, test_files, file_name, should_raise_error): + config["format"] = "excel" + client = Client(**config) + f = f"{absolute_path}/{test_files}/{file_name}" + if should_raise_error: + with pytest.raises(AirbyteTracedException): + next(client.load_dataframes(fp=f)) + else: + read_file = next(client.load_dataframes(fp=f)) + expected = read_excel(f, engine="openpyxl") + assert read_file.equals(expected) + + +@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), + ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) +def test_load_nested_json(client, config, absolute_path, test_files, file_format, file_path): + if file_format == "jsonl": + config["format"] = file_format + client = Client(**config) + f = f"{absolute_path}/{test_files}/{file_path}" with open(f, mode="rb") as file: assert client.load_nested_json(fp=file) @@ -122,6 +143,11 @@ def test_cache_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv" with open(f, mode="rb") as file: assert client._cache_stream(file) + +def test_unzip_stream(client, absolute_path, test_files): + f = f"{absolute_path}/{test_files}/test.csv.zip" + with open(f, mode="rb") as file: + assert client._unzip(file) def test_open_aws_url(): @@ -204,3 +230,11 @@ def patched_open(self): assert call_count == 7 assert sleep_mock.call_count == 5 + + +def test_backoff_handler(caplog): + details = {"tries": 1, "wait": 1} + backoff_handler(details) + expected = [('airbyte', 20, 'Caught retryable error after 1 tries. Waiting 1 seconds then retrying...')] + + assert caplog.record_tuples == expected diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py index 51f06adb6da5..bed365884b80 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py @@ -24,6 +24,7 @@ from airbyte_cdk.utils import AirbyteTracedException from source_file.client import ConfigurationError +from airbyte_protocol.models.airbyte_protocol import Type as MessageType from source_file.source import SourceFile logger = logging.getLogger("airbyte") @@ -99,7 +100,8 @@ def test_nan_to_null(absolute_path, test_files): source = SourceFile() records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) - records = [r.record.data for r in records] + + records = [r.record.data for r in records if r.type == MessageType.RECORD] assert records == [ {"col1": "key1", "col2": 1.11, "col3": None}, {"col1": "key2", "col2": None, "col3": 2.22}, @@ -109,13 +111,14 @@ def test_nan_to_null(absolute_path, test_files): config.update({"format": "yaml", "url": f"{absolute_path}/{test_files}/formats/yaml/demo.yaml"}) records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) - records = [r.record.data for r in records] + records = [r.record.data for r in records if r.type == MessageType.RECORD] assert records == [] config.update({"provider": {"storage": "SSH", "user": "user", "host": "host"}}) with pytest.raises(Exception): - next(source.read(logger=logger, config=config, catalog=catalog)) + for record in source.read(logger=logger, config=config, catalog=catalog): + pass def test_spec(source): @@ -186,7 +189,7 @@ def test_pandas_header_not_none(absolute_path, test_files): source = SourceFile() records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) - records = [r.record.data for r in records] + records = [r.record.data for r in records if r.type == MessageType.RECORD] assert records == [ {"text11": "text21", "text12": "text22"}, ] @@ -205,7 +208,7 @@ def test_pandas_header_none(absolute_path, test_files): source = SourceFile() records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) - records = [r.record.data for r in records] + records = [r.record.data for r in records if r.type == MessageType.RECORD] assert records == [ {"0": "text11", "1": "text12"}, {"0": "text21", "1": "text22"}, @@ -234,4 +237,4 @@ def test_incorrect_reader_options(absolute_path, test_files): ): catalog = get_catalog({"0": {"type": ["string", "null"]}, "1": {"type": ["string", "null"]}}) records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) - records = [r.record.data for r in records] + records = [r.record.data for r in records if r.type == MessageType.RECORD] diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py index 54d63471838e..708648fa8c15 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_firebase_realtime_database import SourceFirebaseRealtimeDatabase +from source_firebase_realtime_database.run import run if __name__ == "__main__": - source = SourceFirebaseRealtimeDatabase() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml index abd8e66c3414..3d694aa9f2f3 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml @@ -10,6 +10,10 @@ data: githubIssueLabel: source-firebase-realtime-database license: MIT name: Firebase Realtime Database + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-firebase-realtime-database registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py index 1424bb5b1b68..780ac7c466df 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py @@ -18,13 +18,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-firebase-realtime-database=source_firebase_realtime_database.run:run", + ], + }, name="source_firebase_realtime_database", description="Source implementation for Firebase Realtime Database.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py new file mode 100644 index 000000000000..19b835a4a7a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_firebase_realtime_database import SourceFirebaseRealtimeDatabase + + +def run(): + source = SourceFirebaseRealtimeDatabase() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-firebolt/main.py b/airbyte-integrations/connectors/source-firebolt/main.py index babb5aad001b..a901e9c4ae29 100644 --- a/airbyte-integrations/connectors/source-firebolt/main.py +++ b/airbyte-integrations/connectors/source-firebolt/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_firebolt import SourceFirebolt +from source_firebolt.run import run if __name__ == "__main__": - source = SourceFirebolt() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-firebolt/metadata.yaml b/airbyte-integrations/connectors/source-firebolt/metadata.yaml index d56f04699b78..824d857b5d7e 100644 --- a/airbyte-integrations/connectors/source-firebolt/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebolt/metadata.yaml @@ -8,6 +8,10 @@ data: icon: firebolt.svg license: MIT name: Firebolt + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-firebolt registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-firebolt/setup.py b/airbyte-integrations/connectors/source-firebolt/setup.py index 13bb7d102b18..3e7be0197a3b 100644 --- a/airbyte-integrations/connectors/source-firebolt/setup.py +++ b/airbyte-integrations/connectors/source-firebolt/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-firebolt=source_firebolt.run:run", + ], + }, name="source_firebolt", description="Source implementation for Firebolt.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py new file mode 100644 index 000000000000..d37e5a6ba911 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_firebolt import SourceFirebolt + + +def run(): + source = SourceFirebolt() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-flexport/main.py b/airbyte-integrations/connectors/source-flexport/main.py index e65198a5e67d..2370e6f8868a 100644 --- a/airbyte-integrations/connectors/source-flexport/main.py +++ b/airbyte-integrations/connectors/source-flexport/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_flexport import SourceFlexport +from source_flexport.run import run if __name__ == "__main__": - source = SourceFlexport() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-flexport/metadata.yaml b/airbyte-integrations/connectors/source-flexport/metadata.yaml index deb99f7ca238..88b125bacb23 100644 --- a/airbyte-integrations/connectors/source-flexport/metadata.yaml +++ b/airbyte-integrations/connectors/source-flexport/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - api.flexport.com - flexport.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-flexport registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-flexport/setup.py b/airbyte-integrations/connectors/source-flexport/setup.py index 0deaf76a6445..8a0a2f201f9b 100644 --- a/airbyte-integrations/connectors/source-flexport/setup.py +++ b/airbyte-integrations/connectors/source-flexport/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-flexport=source_flexport.run:run", + ], + }, name="source_flexport", description="Source implementation for Flexport.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/run.py b/airbyte-integrations/connectors/source-flexport/source_flexport/run.py new file mode 100644 index 000000000000..9bba4b7e96a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_flexport import SourceFlexport + + +def run(): + source = SourceFlexport() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py index e0bc9f142e11..7039ceb25a6d 100644 --- a/airbyte-integrations/connectors/source-freshcaller/main.py +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshcaller import SourceFreshcaller +from source_freshcaller.run import run if __name__ == "__main__": - source = SourceFreshcaller() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml index 5d8dc4615aa9..15eaf3b06ea1 100644 --- a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml @@ -8,6 +8,10 @@ data: icon: freshcaller.svg license: MIT name: Freshcaller + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshcaller registries: cloud: enabled: true @@ -17,7 +21,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshcaller tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py index 27b47c30913f..3c1ee9c3ca91 100644 --- a/airbyte-integrations/connectors/source-freshcaller/setup.py +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-freshcaller=source_freshcaller.run:run", + ], + }, name="source_freshcaller", description="Source implementation for Freshcaller.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py new file mode 100644 index 000000000000..b6757d75d1aa --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshcaller import SourceFreshcaller + + +def run(): + source = SourceFreshcaller() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshdesk/README.md b/airbyte-integrations/connectors/source-freshdesk/README.md index b96f514b3d95..077bb798e93f 100644 --- a/airbyte-integrations/connectors/source-freshdesk/README.md +++ b/airbyte-integrations/connectors/source-freshdesk/README.md @@ -1,4 +1,5 @@ -# Freshdesk Source +# Freshdesk source connector + This is the repository for the Freshdesk source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/freshdesk). @@ -6,114 +7,49 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials +### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/freshdesk) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshdesk/spec.yaml` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshdesk test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-freshdesk spec +poetry run source-freshdesk check --config secrets/config.json +poetry run source-freshdesk discover --config secrets/config.json +poetry run source-freshdesk read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: - -```bash -airbyte-ci connectors --name source-freshdesk build +### Running unit tests +To run unit tests locally, from the connector directory run: ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-freshdesk:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +poetry run pytest unit_tests ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-freshdesk:latest +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-freshdesk build +``` -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-freshdesk:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-freshdesk:dev . -# Running the spec command against your patched connector -docker run airbyte/source-freshdesk:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-freshdesk:dev spec @@ -122,28 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshdesk:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshdesk:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-freshdesk test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-freshdesk test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/freshdesk.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/freshdesk.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/main.py b/airbyte-integrations/connectors/source-freshdesk/main.py index 319505ff4bb5..d32eaa6ca9e5 100644 --- a/airbyte-integrations/connectors/source-freshdesk/main.py +++ b/airbyte-integrations/connectors/source-freshdesk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshdesk import SourceFreshdesk +from source_freshdesk.run import run if __name__ == "__main__": - source = SourceFreshdesk() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml index 4a9cd521ff4a..781a09a40dae 100644 --- a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 - dockerImageTag: 3.0.5 + dockerImageTag: 3.0.7 dockerRepository: airbyte/source-freshdesk documentationUrl: https://docs.airbyte.com/integrations/sources/freshdesk githubIssueLabel: source-freshdesk icon: freshdesk.svg license: MIT name: Freshdesk + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshdesk registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-freshdesk/poetry.lock b/airbyte-integrations/connectors/source-freshdesk/poetry.lock new file mode 100644 index 000000000000..1e53023de5b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.7" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.7.tar.gz", hash = "sha256:00e379e2379b38683992027114a2190f49befec8cbac67d0a2c907786111e77b"}, + {file = "airbyte_cdk-0.58.7-py3-none-any.whl", hash = "sha256:09b31d32899cc6dc91e39716e8d1601503a7884d837752e683d1e3ef7dfe73be"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "1.10.0" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"}, + {file = "backoff-1.10.0.tar.gz", hash = "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "639f5cfef16b8664432e1a27660a1c5dd290c0103e4e3741e2ece95f35f425e4" diff --git a/airbyte-integrations/connectors/source-freshdesk/pyproject.toml b/airbyte-integrations/connectors/source-freshdesk/pyproject.toml new file mode 100644 index 000000000000..48527aa9bc86 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.0.7" +name = "source-freshdesk" +description = "Source implementation for Freshdesk." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/freshdesk" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_freshdesk" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +backoff = "==1.10.0" +airbyte-cdk = "==0.58.7" + +[tool.poetry.scripts] +source-freshdesk = "source_freshdesk.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.2" +pytest-mock = "^3.6" +requests-mock = "^1.11.0" diff --git a/airbyte-integrations/connectors/source-freshdesk/requirements.txt b/airbyte-integrations/connectors/source-freshdesk/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py deleted file mode 100644 index c0d8b408f781..000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "backoff==1.10.0", "requests==2.25.1"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.2", - "pytest-mock~=3.6", - "requests_mock~=1.9.3", -] - -setup( - name="source_freshdesk", - description="Source implementation for Freshdesk.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py new file mode 100644 index 000000000000..5486a3c15061 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshdesk import SourceFreshdesk + + +def run(): + source = SourceFreshdesk() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshsales/main.py b/airbyte-integrations/connectors/source-freshsales/main.py index c5069e9490e2..cdb22eb76f98 100644 --- a/airbyte-integrations/connectors/source-freshsales/main.py +++ b/airbyte-integrations/connectors/source-freshsales/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshsales import SourceFreshsales +from source_freshsales.run import run if __name__ == "__main__": - source = SourceFreshsales() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index afb486eb0a95..5abeea00cab7 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -20,6 +20,10 @@ data: icon: freshsales.svg license: MIT name: Freshsales + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshsales registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-freshsales/setup.py b/airbyte-integrations/connectors/source-freshsales/setup.py index 2cc1107f0c8b..664bc7a4ff49 100644 --- a/airbyte-integrations/connectors/source-freshsales/setup.py +++ b/airbyte-integrations/connectors/source-freshsales/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-freshsales=source_freshsales.run:run", + ], + }, name="source_freshsales", description="Source implementation for Freshsales.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py b/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py new file mode 100644 index 000000000000..5eed96da068e --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshsales import SourceFreshsales + + +def run(): + source = SourceFreshsales() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshservice/Dockerfile b/airbyte-integrations/connectors/source-freshservice/Dockerfile index 7732b3d3d243..3925f6c26c42 100644 --- a/airbyte-integrations/connectors/source-freshservice/Dockerfile +++ b/airbyte-integrations/connectors/source-freshservice/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshservice ./source_freshservice ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.2.0 +LABEL io.airbyte.version=1.3.1 LABEL io.airbyte.name=airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json index 88cbb244c9f1..14bfe7e98d95 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json @@ -12,28 +12,37 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "requested_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "problems", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { "name": "changes", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { diff --git a/airbyte-integrations/connectors/source-freshservice/main.py b/airbyte-integrations/connectors/source-freshservice/main.py index 5048dde91929..084c02978df9 100644 --- a/airbyte-integrations/connectors/source-freshservice/main.py +++ b/airbyte-integrations/connectors/source-freshservice/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshservice import SourceFreshservice +from source_freshservice.run import run if __name__ == "__main__": - source = SourceFreshservice() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 61c73b4c6b10..4262773507e8 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -1,16 +1,20 @@ data: allowedHosts: hosts: - - TODO # Please change to the hostname of the source. + - ${domain_name}/api/v2 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshservice registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: api connectorType: source definitionId: 9bb85338-ea95-4c93-b267-6be89125b267 - dockerImageTag: 1.2.0 + dockerImageTag: 1.3.1 dockerRepository: airbyte/source-freshservice githubIssueLabel: source-freshservice icon: freshservice.svg @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshservice/setup.py b/airbyte-integrations/connectors/source-freshservice/setup.py index 8ccedf94055a..87806214b3dc 100644 --- a/airbyte-integrations/connectors/source-freshservice/setup.py +++ b/airbyte-integrations/connectors/source-freshservice/setup.py @@ -6,23 +6,36 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.55.2", ] -TEST_REQUIREMENTS = [ - "pytest~=6.2", - "pytest-mock~=3.6.1", - "connector-acceptance-test", -] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-freshservice=source_freshservice.run:run", + ], + }, name="source_freshservice", description="Source implementation for Freshservice.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml index fb2cbfe386c3..dab837f5883a 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml @@ -78,12 +78,42 @@ definitions: parent_key: "id" partition_field: "parent_id" + requested_items_stream: + name: "requested_items" + primary_key: "id" + $parameters: + path_extractor: "requested_items" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "tickets/{{ stream_slice.parent_id }}/requested_items" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + error_message: No data collected + - type: DefaultErrorHandler + response_filters: + - http_codes: [429] + action: RETRY + backoff_strategies: + - type: WaitTimeFromHeader + header: "Retry-After" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/tickets_stream" + parent_key: "id" + partition_field: "parent_id" + problems_stream: $ref: "#/definitions/base_stream" name: "problems" primary_key: "id" - incremental_sync: - $ref: "#/definitions/incremental_base" $parameters: path_extractor: "problems" path: "/problems" @@ -92,8 +122,6 @@ definitions: $ref: "#/definitions/base_stream" name: "changes" primary_key: "id" - incremental_sync: - $ref: "#/definitions/incremental_base" $parameters: path_extractor: "changes" path: "/changes" @@ -186,6 +214,7 @@ streams: - "#/definitions/assets_stream" - "#/definitions/purchase_orders_stream" - "#/definitions/software_stream" + - "#/definitions/requested_items_stream" check: type: CheckStream diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py new file mode 100644 index 000000000000..c7a979f0a59b --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshservice import SourceFreshservice + + +def run(): + source = SourceFreshservice() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json index 005af789f286..1b2d07f0fe09 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json @@ -27,6 +27,12 @@ "mobile_phone_number": { "type": ["null", "string"] }, + "member_of_pending_approval": { + "type": ["null", "array"] + }, + "observer_of_pending_approval": { + "type": ["null", "array"] + }, "department_ids": { "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json new file mode 100644 index 000000000000..909a9ed2621c --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "stage": { + "type": ["null", "integer"] + }, + "loaned": { + "type": ["null", "boolean"] + }, + "cost_per_request": { + "type": ["null", "number"] + }, + "remarks": { + "type": ["null", "string"] + }, + "delivery_time": { + "type": ["null", "number"] + }, + "is_parent": { + "type": ["null", "boolean"] + }, + "service_item_id": { + "type": ["null", "integer"] + }, + "service_item_name": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + } + } +} diff --git a/airbyte-integrations/connectors/source-fullstory/main.py b/airbyte-integrations/connectors/source-fullstory/main.py index d9696f00cd3d..ee3e71e54ed5 100644 --- a/airbyte-integrations/connectors/source-fullstory/main.py +++ b/airbyte-integrations/connectors/source-fullstory/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fullstory import SourceFullstory +from source_fullstory.run import run if __name__ == "__main__": - source = SourceFullstory() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fullstory/metadata.yaml b/airbyte-integrations/connectors/source-fullstory/metadata.yaml index a645935c33eb..c2083f3631c8 100644 --- a/airbyte-integrations/connectors/source-fullstory/metadata.yaml +++ b/airbyte-integrations/connectors/source-fullstory/metadata.yaml @@ -8,6 +8,10 @@ data: icon: fullstory.svg license: MIT name: Fullstory + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fullstory registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-fullstory/setup.py b/airbyte-integrations/connectors/source-fullstory/setup.py index 99bc576b124a..4bc344fd16c3 100644 --- a/airbyte-integrations/connectors/source-fullstory/setup.py +++ b/airbyte-integrations/connectors/source-fullstory/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-fullstory=source_fullstory.run:run", + ], + }, name="source_fullstory", description="Source implementation for Fullstory.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py b/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py new file mode 100644 index 000000000000..3b6be606bf59 --- /dev/null +++ b/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fullstory import SourceFullstory + + +def run(): + source = SourceFullstory() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gainsight-px/main.py b/airbyte-integrations/connectors/source-gainsight-px/main.py index 5ae4980cd0e0..35146e9ca972 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/main.py +++ b/airbyte-integrations/connectors/source-gainsight-px/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gainsight_px import SourceGainsightPx +from source_gainsight_px.run import run if __name__ == "__main__": - source = SourceGainsightPx() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml index 2d340cc0adaf..0a65633f7dcc 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml +++ b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.aptrinsic.com/v1 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gainsight-px registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gainsight-px/setup.py b/airbyte-integrations/connectors/source-gainsight-px/setup.py index 3ba161a38e22..4510521fa81c 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/setup.py +++ b/airbyte-integrations/connectors/source-gainsight-px/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gainsight-px=source_gainsight_px.run:run", + ], + }, name="source_gainsight_px", description="Source implementation for Gainsight Px.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py new file mode 100644 index 000000000000..f0e263867dd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gainsight_px import SourceGainsightPx + + +def run(): + source = SourceGainsightPx() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json index 560361ec3b30..5f69da41c02a 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json @@ -58,9 +58,9 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", - "type": "string", - "airbyte_hidden": true + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "airbyte_hidden": true, + "type": "string" }, "days_to_sync_if_history_is_full": { "title": "Days To Sync If History Is Full", diff --git a/airbyte-integrations/connectors/source-gcs/main.py b/airbyte-integrations/connectors/source-gcs/main.py index c98b5b943cc7..a3a044fb142d 100644 --- a/airbyte-integrations/connectors/source-gcs/main.py +++ b/airbyte-integrations/connectors/source-gcs/main.py @@ -2,14 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader +from source_gcs.run import run if __name__ == "__main__": - _args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(_args) - source = SourceGCS(SourceGCSStreamReader(), Config, catalog_path, cursor_cls=Cursor) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index e9364232fbd2..abda6f594cca 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,13 +7,17 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.7 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs icon: gcs.svg license: ELv2 name: GCS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gcs registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index b9574a838971..3bb310b4fbf0 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.55.5", + "airbyte-cdk[file-based]>=0.61.0", "google-cloud-storage==2.12.0", "smart-open[s3]==5.1.0", "pandas==1.5.3", @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gcs=source_gcs.run:run", + ], + }, name="source_gcs", description="Source implementation for Gcs.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py index ebd1117841e1..04dc1e16b8d3 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py @@ -80,3 +80,7 @@ def replace_enum_allOf_and_anyOf(schema): objects_to_check["anyOf"] = objects_to_check.pop("allOf") return super(Config, Config).replace_enum_allOf_and_anyOf(schema) + + @staticmethod + def remove_discriminator(schema) -> None: + pass diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py new file mode 100644 index 000000000000..c4536b2d14e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader + + +def run(): + _args = sys.argv[1:] + try: + catalog_path = AirbyteEntrypoint.extract_catalog(_args) + config_path = AirbyteEntrypoint.extract_config(_args) + state_path = AirbyteEntrypoint.extract_state(_args) + source = SourceGCS( + SourceGCSStreamReader(), + Config, + SourceGCS.read_catalog(catalog_path) if catalog_path else None, + SourceGCS.read_config(config_path) if config_path else None, + SourceGCS.read_state(state_path) if state_path else None, + cursor_cls=Cursor, + ) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + else: + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py index b33ff2b87eb4..7152d9ec5e11 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py @@ -12,13 +12,14 @@ class SourceGCS(FileBasedSource): - def read_config(self, config_path: str) -> Mapping[str, Any]: + @classmethod + def read_config(cls, config_path: str) -> Mapping[str, Any]: """ Override the default read_config to transform the legacy config format into the new one before validating it against the new spec. """ - config = super().read_config(config_path) - if not self._is_file_based_config(config): + config = FileBasedSource.read_config(config_path) + if not cls._is_file_based_config(config): parsed_legacy_config = SourceGCSSpec(**config) converted_config = LegacyConfigTransformer.convert(parsed_legacy_config) emit_configuration_as_airbyte_control_message(converted_config) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py index 3552d75980fd..ec44dd27048e 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py @@ -5,7 +5,6 @@ import itertools import json import logging -from contextlib import contextmanager from datetime import datetime, timedelta from io import IOBase from typing import Iterable, List, Optional @@ -94,7 +93,6 @@ def _handle_file_listing_error(self, exc: Exception, prefix: str, logger: loggin prefix=prefix, ) from exc - @contextmanager def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: """ Open and yield a remote file from GCS for reading. @@ -105,7 +103,4 @@ def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str except OSError as oe: logger.warning(ERROR_MESSAGE_ACCESS.format(uri=file.uri, bucket=self.config.bucket)) logger.exception(oe) - try: - yield result - finally: - result.close() + return result diff --git a/airbyte-integrations/connectors/source-genesys/main.py b/airbyte-integrations/connectors/source-genesys/main.py index 603590f5fe12..d34643d2aa21 100644 --- a/airbyte-integrations/connectors/source-genesys/main.py +++ b/airbyte-integrations/connectors/source-genesys/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_genesys import SourceGenesys +from source_genesys.run import run if __name__ == "__main__": - source = SourceGenesys() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-genesys/metadata.yaml b/airbyte-integrations/connectors/source-genesys/metadata.yaml index f6522beea79e..0bad42c982d3 100644 --- a/airbyte-integrations/connectors/source-genesys/metadata.yaml +++ b/airbyte-integrations/connectors/source-genesys/metadata.yaml @@ -8,6 +8,10 @@ data: icon: genesys.svg license: MIT name: Genesys + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-genesys registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-genesys/setup.py b/airbyte-integrations/connectors/source-genesys/setup.py index 1c118e316680..723222c413e3 100644 --- a/airbyte-integrations/connectors/source-genesys/setup.py +++ b/airbyte-integrations/connectors/source-genesys/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-genesys=source_genesys.run:run", + ], + }, name="source_genesys", description="Source implementation for Genesys.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/run.py b/airbyte-integrations/connectors/source-genesys/source_genesys/run.py new file mode 100644 index 000000000000..1c7ee790a0ca --- /dev/null +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_genesys import SourceGenesys + + +def run(): + source = SourceGenesys() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-getlago/main.py b/airbyte-integrations/connectors/source-getlago/main.py index 0748c658182d..979e8b779cc5 100644 --- a/airbyte-integrations/connectors/source-getlago/main.py +++ b/airbyte-integrations/connectors/source-getlago/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_getlago import SourceGetlago +from source_getlago.run import run if __name__ == "__main__": - source = SourceGetlago() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-getlago/metadata.yaml b/airbyte-integrations/connectors/source-getlago/metadata.yaml index 70779de3b016..de14460248c0 100644 --- a/airbyte-integrations/connectors/source-getlago/metadata.yaml +++ b/airbyte-integrations/connectors/source-getlago/metadata.yaml @@ -8,6 +8,10 @@ data: icon: getlago.svg license: MIT name: Lago + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-getlago registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-getlago/setup.py b/airbyte-integrations/connectors/source-getlago/setup.py index 37d77842438d..b4d2e1a140eb 100644 --- a/airbyte-integrations/connectors/source-getlago/setup.py +++ b/airbyte-integrations/connectors/source-getlago/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-getlago=source_getlago.run:run", + ], + }, name="source_getlago", description="Source implementation for Getlago.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/run.py b/airbyte-integrations/connectors/source-getlago/source_getlago/run.py new file mode 100644 index 000000000000..a7822c1b112a --- /dev/null +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_getlago import SourceGetlago + + +def run(): + source = SourceGetlago() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-github/README.md b/airbyte-integrations/connectors/source-github/README.md index 0a0aad08fcc4..1a60fa938028 100644 --- a/airbyte-integrations/connectors/source-github/README.md +++ b/airbyte-integrations/connectors/source-github/README.md @@ -1,119 +1,55 @@ -# Github Source +# Github source connector + This is the repository for the Github source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/github). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/github). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/github) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_github/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/github) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_github/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source github test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-github spec +poetry run source-github check --config secrets/config.json +poetry run source-github discover --config secrets/config.json +poetry run source-github read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-github build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-github:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-github:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-github:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-github:dev . -# Running the spec command against your patched connector -docker run airbyte/source-github:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-github:dev spec @@ -122,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-github:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-github:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-github test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-github test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/github.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/github.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index 2ff5a27c1f45..5ee30aee5036 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -32,7 +32,7 @@ acceptance_tests: extra_records: yes empty_streams: - name: "events" - bypass_reason: "Only events created within the past 90 days can be showed" + bypass_reason: "Only events created within the past 90 days can be showed. Stream is tested with integration tests." ignored_fields: contributor_activity: - name: weeks diff --git a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl index 9b65df5c424e..8a9f03a3f05d 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl @@ -1,6 +1,6 @@ {"stream":"assignees","data":{"login":"AirbyteEricksson","id":101604444,"node_id":"U_kgDOBg5cXA","avatar_url":"https://avatars.githubusercontent.com/u/101604444?v=4","gravatar_id":"","url":"https://api.github.com/users/AirbyteEricksson","html_url":"https://github.com/AirbyteEricksson","followers_url":"https://api.github.com/users/AirbyteEricksson/followers","following_url":"https://api.github.com/users/AirbyteEricksson/following{/other_user}","gists_url":"https://api.github.com/users/AirbyteEricksson/gists{/gist_id}","starred_url":"https://api.github.com/users/AirbyteEricksson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AirbyteEricksson/subscriptions","organizations_url":"https://api.github.com/users/AirbyteEricksson/orgs","repos_url":"https://api.github.com/users/AirbyteEricksson/repos","events_url":"https://api.github.com/users/AirbyteEricksson/events{/privacy}","received_events_url":"https://api.github.com/users/AirbyteEricksson/received_events","type":"User","site_admin":false,"repository":"airbytehq/integration-test"},"emitted_at":1677668743181} -{"stream":"branches","data":{"name":"feature/branch_0","commit":{"sha":"cbbeaf3ef6eb7217052eae2fe665f655e3813973","url":"https://api.github.com/repos/airbytehq/integration-test/commits/cbbeaf3ef6eb7217052eae2fe665f655e3813973"},"protected":false,"repository":"airbytehq/integration-test"},"emitted_at":1677668743768} -{"stream":"collaborators","data":{"login": "octavia-approvington", "id": 117117241, "node_id": "U_kgDOBvsROQ", "avatar_url": "https://avatars.githubusercontent.com/u/117117241?v=4", "gravatar_id": "", "url": "https://api.github.com/users/octavia-approvington", "html_url": "https://github.com/octavia-approvington", "followers_url": "https://api.github.com/users/octavia-approvington/followers", "following_url": "https://api.github.com/users/octavia-approvington/following{/other_user}", "gists_url": "https://api.github.com/users/octavia-approvington/gists{/gist_id}", "starred_url": "https://api.github.com/users/octavia-approvington/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/octavia-approvington/subscriptions", "organizations_url": "https://api.github.com/users/octavia-approvington/orgs", "repos_url": "https://api.github.com/users/octavia-approvington/repos", "events_url": "https://api.github.com/users/octavia-approvington/events{/privacy}", "received_events_url": "https://api.github.com/users/octavia-approvington/received_events", "type": "User", "site_admin": false, "permissions": {"admin": false, "maintain": false, "push": true, "triage": true, "pull": true}, "role_name": "write", "repository": "airbytehq/integration-test"},"emitted_at":1677668744171} +{"stream":"branches", "data": {"name": "feature/branch_0", "commit": {"sha": "cbbeaf3ef6eb7217052eae2fe665f655e3813973", "url": "https://api.github.com/repos/airbytehq/integration-test/commits/cbbeaf3ef6eb7217052eae2fe665f655e3813973"}, "protected": false, "protection": {"enabled": false, "required_status_checks": {"enforcement_level": "off", "contexts": [], "checks": []}}, "protection_url": "https://api.github.com/repos/airbytehq/integration-test/branches/feature/branch_0/protection", "repository": "airbytehq/integration-test"}, "emitted_at": 1707933365457} +{"stream":"collaborators","data":{"login": "octavia-approvington", "id": 117117241, "node_id": "U_kgDOBvsROQ", "avatar_url": "https://avatars.githubusercontent.com/u/117117241?v=4", "gravatar_id": "", "url": "https://api.github.com/users/octavia-approvington", "html_url": "https://github.com/octavia-approvington", "followers_url": "https://api.github.com/users/octavia-approvington/followers", "following_url": "https://api.github.com/users/octavia-approvington/following{/other_user}", "gists_url": "https://api.github.com/users/octavia-approvington/gists{/gist_id}", "starred_url": "https://api.github.com/users/octavia-approvington/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/octavia-approvington/subscriptions", "organizations_url": "https://api.github.com/users/octavia-approvington/orgs", "repos_url": "https://api.github.com/users/octavia-approvington/repos", "events_url": "https://api.github.com/users/octavia-approvington/events{/privacy}", "received_events_url": "https://api.github.com/users/octavia-approvington/received_events", "type": "User", "site_admin": false, "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "role_name": "admin", "repository": "airbytehq/integration-test"},"emitted_at":1677668744171} {"stream":"comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167","html_url":"https://github.com/airbytehq/integration-test/issues/6#issuecomment-907296167","issue_url":"https://api.github.com/repos/airbytehq/integration-test/issues/6","id":907296167,"node_id":"IC_kwDOF9hP9c42FD2n","user":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"created_at":"2021-08-27T15:43:59Z","updated_at":"2021-08-27T15:43:59Z","author_association":"CONTRIBUTOR","body":"comment for issues https://api.github.com/repos/airbytehq/integration-test/issues/6/comments","reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"performed_via_github_app":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668744803} {"stream":"commit_comment_reactions","data":{"id":154935429,"node_id":"REA_lADOF9hP9c4DT3SJzgk8IIU","user":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"content":"laugh","created_at":"2022-03-20T11:29:29Z","repository":"airbytehq/integration-test","comment_id":55538825},"emitted_at":1677668746490} {"stream":"commit_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/comments/55538825","html_url":"https://github.com/airbytehq/integration-test/commit/cbbeaf3ef6eb7217052eae2fe665f655e3813973#commitcomment-55538825","id":55538825,"node_id":"MDEzOkNvbW1pdENvbW1lbnQ1NTUzODgyNQ==","user":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"position":null,"line":null,"path":null,"commit_id":"cbbeaf3ef6eb7217052eae2fe665f655e3813973","created_at":"2021-08-27T15:43:32Z","updated_at":"2021-08-27T15:43:32Z","author_association":"CONTRIBUTOR","body":"comment for cbbeaf3ef6eb7217052eae2fe665f655e3813973 branch","reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/comments/55538825/reactions","total_count":2,"+1":0,"-1":0,"laugh":1,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"repository":"airbytehq/integration-test"},"emitted_at":1677668747441} @@ -24,7 +24,7 @@ {"stream": "pull_request_stats", "data": {"node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "id": 721435506, "number": 5, "updated_at": "2023-11-16T14:38:58Z", "changed_files": 5, "deletions": 0, "additions": 5, "merged": false, "mergeable": "MERGEABLE", "can_be_rebased": false, "maintainer_can_modify": false, "merge_state_status": "BLOCKED", "comments": 0, "commits": 5, "review_comments": 0, "merged_by": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700557306144} {"stream": "pull_requests", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5", "id": 721435506, "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "html_url": "https://github.com/airbytehq/integration-test/pull/5", "diff_url": "https://github.com/airbytehq/integration-test/pull/5.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/5.patch", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5", "number": 5, "state": "closed", "locked": false, "title": "New PR from feature/branch_4", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "body": null, "created_at": "2021-08-27T15:43:40Z", "updated_at": "2023-11-16T14:38:58Z", "closed_at": "2023-11-16T14:38:58Z", "merged_at": null, "merge_commit_sha": "191309e3da8b36705156348ae73f4dca836533f9", "assignee": null, "assignees": [], "requested_reviewers": [], "requested_teams": [], "labels": [{"id": 3295756566, "node_id": "MDU6TGFiZWwzMjk1NzU2NTY2", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/bug", "name": "bug", "color": "d73a4a", "default": true, "description": "Something isn't working"}, {"id": 3300346197, "node_id": "MDU6TGFiZWwzMzAwMzQ2MTk3", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/critical", "name": "critical", "color": "ededed", "default": false, "description": null}], "milestone": null, "draft": false, "commits_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits", "review_comments_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments", "review_comment_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "head": {"label": "airbytehq:feature/branch_4", "ref": "feature/branch_4", "sha": "31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo_id": 400052213}, "base": {"label": "airbytehq:master", "ref": "master", "sha": "978753aeb56f7b49872279d1b491411a6235aa90", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments", "created_at": "2021-08-26T05:32:43Z", "updated_at": "2023-11-16T14:48:53Z", "pushed_at": "2023-05-03T16:40:56Z", "git_url": "git://github.com/airbytehq/integration-test.git", "ssh_url": "git@github.com:airbytehq/integration-test.git", "clone_url": "https://github.com/airbytehq/integration-test.git", "svn_url": "https://github.com/airbytehq/integration-test", "homepage": null, "size": 11, "stargazers_count": 4, "watchers_count": 4, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 6, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 2, "open_issues": 6, "watchers": 4, "default_branch": "master"}, "repo_id": null}, "_links": {"self": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5"}, "html": {"href": "https://github.com/airbytehq/integration-test/pull/5"}, "issue": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5"}, "comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments"}, "review_comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments"}, "review_comment": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}"}, "commits": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits"}, "statuses": {"href": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083"}}, "author_association": "CONTRIBUTOR", "auto_merge": null, "active_lock_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700585060024} {"stream":"releases","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586","assets_url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586/assets","upload_url":"https://uploads.github.com/repos/airbytehq/integration-test/releases/48581586/assets{?name,label}","html_url":"https://github.com/airbytehq/integration-test/releases/tag/dev-0.9","id":48581586,"author":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"node_id":"MDc6UmVsZWFzZTQ4NTgxNTg2","tag_name":"dev-0.9","target_commitish":"master","name":"9 global release","draft":false,"prerelease":false,"created_at":"2021-08-27T07:03:09Z","published_at":"2021-08-27T15:43:53Z","assets":[],"tarball_url":"https://api.github.com/repos/airbytehq/integration-test/tarball/dev-0.9","zipball_url":"https://api.github.com/repos/airbytehq/integration-test/zipball/dev-0.9","body":"","repository":"airbytehq/integration-test"},"emitted_at":1677668760424} -{"stream": "repositories", "data": {"id": 283046497, "node_id": "MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=", "name": "airbyte", "full_name": "airbytehq/airbyte", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/airbyte", "description": "Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.", "fork": false, "url": "https://api.github.com/repos/airbytehq/airbyte", "forks_url": "https://api.github.com/repos/airbytehq/airbyte/forks", "keys_url": "https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/airbyte/teams", "hooks_url": "https://api.github.com/repos/airbytehq/airbyte/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/airbyte/events", "assignees_url": "https://api.github.com/repos/airbytehq/airbyte/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/airbyte/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/airbyte/tags", "blobs_url": "https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/airbyte/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/airbyte/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/airbyte/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/airbyte/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/airbyte/subscription", "commits_url": "https://api.github.com/repos/airbytehq/airbyte/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/airbyte/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/airbyte/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/airbyte/merges", "archive_url": "https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/airbyte/downloads", "issues_url": "https://api.github.com/repos/airbytehq/airbyte/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/airbyte/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/airbyte/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/airbyte/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/airbyte/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/airbyte/deployments", "created_at": "2020-07-27T23:55:54Z", "updated_at": "2023-11-21T14:55:05Z", "pushed_at": "2023-11-21T16:55:37Z", "git_url": "git://github.com/airbytehq/airbyte.git", "ssh_url": "git@github.com:airbytehq/airbyte.git", "clone_url": "https://github.com/airbytehq/airbyte.git", "svn_url": "https://github.com/airbytehq/airbyte", "homepage": "https://airbyte.com", "size": 455477, "stargazers_count": 12328, "watchers_count": 12328, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": false, "has_discussions": true, "forks_count": 3226, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 5053, "license": {"key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA="}, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": ["airbyte", "bigquery", "change-data-capture", "data", "data-analysis", "data-collection", "data-engineering", "data-ingestion", "data-integration", "elt", "etl", "java", "pipeline", "python", "redshift", "snowflake"], "visibility": "public", "forks": 3226, "open_issues": 5053, "watchers": 12328, "default_branch": "master", "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "security_and_analysis": {"secret_scanning": {"status": "disabled"}, "secret_scanning_push_protection": {"status": "disabled"}, "dependabot_security_updates": {"status": "enabled"}, "secret_scanning_validity_checks": {"status": "disabled"}}, "organization": "airbytehq"}, "emitted_at": 1700585836592} +{"stream":"repositories","data":{"id":283046497,"node_id":"MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=","name":"airbyte","full_name":"airbytehq/airbyte","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/airbyte","description":"The leading data integration platform for ETL / ELT data pipelines from APIs, databases & files to data warehouses, data lakes & data lakehouses. Both self-hosted and Cloud-hosted.","fork":false,"url":"https://api.github.com/repos/airbytehq/airbyte","forks_url":"https://api.github.com/repos/airbytehq/airbyte/forks","keys_url":"https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/airbyte/teams","hooks_url":"https://api.github.com/repos/airbytehq/airbyte/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/airbyte/events","assignees_url":"https://api.github.com/repos/airbytehq/airbyte/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/airbyte/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/airbyte/tags","blobs_url":"https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/airbyte/languages","stargazers_url":"https://api.github.com/repos/airbytehq/airbyte/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/airbyte/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/airbyte/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/airbyte/subscription","commits_url":"https://api.github.com/repos/airbytehq/airbyte/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/airbyte/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/airbyte/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/airbyte/merges","archive_url":"https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/airbyte/downloads","issues_url":"https://api.github.com/repos/airbytehq/airbyte/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/airbyte/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/airbyte/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/airbyte/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/airbyte/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/airbyte/deployments","created_at":"2020-07-27T23:55:54Z","updated_at":"2024-01-26T13:38:04Z","pushed_at":"2024-01-26T13:46:31Z","git_url":"git://github.com/airbytehq/airbyte.git","ssh_url":"git@github.com:airbytehq/airbyte.git","clone_url":"https://github.com/airbytehq/airbyte.git","svn_url":"https://github.com/airbytehq/airbyte","homepage":"https://airbyte.com","size":486685,"stargazers_count":12924,"watchers_count":12924,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"has_discussions":true,"forks_count":3381,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":5107,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"allow_forking":true,"is_template":false,"web_commit_signoff_required":false,"topics":["bigquery","change-data-capture","data","data-analysis","data-collection","data-engineering","data-integration","data-pipeline","elt","etl","java","mssql","mysql","pipeline","postgresql","python","redshift","s3","self-hosted","snowflake"],"visibility":"public","forks":3381,"open_issues":5107,"watchers":12924,"default_branch":"master","permissions":{"admin":true,"maintain":true,"push":true,"triage":true,"pull":true},"security_and_analysis":{"secret_scanning":{"status":"disabled"},"secret_scanning_push_protection":{"status":"disabled"},"dependabot_security_updates":{"status":"enabled"},"secret_scanning_validity_checks":{"status":"disabled"}},"organization":"airbytehq"},"emitted_at":1706276794871} {"stream":"review_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726","pull_request_review_id":742633128,"id":699253726,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDY5OTI1MzcyNg==","diff_hunk":"@@ -0,0 +1 @@\n+text_for_file_","path":"github_sources/file_1.txt","commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","original_commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"body":"Good point","created_at":"2021-08-31T12:01:15Z","updated_at":"2021-08-31T12:01:15Z","html_url":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726","pull_request_url":"https://api.github.com/repos/airbytehq/integration-test/pulls/4","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726"},"html":{"href":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726"},"pull_request":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/4"}},"reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726/reactions","total_count":1,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"start_line":null,"original_start_line":null,"start_side":null,"line":1,"original_line":1,"side":"RIGHT","original_position":1,"position":1,"subject_type":"line","repository":"airbytehq/integration-test"},"emitted_at":1695375624151} {"stream":"reviews","data":{"node_id":"MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NzQwNjU5Nzk4","id":740659798,"body":"Review commit for branch feature/branch_4","state":"COMMENTED","html_url":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798","author_association":"CONTRIBUTOR","submitted_at":"2021-08-27T15:43:42Z","created_at":"2021-08-27T15:43:42Z","updated_at":"2021-08-27T15:43:42Z","user":{"node_id":"MDQ6VXNlcjc0MzkwMQ==","id":743901,"login":"gaart","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","html_url":"https://github.com/gaart","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","pull_request_url":"https://github.com/airbytehq/integration-test/pull/5","commit_id":"31a3e3f19fefce60fba6bfc69dd2b3fb5195a083","_links":{"html":{"href":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798"},"pull_request":{"href":"https://github.com/airbytehq/integration-test/pull/5"}}},"emitted_at":1677668764954} {"stream":"stargazers","data":{"starred_at":"2021-08-27T16:23:34Z","user":{"login":"VasylLazebnyk","id":68591643,"node_id":"MDQ6VXNlcjY4NTkxNjQz","avatar_url":"https://avatars.githubusercontent.com/u/68591643?v=4","gravatar_id":"","url":"https://api.github.com/users/VasylLazebnyk","html_url":"https://github.com/VasylLazebnyk","followers_url":"https://api.github.com/users/VasylLazebnyk/followers","following_url":"https://api.github.com/users/VasylLazebnyk/following{/other_user}","gists_url":"https://api.github.com/users/VasylLazebnyk/gists{/gist_id}","starred_url":"https://api.github.com/users/VasylLazebnyk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VasylLazebnyk/subscriptions","organizations_url":"https://api.github.com/users/VasylLazebnyk/orgs","repos_url":"https://api.github.com/users/VasylLazebnyk/repos","events_url":"https://api.github.com/users/VasylLazebnyk/events{/privacy}","received_events_url":"https://api.github.com/users/VasylLazebnyk/received_events","type":"User","site_admin":false},"repository":"airbytehq/integration-test","user_id":68591643},"emitted_at":1677668765231} diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py index aa6b652e953c..4d37ce6cccf5 100644 --- a/airbyte-integrations/connectors/source-github/main.py +++ b/airbyte-integrations/connectors/source-github/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_github import SourceGithub -from source_github.config_migrations import MigrateBranch, MigrateRepository +from source_github.run import run if __name__ == "__main__": - source = SourceGithub() - MigrateRepository.migrate(sys.argv[1:], source) - MigrateBranch.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml index 0ab538e21a6c..9ff2bc163d6a 100644 --- a/airbyte-integrations/connectors/source-github/metadata.yaml +++ b/airbyte-integrations/connectors/source-github/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - ${api_url} connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e - dockerImageTag: 1.5.4 + dockerImageTag: 1.6.3 dockerRepository: airbyte/source-github documentationUrl: https://docs.airbyte.com/integrations/sources/github githubIssueLabel: source-github @@ -18,8 +18,13 @@ data: license: MIT maxSecondsBetweenMessages: 5400 name: GitHub + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-github registries: cloud: + dockerImageTag: 1.5.7 enabled: true oss: enabled: true diff --git a/airbyte-integrations/connectors/source-github/poetry.lock b/airbyte-integrations/connectors/source-github/poetry.lock new file mode 100644 index 000000000000..24d8fe9af20e --- /dev/null +++ b/airbyte-integrations/connectors/source-github/poetry.lock @@ -0,0 +1,1108 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.23.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, + {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "sgqlc" +version = "16.3" +description = "Simple GraphQL Client" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "sgqlc-16.3-py3-none-any.whl", hash = "sha256:89d468386a4ba4b5ade991623228b6fb0a25bea1f25643ccac130fb3ef565b72"}, + {file = "sgqlc-16.3.tar.gz", hash = "sha256:be08857775aa3e65ef7b2c1f0cdcc65dd5794907b162b393c189187fee664558"}, +] + +[package.dependencies] +graphql-core = ">=3.1.7,<4.0.0" + +[package.extras] +requests = ["requests"] +websocket = ["websocket-client"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "40cc246c45e6c2d626e016673f3aa60794f3464d82c8ccd0b62a6b66df2b30da" diff --git a/airbyte-integrations/connectors/source-github/pyproject.toml b/airbyte-integrations/connectors/source-github/pyproject.toml new file mode 100644 index 000000000000..54b0e89c3bef --- /dev/null +++ b/airbyte-integrations/connectors/source-github/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.6.2" +name = "source-github" +description = "Source implementation for Github." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/github" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_github" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0.62.1" +sgqlc = "==16.3" + +[tool.poetry.scripts] +source-github = "source_github.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +freezegun = "^1.2" +pytest-mock = "^3.6.1" +pytest = "^6.2" +responses = "^0.23.1" diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py deleted file mode 100644 index 8b5f90f29e12..000000000000 --- a/airbyte-integrations/connectors/source-github/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "sgqlc"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2", "responses~=0.23.1", "freezegun~=1.2"] - -setup( - name="source_github", - description="Source implementation for Github.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-github/source_github/run.py b/airbyte-integrations/connectors/source-github/source_github/run.py new file mode 100644 index 000000000000..3abce9724842 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_github import SourceGithub +from source_github.config_migrations import MigrateBranch, MigrateRepository + + +def run(): + source = SourceGithub() + MigrateRepository.migrate(sys.argv[1:], source) + MigrateBranch.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json b/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json index 20363364aa4c..f8eb185d30da 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json @@ -25,6 +25,9 @@ "protection": { "type": ["null", "object"], "properties": { + "enabled": { + "type": ["null", "boolean"] + }, "required_status_checks": { "type": ["null", "object"], "properties": { @@ -36,6 +39,20 @@ "items": { "type": ["null", "string"] } + }, + "checks": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "context": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json b/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json index dc3d385337ed..97949b9cb869 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json @@ -65,13 +65,19 @@ "permissions": { "type": ["null", "object"], "properties": { - "pull": { + "admin": { + "type": ["null", "boolean"] + }, + "maintain": { "type": ["null", "boolean"] }, "push": { "type": ["null", "boolean"] }, - "admin": { + "pull": { + "type": ["null", "boolean"] + }, + "triage": { "type": ["null", "boolean"] } } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json index 3bc3eedc3e9f..9fc4d62fd344 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json @@ -50,6 +50,99 @@ "issue": { "type": ["null", "object"], "properties": { + "active_lock_reason": { + "type": ["null", "string"] + }, + "assignee": { + "$ref": "user.json" + }, + "assignees": { + "type": ["null", "array"], + "items": { + "$ref": "user.json" + } + }, + "author_association": { + "type": ["null", "string"] + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "comments": { + "type": ["null", "integer"] + }, + "draft": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } + } + }, + "locked": { + "type": ["null", "boolean"] + }, + "milestone": { + "type": ["null", "object"] + }, + "performed_via_github_app": { + "type": ["null", "object"] + }, + "state_reason": { + "type": ["null", "string"] + }, + "pull_request": { + "type": ["null", "object"], + "properties": { + "merged_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "diff_url": { "type": ["string", "null"] }, + "html_url": { "type": ["string", "null"] }, + "patch_url": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] } + } + }, + "timeline_url": { + "type": ["null", "string"] + }, + "reactions": { + "$ref": "reactions.json" + }, "id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json index 3abd58ae3f08..9c48fb4dbd68 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json @@ -970,7 +970,7 @@ "$ref": "events/reviewed.json" }, "commented": { - "$ref": "events/commented.json" + "$ref": "events/comment.json" }, "commit_commented": { "$ref": "events/commented.json" diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json index 5e5af5f9ccd7..c0a706660e36 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json @@ -244,6 +244,9 @@ }, "repo_id": { "type": ["null", "integer"] + }, + "user": { + "$ref": "user.json" } } }, @@ -264,6 +267,12 @@ }, "repo_id": { "type": ["null", "integer"] + }, + "repo": { + "type": ["null", "object"] + }, + "user": { + "$ref": "user.json" } } }, diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json b/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json index 36ce483faa91..d4e5d270d4b6 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json @@ -306,6 +306,22 @@ "type": ["null", "string"] } } + }, + "secret_scanning_validity_checks": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "string"] + } + } + }, + "dependabot_security_updates": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json index 19a0f40395ac..e3c4403ca94e 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json @@ -43,6 +43,8 @@ "description": "Issues are a great way to keep track of tasks, enhancements, and bugs for your projects.", "type": "object", "properties": { + "author_association": { "type": ["string", "null"] }, + "performed_via_github_app": { "type": ["object", "null"] }, "id": { "type": "integer" }, "node_id": { "type": "string" }, "url": { "type": "string" }, @@ -247,6 +249,7 @@ "locked": { "type": "boolean" }, "active_lock_reason": { "type": ["string", "null"] }, "comments": { "type": "integer" }, + "draft": { "type": ["boolean", "null"] }, "pull_request": { "type": "object", "properties": { @@ -775,6 +778,41 @@ } } } + }, + "reactions": { + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "total_count": { + "type": "integer" + }, + "+1": { + "type": "integer" + }, + "-1": { + "type": "integer" + }, + "laugh": { + "type": "integer" + }, + "confused": { + "type": "integer" + }, + "heart": { + "type": "integer" + }, + "hooray": { + "type": "integer" + }, + "eyes": { + "type": "integer" + }, + "rocket": { + "type": "integer" + } + } } } } diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 04e02fbadf21..d3519052a036 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -61,6 +61,9 @@ class SourceGithub(AbstractSource): + + continue_sync_on_stream_failure = True + @staticmethod def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator) -> Tuple[List[str], List[str]]: """ @@ -123,14 +126,7 @@ def get_access_token(config: Mapping[str, Any]): def _get_authenticator(self, config: Mapping[str, Any]): _, token = self.get_access_token(config) tokens = [t.strip() for t in token.split(constants.TOKEN_SEPARATOR)] - requests_per_hour = config.get("requests_per_hour") - if requests_per_hour: - return MultipleTokenAuthenticatorWithRateLimiter( - tokens=tokens, - auth_method="token", - requests_per_hour=requests_per_hour, - ) - return MultipleTokenAuthenticator(tokens=tokens, auth_method="token") + return MultipleTokenAuthenticatorWithRateLimiter(tokens=tokens) def _validate_and_transform_config(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: config = self._ensure_default_values(config) diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index 8c24d76278e7..edfb6f9a6c39 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -130,13 +130,6 @@ "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", "order": 4, "pattern_descriptor": "org/repo/branch1 org/repo/branch2" - }, - "requests_per_hour": { - "type": "integer", - "title": "Max requests per hour", - "description": "The GitHub API allows for a maximum of 5000 requests per hour (15000 for Github Enterprise). You can specify a lower value to limit your use of the API quota.", - "minimum": 1, - "order": 5 } } }, diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index fac84f7ed531..c54b1fb15e06 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from airbyte_cdk.utils import AirbyteTracedException from requests.exceptions import HTTPError from . import constants @@ -25,7 +26,7 @@ get_query_pull_requests, get_query_reviews, ) -from .utils import getter +from .utils import GitHubAPILimitException, getter class GithubStreamABC(HttpStream, ABC): @@ -38,6 +39,8 @@ class GithubStreamABC(HttpStream, ABC): stream_base_params = {} def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs): + if kwargs.get("authenticator"): + kwargs["authenticator"].max_time = self.max_time super().__init__(**kwargs) self.access_token_type = access_token_type @@ -126,16 +129,25 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: # we again could have 5000 per another hour. min_backoff_time = 60.0 - retry_after = response.headers.get("Retry-After") if retry_after is not None: - return max(float(retry_after), min_backoff_time) + backoff_time_in_seconds = max(float(retry_after), min_backoff_time) + return self.get_waiting_time(backoff_time_in_seconds) reset_time = response.headers.get("X-RateLimit-Reset") if reset_time: - return max(float(reset_time) - time.time(), min_backoff_time) + backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time) + return self.get_waiting_time(backoff_time_in_seconds) + + def get_waiting_time(self, backoff_time_in_seconds): + if backoff_time_in_seconds < self.max_time: + return backoff_time_in_seconds + else: + self._session.auth.update_token() # New token will be used in next request + return 1 - def check_graphql_rate_limited(self, response_json) -> bool: + @staticmethod + def check_graphql_rate_limited(response_json: dict) -> bool: errors = response_json.get("errors") if errors: for error in errors: @@ -203,6 +215,9 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter raise e self.logger.warning(error_msg) + except GitHubAPILimitException as e: + message = f"Stream: `{self.name}`, slice: `{stream_slice}`. Limits for all provided tokens are reached, please try again later" + raise AirbyteTracedException(message) from e class GithubStream(GithubStreamABC): @@ -738,7 +753,11 @@ def path( return "graphql" def should_retry(self, response: requests.Response) -> bool: - return True if response.json().get("errors") else super().should_retry(response) + if response.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT): + self.page_size = int(self.page_size / 2) + return True + self.page_size = constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.large_stream else constants.DEFAULT_PAGE_SIZE + return super().should_retry(response) or response.json().get("errors") def _get_repository_name(self, repository: Mapping[str, Any]) -> str: return repository["owner"]["login"] + "/" + repository["name"] diff --git a/airbyte-integrations/connectors/source-github/source_github/utils.py b/airbyte-integrations/connectors/source-github/source_github/utils.py index 285582d815be..3479e7c12b43 100644 --- a/airbyte-integrations/connectors/source-github/source_github/utils.py +++ b/airbyte-integrations/connectors/source-github/source_github/utils.py @@ -2,14 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging import time +from dataclasses import dataclass from itertools import cycle -from types import SimpleNamespace -from typing import List +from typing import Any, List, Mapping +import pendulum +import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator @@ -32,6 +34,18 @@ def read_full_refresh(stream_instance: Stream): yield record +class GitHubAPILimitException(Exception): + """General class for Rate Limits errors""" + + +@dataclass +class Token: + count_rest: int = 5000 + count_graphql: int = 5000 + reset_at_rest: pendulum.DateTime = pendulum.now() + reset_at_graphql: pendulum.DateTime = pendulum.now() + + class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator): """ Each token in the cycle is checked against the rate limiter. @@ -40,49 +54,99 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator): the first token becomes available again. """ - DURATION = 3600 # seconds + DURATION = pendulum.duration(seconds=3600) # Duration at which the current rate limit window resets - def __init__(self, tokens: List[str], requests_per_hour: int, auth_method: str = "Bearer", auth_header: str = "Authorization"): + def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"): self._auth_method = auth_method self._auth_header = auth_header - now = time.time() - self._requests_per_hour = requests_per_hour - self._tokens = {t: SimpleNamespace(count=self._requests_per_hour, update_at=now) for t in tokens} + self._tokens = {t: Token() for t in tokens} + self.check_all_tokens() self._tokens_iter = cycle(self._tokens) + self._active_token = next(self._tokens_iter) + self._max_time = 60 * 10 # 10 minutes as default @property def auth_header(self) -> str: return self._auth_header + def get_auth_header(self) -> Mapping[str, Any]: + """The header to set on outgoing HTTP requests""" + if self.auth_header: + return {self.auth_header: self.token} + return {} + + def __call__(self, request): + """Attach the HTTP headers required to authenticate on the HTTP request""" + while True: + current_token = self._tokens[self.current_active_token] + if "graphql" in request.path_url: + if self.process_token(current_token, "count_graphql", "reset_at_graphql"): + break + else: + if self.process_token(current_token, "count_rest", "reset_at_rest"): + break + + request.headers.update(self.get_auth_header()) + + return request + + @property + def current_active_token(self) -> str: + return self._active_token + + def update_token(self) -> None: + self._active_token = next(self._tokens_iter) + @property def token(self) -> str: - while True: - token = next(self._tokens_iter) - if self._check_token(token): - return f"{self._auth_method} {token}" - def _check_token(self, token: str): + token = self.current_active_token + return f"{self._auth_method} {token}" + + @property + def max_time(self) -> int: + return self._max_time + + @max_time.setter + def max_time(self, value: int) -> None: + self._max_time = value + + def _check_token_limits(self, token: str): """check that token is not limited""" - self._refill() - if self._sleep(): - self._refill() - if self._tokens[token].count > 0: - self._tokens[token].count -= 1 - return True + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} + rate_limit_info = ( + requests.get( + "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method) + ) + .json() + .get("resources") + ) + token_info = self._tokens[token] + remaining_info_core = rate_limit_info.get("core") + token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp( + remaining_info_core.get("reset") + ) + + remaining_info_graphql = rate_limit_info.get("graphql") + token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp( + remaining_info_graphql.get("reset") + ) - def _refill(self): - """refill all needed tokens""" - now = time.time() - for token, ns in self._tokens.items(): - if now - ns.update_at >= self.DURATION: - ns.update_at = now - ns.count = self._requests_per_hour - - def _sleep(self): - """sleep only if all tokens is exhausted""" - now = time.time() - if sum([ns.count for ns in self._tokens.values()]) == 0: - sleep_time = self.DURATION - (now - min([ns.update_at for ns in self._tokens.values()])) - logging.warning("Sleeping for %.1f seconds to enforce the limit of %d requests per hour.", sleep_time, self._requests_per_hour) - time.sleep(sleep_time) + def check_all_tokens(self): + for token in self._tokens: + self._check_token_limits(token) + + def process_token(self, current_token, count_attr, reset_attr): + if getattr(current_token, count_attr) > 0: + setattr(current_token, count_attr, getattr(current_token, count_attr) - 1) return True + elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()): + min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values()) + if min_time_to_wait < self.max_time: + time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0) + self.check_all_tokens() + else: + raise GitHubAPILimitException(f"Rate limits for all tokens ({count_attr}) were reached") + else: + self.update_token() + return False diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index c3d9c1c98188..f4e454dfab98 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -2,4 +2,18 @@ import os +import pytest +import responses + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" + + +@pytest.fixture(name="rate_limit_mock_response") +def rate_limit_mock_response(): + rate_limit_response = { + "resources": { + "core": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 4070908800}, + "graphql": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 4070908800}, + } + } + responses.add(responses.GET, "https://api.github.com/rate_limit", json=rate_limit_response) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/config.py new file mode 100644 index 000000000000..d03745efb614 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/config.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict, List + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "credentials": {"option_title": "PAT Credentials", "personal_access_token": "GITHUB_TEST_TOKEN"}, + "start_date": "2020-05-01T00:00:00Z", + } + + def with_repositories(self, repositories: List[str]) -> "ConfigBuilder": + self._config["repositories"] = repositories + return self + + def with_client_secret(self, client_secret: str) -> "ConfigBuilder": + self._config["client_secret"] = client_secret + return self + + def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": + self._config["start_date"] = start_datetime.isoformat()[:-13] + "Z" + return self + + def with_branches(self, branches: List[str]) -> "ConfigBuilder": + self._config["branches"] = branches + return self + + def with_api_url(self, api_url: str) -> "ConfigBuilder": + self._config["api_url"] = api_url + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py new file mode 100644 index 000000000000..a3f98b1a2cd5 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py @@ -0,0 +1,197 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStreamStatus, Level, TraceType +from source_github import SourceGithub + +from .config import ConfigBuilder + +_CONFIG = ConfigBuilder().with_repositories(["airbytehq/integration-test"]).build() + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="events", sync_mode=sync_mode).build() + + +class EventsTest(TestCase): + def setUp(self) -> None: + """Base setup for all tests. Add responses for: + 1. rate limit checker + 2. repositories + 3. branches + """ + + self.r_mock = HttpMocker() + self.r_mock.__enter__() + self.r_mock.get( + HttpRequest( + url="https://api.github.com/rate_limit", + query_params={}, + headers={ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Authorization": "token GITHUB_TEST_TOKEN", + }, + ), + HttpResponse( + json.dumps( + { + "resources": { + "core": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 5070908800}, + "graphql": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 5070908800}, + } + } + ), + 200, + ), + ) + + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps({"full_name": "airbytehq/integration-test", "default_branch": "master"}), 200), + ) + + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/branches", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps([{"repository": "airbytehq/integration-test", "name": "master"}]), 200), + ) + + def teardown(self): + """Stops and resets HttpMocker instance.""" + self.r_mock.__exit__() + + def test_read_full_refresh_no_pagination(self): + """Ensure http integration and record extraction""" + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps(find_template("events", __file__)), 200), + ) + + source = SourceGithub() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 2 + + def test_read_transformation(self): + """Ensure transformation applied to all records""" + + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps(find_template("events", __file__)), 200), + ) + + source = SourceGithub() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 2 + assert all(("repository", "airbytehq/integration-test") in x.record.data.items() for x in actual_messages.records) + + def test_full_refresh_with_pagination(self): + """Ensure pagination""" + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse( + body=json.dumps(find_template("events", __file__)), + status_code=200, + headers={"Link": '; rel="next"'.format(_CONFIG.get("repositories")[0])}, + ), + ) + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100, "page": 2}, + ), + HttpResponse( + body=json.dumps(find_template("events", __file__)), + status_code=200, + ), + ) + source = SourceGithub() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 4 + + def test_given_state_more_recent_than_some_records_when_read_incrementally_then_filter_records(self): + """Ensure incremental sync. + Stream `Events` is semi-incremental, so all requests will be performed and only new records will be extracted""" + + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps(find_template("events", __file__)), 200), + ) + + source = SourceGithub() + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder() + .with_stream_state("events", {"airbytehq/integration-test": {"created_at": "2022-06-09T10:00:00Z"}}) + .build(), + ) + assert len(actual_messages.records) == 1 + + def test_when_read_incrementally_then_emit_state_message(self): + """Ensure incremental sync emits correct stream state message""" + + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse(json.dumps(find_template("events", __file__)), 200), + ) + + source = SourceGithub() + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder() + .with_stream_state("events", {"airbytehq/integration-test": {"created_at": "2020-06-09T10:00:00Z"}}) + .build(), + ) + assert actual_messages.state_messages[0].state.data == {'events': {'airbytehq/integration-test': {'created_at': '2022-06-09T12:47:28Z'}}} + + def test_read_handles_expected_error_correctly_and_exits_with_complete_status(self): + """Ensure read() method does not raise an Exception and log message with error is in output""" + self.r_mock.get( + HttpRequest( + url=f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events", + query_params={"per_page": 100}, + ), + HttpResponse('{"message":"some_error_message"}', 403), + ) + source = SourceGithub() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert Level.ERROR in [x.log.level for x in actual_messages.logs] + events_stream_complete_message = [x for x in actual_messages.trace_messages if x.trace.type == TraceType.STREAM_STATUS][-1] + assert events_stream_complete_message.trace.stream_status.stream_descriptor.name == 'events' + assert events_stream_complete_message.trace.stream_status.status == AirbyteStreamStatus.COMPLETE diff --git a/airbyte-integrations/connectors/source-github/unit_tests/resource/http/response/events.json b/airbyte-integrations/connectors/source-github/unit_tests/resource/http/response/events.json new file mode 100644 index 000000000000..b0a2eca6eb32 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/resource/http/response/events.json @@ -0,0 +1,63 @@ +[ + { + "id": "22249084964", + "type": "PushEvent", + "actor": { + "id": 583231, + "login": "octocat", + "display_login": "octocat", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4" + }, + "repo": { + "id": 1296269, + "name": "octocat/Hello-World", + "url": "https://api.github.com/repos/octocat/Hello-World" + }, + "payload": { + "push_id": 10115855396, + "size": 1, + "distinct_size": 1, + "ref": "refs/heads/master", + "head": "7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300", + "before": "883efe034920928c47fe18598c01249d1a9fdabd", + "commits": [ + { + "sha": "7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300", + "author": { + "email": "octocat@github.com", + "name": "Monalisa Octocat" + }, + "message": "commit", + "distinct": true, + "url": "https://api.github.com/repos/octocat/Hello-World/commits/7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300" + } + ] + }, + "public": true, + "created_at": "2022-06-09T12:47:28Z" + }, + { + "id": "22237752260", + "type": "WatchEvent", + "actor": { + "id": 583231, + "login": "octocat", + "display_login": "octocat", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4" + }, + "repo": { + "id": 1296269, + "name": "octocat/Hello-World", + "url": "https://api.github.com/repos/octocat/Hello-World" + }, + "payload": { + "action": "started" + }, + "public": true, + "created_at": "2022-06-08T23:29:25Z" + } +] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py new file mode 100644 index 000000000000..f7fabd23c04d --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py @@ -0,0 +1,145 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import patch + +import pendulum +import pytest +import responses +from airbyte_cdk.utils import AirbyteTracedException +from freezegun import freeze_time +from source_github import SourceGithub +from source_github.streams import Organizations +from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter, read_full_refresh + + +@responses.activate +def test_multiple_tokens(rate_limit_mock_response): + authenticator = SourceGithub()._get_authenticator({"access_token": "token_1, token_2, token_3"}) + assert isinstance(authenticator, MultipleTokenAuthenticatorWithRateLimiter) + assert ["token_1", "token_2", "token_3"] == list(authenticator._tokens) + + +@responses.activate +def test_authenticator_counter(rate_limit_mock_response): + """ + This test ensures that the rate limiter: + 1. correctly handles the available limits from GitHub API and saves it. + 2. correctly counts the number of requests made. + """ + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(5000, 5000), (5000, 5000), (5000, 5000)] + organization_args = {"organizations": ["org1", "org2"], "authenticator": authenticator} + stream = Organizations(**organization_args) + responses.add("GET", "https://api.github.com/orgs/org1", json={"id": 1}) + responses.add("GET", "https://api.github.com/orgs/org2", json={"id": 2}) + list(read_full_refresh(stream)) + assert authenticator._tokens["token1"].count_rest == 4998 + + +@responses.activate +def test_multiple_token_authenticator_with_rate_limiter(): + """ + This test ensures that: + 1. The rate limiter iterates over all tokens one-by-one after the previous is fully drained. + 2. Counter is set to zero after 1500 requests were made. (500 available requests per key were set as default) + 3. Exception is handled and log warning message could be found in output. Connector does not raise AirbyteTracedException because there might be GraphQL streams with remaining request we still can read. + """ + + counter_rate_limits = 0 + counter_orgs = 0 + + def request_callback_rate_limits(request): + nonlocal counter_rate_limits + while counter_rate_limits < 3: + counter_rate_limits += 1 + resp_body = { + "resources": { + "core": {"limit": 500, "used": 0, "remaining": 500, "reset": 4070908800}, + "graphql": {"limit": 500, "used": 0, "remaining": 500, "reset": 4070908800}, + } + } + return (200, {}, json.dumps(resp_body)) + + responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits) + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + organization_args = {"organizations": ["org1"], "authenticator": authenticator} + stream = Organizations(**organization_args) + + def request_callback_orgs(request): + nonlocal counter_orgs + while counter_orgs < 1_501: + counter_orgs += 1 + resp_body = {"id": 1} + headers = {"Link": '; rel="next"'} + return (200, headers, json.dumps(resp_body)) + + responses.add_callback( + responses.GET, + "https://api.github.com/orgs/org1", + callback=request_callback_orgs, + content_type="application/json", + ) + with pytest.raises(AirbyteTracedException) as e: + list(read_full_refresh(stream)) + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(0, 500), (0, 500), (0, 500)] + message = ( + "Stream: `organizations`, slice: `{'organization': 'org1'}`. Limits for all provided tokens are reached, please try again later" + ) + assert e.value.internal_message == message + + +@freeze_time("2021-01-01 12:00:00") +@responses.activate +@patch("time.sleep") +def test_multiple_token_authenticator_with_rate_limiter_and_sleep(sleep_mock, caplog): + """ + This test ensures that: + 1. The rate limiter will only wait (sleep) for token availability if the nearest available token appears within 600 seconds (see max_time). + 2. Token Counter is reset to new values after 1500 requests were made and last token is still in use. + """ + + counter_rate_limits = 0 + counter_orgs = 0 + ACCEPTED_WAITING_TIME_IN_SECONDS = 595 + reset_time = (pendulum.now() + pendulum.duration(seconds=ACCEPTED_WAITING_TIME_IN_SECONDS)).int_timestamp + + def request_callback_rate_limits(request): + nonlocal counter_rate_limits + while counter_rate_limits < 6: + counter_rate_limits += 1 + resp_body = { + "resources": { + "core": {"limit": 500, "used": 0, "remaining": 500, "reset": reset_time}, + "graphql": {"limit": 500, "used": 0, "remaining": 500, "reset": reset_time}, + } + } + return (200, {}, json.dumps(resp_body)) + + responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits) + authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"]) + organization_args = {"organizations": ["org1"], "authenticator": authenticator} + stream = Organizations(**organization_args) + + def request_callback_orgs(request): + nonlocal counter_orgs + while counter_orgs < 1_501: + counter_orgs += 1 + resp_body = {"id": 1} + headers = {"Link": '; rel="next"'} + return (200, headers, json.dumps(resp_body)) + return (200, {}, json.dumps({"id": 2})) + + responses.add_callback( + responses.GET, + "https://api.github.com/orgs/org1", + callback=request_callback_orgs, + content_type="application/json", + ) + + list(read_full_refresh(stream)) + sleep_mock.assert_called_once_with(ACCEPTED_WAITING_TIME_IN_SECONDS) + assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(500, 500), (500, 500), (498, 500)] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 8ec9d79c574d..8942c27f93ac 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -2,20 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime import logging import os -import time from unittest.mock import MagicMock import pytest import responses from airbyte_cdk.models import AirbyteConnectionStatus, Status from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from freezegun import freeze_time from source_github import constants from source_github.source import SourceGithub -from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter from .utils import command_check @@ -27,6 +23,11 @@ def check_source(repo_line: str) -> AirbyteConnectionStatus: return source.check(logger_mock, config) +def test_source_will_continue_sync_on_stream_failure(): + source = SourceGithub() + assert source.continue_sync_on_stream_failure + + @responses.activate @pytest.mark.parametrize( "config, expected", @@ -42,7 +43,7 @@ def check_source(repo_line: str) -> AirbyteConnectionStatus: ({"access_token": "test_token", "repository": "airbyte/test"}, True), ), ) -def test_check_start_date(config, expected): +def test_check_start_date(config, expected, rate_limit_mock_response): responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "test_full_name"}) source = SourceGithub() status, _ = source.check_connection(logger=logging.getLogger("airbyte"), config=config) @@ -73,18 +74,18 @@ def test_connection_fail_due_to_config_error(api_url, deployment_env, expected_m @responses.activate -def test_check_connection_repos_only(): +def test_check_connection_repos_only(rate_limit_mock_response): responses.add("GET", "https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte"}) status = check_source("airbytehq/airbyte airbytehq/airbyte airbytehq/airbyte") assert not status.message assert status.status == Status.SUCCEEDED # Only one request since 3 repos have same name - assert len(responses.calls) == 1 + assert len(responses.calls) == 2 @responses.activate -def test_check_connection_repos_and_org_repos(): +def test_check_connection_repos_and_org_repos(rate_limit_mock_response): repos = [{"name": f"name {i}", "full_name": f"full name {i}", "updated_at": "2020-01-01T00:00:00Z"} for i in range(1000)] responses.add( "GET", "https://api.github.com/repos/airbyte/test", json={"full_name": "airbyte/test", "organization": {"login": "airbyte"}} @@ -99,11 +100,11 @@ def test_check_connection_repos_and_org_repos(): assert not status.message assert status.status == Status.SUCCEEDED # Two requests for repos and two for organization - assert len(responses.calls) == 4 + assert len(responses.calls) == 5 @responses.activate -def test_check_connection_org_only(): +def test_check_connection_org_only(rate_limit_mock_response): repos = [{"name": f"name {i}", "full_name": f"full name {i}", "updated_at": "2020-01-01T00:00:00Z"} for i in range(1000)] responses.add("GET", "https://api.github.com/orgs/airbytehq/repos", json=repos) @@ -111,7 +112,7 @@ def test_check_connection_org_only(): assert not status.message assert status.status == Status.SUCCEEDED # One request to check organization - assert len(responses.calls) == 1 + assert len(responses.calls) == 2 @responses.activate @@ -183,7 +184,8 @@ def test_get_org_repositories(): assert set(organisations) == {"airbytehq", "docker"} -def test_organization_or_repo_available(monkeypatch): +@responses.activate +def test_organization_or_repo_available(monkeypatch, rate_limit_mock_response): monkeypatch.setattr(SourceGithub, "_get_org_repositories", MagicMock(return_value=(False, False))) source = SourceGithub() with pytest.raises(Exception) as exc_info: @@ -238,55 +240,16 @@ def test_check_config_repository(): assert command_check(source, config) -def test_streams_no_streams_available_error(monkeypatch): +@responses.activate +def test_streams_no_streams_available_error(monkeypatch, rate_limit_mock_response): monkeypatch.setattr(SourceGithub, "_get_org_repositories", MagicMock(return_value=(False, False))) with pytest.raises(AirbyteTracedException) as e: SourceGithub().streams(config={"access_token": "test_token", "repository": "airbytehq/airbyte-test"}) assert str(e.value) == "No streams available. Please check permissions" -def test_multiple_token_authenticator_with_rate_limiter(monkeypatch): - - called_args = [] - - def sleep_mock(seconds): - frozen_time.tick(delta=datetime.timedelta(seconds=seconds)) - called_args.append(seconds) - - monkeypatch.setattr(time, "sleep", sleep_mock) - - with freeze_time("2021-01-01 12:00:00") as frozen_time: - - authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2"], requests_per_hour=4) - authenticator._tokens["token1"].count = 2 - - assert authenticator.token == "Bearer token1" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token1" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - - # token1 is fully exhausted, token2 is still used - assert authenticator._tokens["token1"].count == 0 - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert authenticator.token == "Bearer token2" - frozen_time.tick(delta=datetime.timedelta(seconds=1)) - assert called_args == [] - - # now we have to sleep because all tokens are exhausted - assert authenticator.token == "Bearer token1" - assert called_args == [3594.0] - - assert authenticator._tokens["token1"].count == 3 - assert authenticator._tokens["token2"].count == 4 - - @responses.activate -def test_streams_page_size(): +def test_streams_page_size(rate_limit_mock_response): responses.get("https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte", "default_branch": "master"}) responses.get("https://api.github.com/repos/airbytehq/airbyte/branches", json=[{"repository": "airbytehq/airbyte", "name": "master"}]) @@ -322,7 +285,7 @@ def test_streams_page_size(): ({"access_token": "test_token", "repository": "airbyte/test"}, 39), ), ) -def test_streams_config_start_date(config, expected): +def test_streams_config_start_date(config, expected, rate_limit_mock_response): responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "airbyte/test"}) responses.add( responses.GET, diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index 87d9c3478cd3..24f02e254e1a 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -234,6 +234,7 @@ def test_stream_organizations_read(): def test_stream_teams_read(): organization_args = {"organizations": ["org1", "org2"]} stream = Teams(**organization_args) + stream._session.cache.clear() responses.add("GET", "https://api.github.com/orgs/org1/teams", json=[{"id": 1}, {"id": 2}]) responses.add("GET", "https://api.github.com/orgs/org2/teams", json=[{"id": 3}]) records = list(read_full_refresh(stream)) @@ -533,7 +534,8 @@ def test_stream_project_columns(): projects_stream = Projects(**repository_args_with_start_date) stream = ProjectColumns(projects_stream, **repository_args_with_start_date) - + projects_stream._session.cache.clear() + stream._session.cache.clear() stream_state = {} records = read_incremental(stream, stream_state=stream_state) @@ -918,7 +920,7 @@ def request_callback(request): @responses.activate -def test_stream_team_members_full_refresh(caplog): +def test_stream_team_members_full_refresh(caplog, rate_limit_mock_response): organization_args = {"organizations": ["org1"]} repository_args = {"repositories": [], "page_size_for_large_streams": 100} @@ -959,6 +961,7 @@ def test_stream_commit_comment_reactions_incremental_read(): repository_args = {"repositories": ["airbytehq/integration-test"], "page_size_for_large_streams": 100} stream = CommitCommentReactions(**repository_args) + stream._parent_stream._session.cache.clear() responses.add( "GET", @@ -1305,7 +1308,7 @@ def request_callback(request): @responses.activate -def test_stream_projects_v2_graphql_retry(): +def test_stream_projects_v2_graphql_retry(rate_limit_mock_response): repository_args_with_start_date = { "start_date": "2022-01-01T00:00:00Z", "page_size_for_large_streams": 20, @@ -1368,7 +1371,7 @@ def test_stream_contributor_activity_parse_empty_response(caplog): @responses.activate -def test_stream_contributor_activity_accepted_response(caplog): +def test_stream_contributor_activity_accepted_response(caplog, rate_limit_mock_response): responses.add( responses.GET, "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", @@ -1376,17 +1379,17 @@ def test_stream_contributor_activity_accepted_response(caplog): status=200, ) responses.add( - responses.GET, - "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", - json={"full_name": "airbytehq/test_airbyte", "default_branch": "default_branch"}, - status=200, - ) + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", + json={"full_name": "airbytehq/test_airbyte", "default_branch": "default_branch"}, + status=200, + ) responses.add( - responses.GET, - "https://api.github.com/repos/airbytehq/test_airbyte/branches?per_page=100", - json={}, - status=200, - ) + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte/branches?per_page=100", + json={}, + status=200, + ) resp = responses.add( responses.GET, "https://api.github.com/repos/airbytehq/test_airbyte/stats/contributors?per_page=100", @@ -1398,9 +1401,14 @@ def test_stream_contributor_activity_accepted_response(caplog): configured_catalog = { "streams": [ { - "stream": {"name": "contributor_activity", "json_schema": {}, "supported_sync_modes": ["full_refresh"],"source_defined_primary_key": [["id"]]}, + "stream": { + "name": "contributor_activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "overwrite", } ] } diff --git a/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py deleted file mode 100644 index e7e3adf6bf81..000000000000 --- a/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.sources.streams.http.auth import MultipleTokenAuthenticator -from source_github import SourceGithub - - -def test_single_token(): - authenticator = SourceGithub()._get_authenticator({"access_token": "123"}) - assert isinstance(authenticator, MultipleTokenAuthenticator) - assert ["123"] == authenticator._tokens - authenticator = SourceGithub()._get_authenticator({"credentials": {"access_token": "123"}}) - assert ["123"] == authenticator._tokens - authenticator = SourceGithub()._get_authenticator({"credentials": {"personal_access_token": "123"}}) - assert ["123"] == authenticator._tokens - - -def test_multiple_tokens(): - authenticator = SourceGithub()._get_authenticator({"access_token": "123, 456"}) - assert isinstance(authenticator, MultipleTokenAuthenticator) - assert ["123", "456"] == authenticator._tokens diff --git a/airbyte-integrations/connectors/source-gitlab/README.md b/airbyte-integrations/connectors/source-gitlab/README.md index 6b4a4f0ad561..67d7ce2a8c7a 100644 --- a/airbyte-integrations/connectors/source-gitlab/README.md +++ b/airbyte-integrations/connectors/source-gitlab/README.md @@ -1,116 +1,55 @@ -# Gitlab Source +# Gitlab source connector + This is the repository for the Gitlab source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/gitlab). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/gitlab). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gitlab) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gitlab/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/gitlab) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gitlab/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source gitlab test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-gitlab spec +poetry run source-gitlab check --config secrets/config.json +poetry run source-gitlab discover --config secrets/config.json +poetry run source-gitlab read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-gitlab build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-gitlab:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-gitlab:latest -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-gitlab:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-gitlab:dev . -# Running the spec command against your patched connector -docker run airbyte/source-gitlab:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-gitlab:dev spec @@ -119,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gitlab:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gitlab:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-gitlab test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gitlab test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gitlab.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/gitlab.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml index 7ad85ba02cee..148e387d8c9f 100644 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml @@ -18,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: 1.8.4 + disable_for_version: 3.0.0 basic_read: tests: - config_path: "secrets/config.json" @@ -29,6 +29,11 @@ acceptance_tests: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + projects: + - name: "updated_at" + bypass_reason: "value can be changed" + - name: "code_suggestions" + bypass_reason: "value can be changed" - config_path: "secrets/config_with_ids.json" timeout_seconds: 3600 empty_streams: @@ -42,6 +47,11 @@ acceptance_tests: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + projects: + - name: "updated_at" + bypass_reason: "value can be changed" + - name: "code_suggestions" + bypass_reason: "value can be changed" - config_path: "secrets/config_oauth.json" timeout_seconds: 3600 expect_records: @@ -50,6 +60,11 @@ acceptance_tests: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + projects: + - name: "updated_at" + bypass_reason: "value can be changed" + - name: "code_suggestions" + bypass_reason: "value can be changed" incremental: tests: - config_path: "secrets/config_with_ids.json" diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl index b1c7dfe2eb8f..4a3fab124471 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl @@ -1,40 +1,36 @@ {"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1696947569422} {"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1696947569423} {"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1696947569423} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948219689} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948220075} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1708004403504} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1708004403203} {"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686568061140} -{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696948541619} -{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541622} -{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541624} -{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1696948783668} -{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1696948783989} -{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1696948784394} -{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1696949059273} -{"stream": "epic_issues", "data": {"id": 80659730, "iid": 13, "project_id": 25032440, "title": "Start a free trial of GitLab Gold - no credit card required :rocket:", "description": "At any point while using the free version of GitLab you can start a trial of GitLab Gold for free for 30 days. With a GitLab Gold trial, you'll get access to all of the most popular features across all of the paid tiers within GitLab. \n \n:white_check_mark: Reduce risk by requiring team leaders to approve merge requests.\n \n:white_check_mark: Ensure code quality with Multiple code reviews.\n \n:white_check_mark: Run your CI pipelines for up to 50,000 minutes (~9,500 CI builds).\n \n:white_check_mark: Plan and organize parallel development with multiple issue boards.\n \n:white_check_mark: Report on the productivity of each team in your organization by using issue analytics. \n \n:white_check_mark: Dynamically scan Docker images for vulnerabilities before production pushes. \n \n:white_check_mark: Scan security vulnerabilities, license compliance and dependencies in your CI pipelines. \n \n:white_check_mark: Get alerted when your application performance degrades. \n \n:white_check_mark: And so much more, [you can view all the features here](https://about.gitlab.com/pricing/gitlab-com/feature-comparison/). \n \n## Next steps\n* [ ] [Click here to start a trial of GitLab Gold.](https://gitlab.com/-/trial_registrations/new?glm_content=user_onboarding_whats_in_paid_tiers&glm_source=gitlab.com)", "state": "opened", "created_at": "2021-03-10T17:16:56.091Z", "updated_at": "2023-10-10T11:44:39.796Z", "closed_at": null, "closed_by": null, "labels": ["Novice"], "milestone": null, "assignees": [8375961], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/learn-gitlab/-/issues/13", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 1, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25032440/issues/13", "notes": "https://gitlab.com/api/v4/projects/25032440/issues/13/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25032440/issues/13/award_emoji", "project": "https://gitlab.com/api/v4/projects/25032440", "closed_as_duplicate_of": null}, "references": {"short": "#13", "relative": "#13", "full": "airbyte.io/learn-gitlab#13"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 3762298, "relative_position": -513, "milestone_id": null, "assignee_id": 8375961, "author_id": 8375961}, "emitted_at": 1696949059274} -{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354572} -{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354574} -{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354576} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696949674671} -{"stream": "epics", "data": {"id": 1977226, "iid": 2, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Test epic", "description": null, "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": null, "start_date_is_fixed": false, "start_date_fixed": null, "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": null, "due_date": null, "due_date_is_fixed": false, "due_date_fixed": null, "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/2", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/2", "references": {"short": "&2", "relative": "&2", "full": "airbyte.io&2"}, "created_at": "2023-10-10T10:37:36.529Z", "updated_at": "2023-10-10T11:44:50.107Z", "closed_at": null, "labels": [], "upvotes": 0, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/2", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/2/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906098} -{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2023-10-10T11:44:49.999Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906100} -{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1696950309747} -{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1696950309749} -{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1696950309750} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098000} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098001} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686568098411} +{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696948541619} +{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541622} +{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541624} +{"stream": "merge_request_commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T12:48:41.000Z", "parent_ids": [], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T12:48:41.000Z", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T12:48:41.000Z", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276, "merge_request_iid": 3}, "emitted_at": 1706206678393} +{"stream": "groups", "data": {"id": 11266951, "web_url": "https://gitlab.com/groups/airbyte.io", "name": "airbyte.io", "path": "airbyte.io", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "math_rendering_limits_enabled": true, "lock_math_rendering_limits_enabled": false, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "airbyte.io", "full_path": "airbyte.io", "created_at": "2021-03-10T17:16:37.549Z", "parent_id": null, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "marked_for_deletion_on": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bzmDjXx-Cz48snUcJfK8", "enabled_git_access_protocol": "all", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": 10000, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": false, "service_access_tokens_expiration_enforced": true, "membership_lock": false, "ip_restriction_ranges": null, "projects": [{"id": 25156633, "path_with_namespace": "airbyte.io/ci-test-project"}, {"id": 25032440, "path_with_namespace": "airbyte.io/learn-gitlab"}, {"id": 25032439, "path_with_namespace": "airbyte.io/documentation"}]}, "emitted_at": 1707997681702} +{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1696949059273} +{"stream": "epic_issues", "data": {"id": 80659730, "iid": 13, "project_id": 25032440, "title": "Start a free trial of GitLab Gold - no credit card required :rocket:", "description": "At any point while using the free version of GitLab you can start a trial of GitLab Gold for free for 30 days. With a GitLab Gold trial, you'll get access to all of the most popular features across all of the paid tiers within GitLab. \n \n:white_check_mark: Reduce risk by requiring team leaders to approve merge requests.\n \n:white_check_mark: Ensure code quality with Multiple code reviews.\n \n:white_check_mark: Run your CI pipelines for up to 50,000 minutes (~9,500 CI builds).\n \n:white_check_mark: Plan and organize parallel development with multiple issue boards.\n \n:white_check_mark: Report on the productivity of each team in your organization by using issue analytics. \n \n:white_check_mark: Dynamically scan Docker images for vulnerabilities before production pushes. \n \n:white_check_mark: Scan security vulnerabilities, license compliance and dependencies in your CI pipelines. \n \n:white_check_mark: Get alerted when your application performance degrades. \n \n:white_check_mark: And so much more, [you can view all the features here](https://about.gitlab.com/pricing/gitlab-com/feature-comparison/). \n \n## Next steps\n* [ ] [Click here to start a trial of GitLab Gold.](https://gitlab.com/-/trial_registrations/new?glm_content=user_onboarding_whats_in_paid_tiers&glm_source=gitlab.com)", "state": "opened", "created_at": "2021-03-10T17:16:56.091Z", "updated_at": "2023-10-10T11:44:39.796Z", "closed_at": null, "closed_by": null, "labels": ["Novice"], "milestone": null, "assignees": [8375961], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/learn-gitlab/-/issues/13", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 1, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25032440/issues/13", "notes": "https://gitlab.com/api/v4/projects/25032440/issues/13/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25032440/issues/13/award_emoji", "project": "https://gitlab.com/api/v4/projects/25032440", "closed_as_duplicate_of": null}, "references": {"short": "#13", "relative": "#13", "full": "airbyte.io/learn-gitlab#13"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 3762298, "relative_position": -513, "milestone_id": null, "assignee_id": 8375961, "author_id": 8375961}, "emitted_at": 1696949059274} +{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354572} +{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354574} +{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354576} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696949674671} +{"stream": "epics", "data": {"id": 1977226, "iid": 2, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Test epic", "description": null, "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": null, "start_date_is_fixed": false, "start_date_fixed": null, "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": null, "due_date": null, "due_date_is_fixed": false, "due_date_fixed": null, "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/2", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/2", "references": {"short": "&2", "relative": "&2", "full": "airbyte.io&2"}, "created_at": "2023-10-10T10:37:36.529Z", "updated_at": "2023-10-10T11:44:50.107Z", "closed_at": null, "labels": [], "upvotes": 0, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/2", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/2/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906098} +{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2023-10-10T11:44:49.999Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906100} +{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703256223650} +{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703256223651} +{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1703256223652} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704733346934} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704733346935} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1704733347322} {"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1696950582334} {"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1696950582334} {"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1696950582334} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696950910144} -{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1696951654063} -{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696951865405} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1696951865406} -{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696951865406} -{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1696952086155} -{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1696952086460} -{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1696952086890} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703256897835} +{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "code_suggestions": true, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "repository_object_format": "sha1", "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "model_registry_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2024-01-20T20:11:02.162Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "warn_about_potentially_unwanted_characters": true, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1707342174450} +{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257845052} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1703257845053} +{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257845054} {"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686568104768} {"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686568104771} {"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686568104771} @@ -43,12 +39,11 @@ {"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123880} {"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} {"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1696952237932} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696952237934} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386414} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386416} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696952387022} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185586} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185588} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185590} -{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696952237934} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386414} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/c72f0ecc5fd34318c337f919b4398dc56f4fd8c6176f85b61c45500c9a3cc84d?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386416} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696952387022} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl index 320db38aa3a1..09d148bf2fc3 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl @@ -1,50 +1,45 @@ {"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1686567225920} {"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1686567225922} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696947713101} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192490} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192491} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686567192861} -{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1696948297499} -{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1696948297896} -{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1696948298305} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696947713101} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704732685403} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704732685404} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1704732685727} +{"stream": "merge_request_commits", "data": {"id": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "short_id": "9b0c5cf3", "created_at": "2021-03-18T12:41:51.000Z", "parent_ids": [], "title": "update readme.md", "message": "update readme.md\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T12:41:51.000Z", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T12:41:51.000Z", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "project_id": 25157276, "merge_request_iid": 2}, "emitted_at": 1706270045163} {"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686567198876} {"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686567198878} {"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686567198878} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948628546} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948628851} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1696948873593} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696948873594} -{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1696949138497} -{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1696949138806} -{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1696949139214} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1708004403203} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1708004403504} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696948873594} +{"stream": "groups", "data": {"id": 61014943, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2", "name": "Test SG Public 2", "path": "test-sg-public-2", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "math_rendering_limits_enabled": true, "lock_math_rendering_limits_enabled": false, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2", "created_at": "2022-12-02T08:48:04.727Z", "parent_id": 61014902, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941eGbxua89EPU8uu4snVuj", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541858, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-project-1"}]}, "emitted_at": 1707997680781} {"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435261} {"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435263} {"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435264} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993328} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993329} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696949993941} -{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183576} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567183576} -{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183577} -{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184540} -{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184541} -{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1686567184541} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993328} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/c72f0ecc5fd34318c337f919b4398dc56f4fd8c6176f85b61c45500c9a3cc84d?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993329} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696949993941} +{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257027266} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1703257027267} +{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257027267} +{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703257635545} +{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703257635547} +{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1703257635548} {"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686567186609} -{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1696950432789} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225240} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225242} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225243} -{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696950689861} -{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689864} -{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689866} -{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969206} -{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969209} -{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969210} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696951717050} +{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "code_suggestions": true, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "repository_object_format": "sha1", "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "model_registry_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2024-01-20T20:11:02.162Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "warn_about_potentially_unwanted_characters": true, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1707342174450} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326525} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326527} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326528} +{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696950689861} +{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689864} +{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689866} +{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969206} +{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969209} +{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969210} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696951717050} {"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207747} {"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1686567207748} {"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207748} {"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1686567197935} {"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1686567197937} {"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1686567197937} -{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} +{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/1826158e78620962a6d53185476e96e14ad37ae49cb06c3bf6b617a50d4a9671?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} diff --git a/airbyte-integrations/connectors/source-gitlab/main.py b/airbyte-integrations/connectors/source-gitlab/main.py index 12b7cc841691..1c322c2f2c48 100644 --- a/airbyte-integrations/connectors/source-gitlab/main.py +++ b/airbyte-integrations/connectors/source-gitlab/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gitlab import SourceGitlab -from source_gitlab.config_migrations import MigrateGroups, MigrateProjects +from source_gitlab.run import run if __name__ == "__main__": - source = SourceGitlab() - MigrateGroups.migrate(sys.argv[1:], source) - MigrateProjects.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml index c4015a62d8ee..4485a58c3a17 100644 --- a/airbyte-integrations/connectors/source-gitlab/metadata.yaml +++ b/airbyte-integrations/connectors/source-gitlab/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 - dockerImageTag: 2.0.0 + dockerImageTag: 3.0.0 dockerRepository: airbyte/source-gitlab documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab githubIssueLabel: source-gitlab icon: gitlab.svg license: MIT name: Gitlab + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gitlab registries: cloud: enabled: true @@ -25,6 +29,14 @@ data: releaseStage: generally_available releases: breakingChanges: + 3.0.0: + message: + In this release, merge_request_commits stream schema has been fixed so that it returns commits for each merge_request. + Users will need to refresh the source schema and reset merge_request_commits stream after upgrading. + upgradeDeadline: "2024-02-13" + scopedImpact: + - scopeType: stream + impactedScopes: ["merge_request_commits"] 2.0.0: message: In this release, several streams were updated to date-time field format, as declared in the Gitlab API. diff --git a/airbyte-integrations/connectors/source-gitlab/poetry.lock b/airbyte-integrations/connectors/source-gitlab/poetry.lock new file mode 100644 index 000000000000..7056f01bf6e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/poetry.lock @@ -0,0 +1,1253 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "vcrpy" +version = "4.1.1" +description = "Automatically mock your HTTP interactions to simplify and speed up testing" +optional = false +python-versions = ">=3.5" +files = [ + {file = "vcrpy-4.1.1-py2.py3-none-any.whl", hash = "sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162"}, + {file = "vcrpy-4.1.1.tar.gz", hash = "sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599"}, +] + +[package.dependencies] +PyYAML = "*" +six = ">=1.5" +wrapt = "*" +yarl = {version = "*", markers = "python_version >= \"3.6\""} + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "94cc27fe8a4e14f6d9cfaaa0281dce3fd7ac7082d63c6152fa24a455a9872070" diff --git a/airbyte-integrations/connectors/source-gitlab/pyproject.toml b/airbyte-integrations/connectors/source-gitlab/pyproject.toml new file mode 100644 index 000000000000..bf22f7b57fe4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.1.2" +name = "source-gitlab" +description = "Source implementation for Gitlab." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/gitlab" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_gitlab" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.8" +vcrpy = "==4.1.1" + +[tool.poetry.scripts] +source-gitlab = "source_gitlab.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.12.0" +requests-mock = "^1.9.3" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-gitlab/requirements.txt b/airbyte-integrations/connectors/source-gitlab/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py deleted file mode 100644 index 682fadb8af03..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy==4.1.1"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "requests_mock", "pytest-mock"] - -setup( - name="source_gitlab", - description="Source implementation for Gitlab.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py index 0f963256cbcb..ec47f547f591 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py @@ -95,12 +95,10 @@ def migrate(cls, args: List[str], source: SourceGitlab) -> None: class MigrateGroups(MigrateStringToArray): - migrate_from_key: str = "groups" migrate_to_key: str = "groups_list" class MigrateProjects(MigrateStringToArray): - migrate_from_key: str = "projects" migrate_to_key: str = "projects_list" diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py new file mode 100644 index 000000000000..ddaf36b55b1c --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gitlab import SourceGitlab +from source_gitlab.config_migrations import MigrateGroups, MigrateProjects + + +def run(): + source = SourceGitlab() + MigrateGroups.migrate(sys.argv[1:], source) + MigrateProjects.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json index 89a7d2f5ae31..55b6809a6683 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json @@ -37,6 +37,17 @@ "type": ["null", "string"], "format": "date-time" }, + "extended_trailers": { + "type": ["null", "object"], + "properties": { + "Cc": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, "committer_name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json index 7750ef4fdbc9..95a42bcb0b2e 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json @@ -216,6 +216,12 @@ }, "group_id": { "type": ["null", "integer"] + }, + "human_readable_end_date": { + "type": ["null", "string"] + }, + "human_readable_timestamp": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json index be75d5230d67..7ad17999a9f4 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json @@ -19,6 +19,9 @@ "id": { "type": ["null", "integer"] }, + "organization_id": { + "type": ["null", "integer"] + }, "default_branch_protection_defaults": { "type": ["null", "object"], "properties": { @@ -35,6 +38,17 @@ } } } + }, + "allowed_to_push": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "access_level": { + "type": ["null", "integer"] + } + } + } } } }, @@ -71,9 +85,15 @@ "subgroup_creation_level": { "type": ["null", "string"] }, + "enabled_git_access_protocol": { + "type": ["null", "string"] + }, "emails_disabled": { "type": ["null", "boolean"] }, + "emails_enabled": { + "type": ["null", "boolean"] + }, "mentions_disabled": { "type": ["null", "boolean"] }, @@ -144,6 +164,15 @@ }, "shared_runners_setting": { "type": ["null", "string"] + }, + "service_access_tokens_expiration_enforced": { + "type": ["null", "boolean"] + }, + "lock_math_rendering_limits_enabled": { + "type": ["null", "boolean"] + }, + "math_rendering_limits_enabled": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json index 4ec4e6e0aace..42dca78a7370 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json @@ -118,6 +118,9 @@ }, "id": { "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] } } }, @@ -141,6 +144,12 @@ }, "username": { "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] } } }, @@ -164,6 +173,18 @@ }, "username": { "type": ["null", "string"] + }, + "human_readable_end_date": { + "type": ["null", "string"] + }, + "human_readable_timestamp": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] } } }, @@ -235,20 +256,32 @@ } }, "epic": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] + }, + "human_readable_end_date": { + "type": ["null", "string"] + }, + "human_readable_timestamp": { + "type": ["null", "string"] + } } }, "epic_iid": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json index 4c41e56b46a5..00fb3b5d6aad 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json @@ -11,6 +11,9 @@ "stage": { "type": ["null", "string"] }, + "archived": { + "type": ["null", "boolean"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json index 0045a1443c16..5e4410e4fb09 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json @@ -2,233 +2,84 @@ "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, "project_id": { "type": ["null", "integer"] }, - "title": { - "type": ["null", "string"] - }, - "description": { + "id": { "type": ["null", "string"] }, - "state": { + "short_id": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"], "format": "date-time" }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "merged_by": { - "type": ["null", "object"] - }, - "merged_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "prepared_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "closed_by": { - "type": ["null", "object"] - }, - "closed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "target_branch": { - "type": ["null", "string"] - }, - "source_branch": { - "type": ["null", "string"] - }, - "user_notes_count": { - "type": ["null", "integer"] - }, - "upvotes": { - "type": ["null", "integer"] - }, - "downvotes": { - "type": ["null", "integer"] - }, - "author": { - "type": ["null", "object"] - }, - "assignees": { - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "assignee": { - "type": ["null", "object"] - }, - "reviewers": { + "parent_ids": { "type": ["null", "array"], "items": { - "type": "object" + "type": ["null", "string"] } }, - "source_project_id": { - "type": ["null", "integer"] - }, - "target_project_id": { - "type": ["null", "integer"] - }, - "labels": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "work_in_progress": { - "type": ["null", "boolean"] - }, - "milestone": { - "type": ["null", "object"] - }, - "merge_when_pipeline_succeeds": { - "type": ["null", "boolean"] - }, - "merge_status": { - "type": ["null", "string"] - }, - "sha": { - "type": ["null", "string"] - }, - "merge_commit_sha": { + "title": { "type": ["null", "string"] }, - "squash_commit_sha": { + "message": { "type": ["null", "string"] }, - "discussion_locked": { - "type": ["null", "boolean"] - }, - "should_remove_source_branch": { - "type": ["null", "boolean"] - }, - "force_remove_source_branch": { - "type": ["null", "boolean"] - }, - "reference": { + "author_name": { "type": ["null", "string"] }, - "references": { - "type": ["null", "object"] - }, - "web_url": { + "author_email": { "type": ["null", "string"] }, - "time_stats": { - "type": ["null", "object"] - }, - "squash": { - "type": ["null", "boolean"] - }, - "task_completion_status": { - "type": ["null", "object"] - }, - "has_conflicts": { - "type": ["null", "boolean"] - }, - "blocking_discussions_resolved": { - "type": ["null", "boolean"] - }, - "approvals_before_merge": { - "type": ["null", "boolean", "string", "object"] - }, - "subscribed": { - "type": ["null", "boolean"] - }, - "changes_count": { - "type": ["null", "integer", "string"] - }, - "latest_build_started_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "latest_build_finished_at": { + "authored_date": { "type": ["null", "string"], "format": "date-time" }, - "first_deployed_to_production_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "pipeline": { - "type": ["null", "object"] - }, - "head_pipeline": { - "type": ["null", "object", "string", "boolean", "integer"] - }, - "diff_refs": { + "extended_trailers": { "type": ["null", "object"], "properties": { - "base_sha": { - "type": ["null", "string"] - }, - "head_sha": { - "type": ["null", "string"] - }, - "start_sha": { - "type": ["null", "string"] + "Cc": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } }, - "merge_error": { - "type": ["null", "boolean", "string"] - }, - "first_contribution": { - "type": ["null", "boolean"] + "committer_name": { + "type": ["null", "string"] }, - "user": { - "type": ["null", "object"] + "committer_email": { + "type": ["null", "string"] }, - "merge_request_iid": { - "type": ["null", "integer"] + "committed_date": { + "type": ["null", "string"], + "format": "date-time" }, - "draft": { - "type": ["null", "boolean"] + "trailers": { + "type": ["null", "object"] }, - "detailed_merge_status": { + "web_url": { "type": ["null", "string"] }, - "squash_on_merge": { - "type": ["null", "boolean"] - }, - "merge_user": { + "stats": { "type": ["null", "object"], "properties": { - "id": { + "additions": { "type": ["null", "integer"] }, - "name": { - "type": ["null", "string"] - }, - "username": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "avatar_url": { - "type": ["null", "string"] + "deletions": { + "type": ["null", "integer"] }, - "web_url": { - "type": ["null", "string"] + "total": { + "type": ["null", "integer"] } } + }, + "merge_request_iid": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json index ae8197c69bdd..570cfa6b6e1b 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json @@ -194,6 +194,9 @@ }, "web_url": { "type": ["null", "string"] + }, + "locked": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json index 5bd2853879a2..fbeb962fbad1 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json @@ -63,6 +63,9 @@ }, "web_url": { "type": ["null", "string"] + }, + "locked": { + "type": ["null", "boolean"] } } }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json index 21c5bc9abf8e..c273e2f4a63b 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json @@ -123,6 +123,9 @@ }, "members": { "type": ["null", "string"] + }, + "cluster_agents": { + "type": ["null", "string"] } } }, @@ -329,6 +332,15 @@ }, "packages_size": { "type": ["null", "integer"] + }, + "container_registry_size": { + "type": ["null", "integer"] + }, + "pipeline_artifacts_size": { + "type": ["null", "integer"] + }, + "uploads_size": { + "type": ["null", "integer"] } } }, @@ -483,6 +495,21 @@ }, "merge_trains_skip_train_allowed": { "type": ["null", "boolean"] + }, + "code_suggestions": { + "type": ["null", "boolean"] + }, + "model_registry_access_level": { + "type": ["null", "string"] + }, + "ci_restrict_pipeline_cancellation_role": { + "type": ["null", "string"] + }, + "repository_object_format": { + "type": ["null", "string"] + }, + "warn_about_potentially_unwanted_characters": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index d561009f13bc..bf269f83ed82 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -92,7 +92,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp elif isinstance(response_data, dict): yield self.transform(response_data, **kwargs) else: - Exception(f"Unsupported type of response data for stream {self.name}") + self.logger.info(f"Unsupported type of response data for stream {self.name}") def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): for key in self.flatten_id_keys: @@ -166,7 +166,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_state = current_state.get(self.cursor_field) current_state_value = current_state or latest_cursor_value max_value = max(pendulum.parse(current_state_value), pendulum.parse(latest_cursor_value)) - current_stream_state[str(project_id)] = {self.cursor_field: str(max_value)} + current_stream_state[str(project_id)] = {self.cursor_field: max_value.to_iso8601_string()} return current_stream_state @staticmethod @@ -339,8 +339,10 @@ class MergeRequests(IncrementalGitlabChildStream): class MergeRequestCommits(GitlabChildStream): + """Docs: https://docs.gitlab.com/ee/api/merge_requests.html#get-single-merge-request-commits""" + path_list = ["project_id", "iid"] - path_template = "projects/{project_id}/merge_requests/{iid}" + path_template = "projects/{project_id}/merge_requests/{iid}/commits" def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): super().transform(record, stream_slice, **kwargs) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json new file mode 100644 index 000000000000..71f30753dc6e --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json @@ -0,0 +1 @@ +{ "groups": "a b c", "groups_list": ["a", "c", "b"] } diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..b61fb232906d --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +from source_gitlab.config_migrations import MigrateGroups +from source_gitlab.source import SourceGitlab + +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" + + +def test_should_migrate(): + assert MigrateGroups._should_migrate({"groups": "group group2 group3"}) is True + assert MigrateGroups._should_migrate({"groups_list": ["test", "group2", "group3"]}) is False + + +def test__modify_and_save(): + source = SourceGitlab() + expected = {"groups": "a b c", "groups_list": ["b", "c", "a"]} + modified_config = MigrateGroups._modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config={"groups": "a b c"}) + assert modified_config["groups_list"].sort() == expected["groups_list"].sort() + assert modified_config.get("groups") diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index 8874e957c06e..5454ee1d7d76 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -64,12 +64,39 @@ def test_connection_fail_due_to_api_error(errror_code, expected_status, config, assert msg.startswith("Unable to connect to Gitlab API with the provided Private Access Token") +def test_connection_fail_due_to_api_error_oauth(oauth_config, mocker, requests_mock): + mocker.patch("time.sleep") + test_response = { + "access_token": "new_access_token", + "expires_in": 7200, + "created_at": 1735689600, + # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00" + "refresh_token": "new_refresh_token", + } + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response) + requests_mock.get("/api/v4/groups", status_code=500) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger(), oauth_config) + assert status is False + assert msg.startswith("Unable to connect to Gitlab API with the provided credentials") + + def test_connection_fail_due_to_expired_access_token_error(oauth_config, requests_mock): - expected = "Unable to refresh the `access_token`, please re-auth in Source > Settings." + expected = "Unable to refresh the `access_token`, please re-authenticate in Sources > Settings." requests_mock.post("https://gitlab.com/oauth/token", status_code=401) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config) - assert status is False, expected in msg + assert status is False + assert expected in msg + + +def test_connection_refresh_access_token(oauth_config, requests_mock): + expected = "Unknown error occurred while checking the connection" + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json={"access_token": "new access token"}) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config) + assert status is False + assert expected in msg def test_refresh_expired_access_token_on_error(oauth_config, requests_mock): @@ -108,3 +135,27 @@ def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, ex } status, msg = source.check_connection(logging.getLogger(), config) assert (status, msg) == (False, expected_message) + + +def test_try_refresh_access_token(oauth_config, requests_mock): + test_response = { + "access_token": "new_access_token", + "expires_in": 7200, + "created_at": 1735689600, + # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00" + "refresh_token": "new_refresh_token", + } + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response) + + expected = {"api_url": "gitlab.com", + "credentials": {"access_token": "new_access_token", + "auth_type": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "new_refresh_token", + "token_expiry_date": "2025-01-01T02:00:00+00:00"}, + "start_date": "2021-01-01T00:00:00Z"} + + source = SourceGitlab() + source._auth_params(oauth_config) + assert source._try_refresh_access_token(logger=logging.getLogger(), config=oauth_config) == expected diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index 7fd342f45c47..9c29fa8808f9 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -3,8 +3,10 @@ # import datetime +from unittest.mock import MagicMock import pytest +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.auth import NoAuth from source_gitlab.streams import ( Branches, @@ -205,7 +207,7 @@ def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_ [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}], ), ( - "/api/v4/projects/p_1/merge_requests/mr_1", + "/api/v4/projects/p_1/merge_requests/mr_1/commits", [ { "id": "mrc_1", @@ -276,3 +278,53 @@ def test_transform(requests_mock, stream, response_mocks, expected_records, requ def test_updated_state(stream, current_state, latest_record, new_state, request): stream = request.getfixturevalue(stream) assert stream.get_updated_state(current_state, latest_record) == new_state + + +def test_parse_response_unsuported_response_type(request, caplog): + stream = request.getfixturevalue("pipelines") + from unittest.mock import MagicMock + response = MagicMock() + response.status_code = 200 + response.json = MagicMock(return_value="") + list(stream.parse_response(response=response)) + assert "Unsupported type of response data for stream pipelines" in caplog.text + + +def test_stream_slices_child_stream(request, requests_mock): + commits = request.getfixturevalue("commits") + requests_mock.get("https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1", + json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}]) + + slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={"13082000": {""'created_at': "2021-03-10T23:58:1213"}})) + assert slices + + +def test_next_page_token(request): + response = MagicMock() + response.status_code = 200 + response.json = MagicMock(return_value=["some data"]) + commits = request.getfixturevalue("commits") + assert not commits.next_page_token(response) + data = ["some data" for x in range(0, 50)] + response.json = MagicMock(return_value=data) + assert commits.next_page_token(response) == {'page': 2} + response.json = MagicMock(return_value={"data": "some data"}) + assert not commits.next_page_token(response) + + +def test_availability_strategy(request): + commits = request.getfixturevalue("commits") + assert not commits.availability_strategy + + +def test_request_params(request): + commits = request.getfixturevalue("commits") + expected = {'per_page': 50, 'page': 2, 'with_stats': True} + assert commits.request_params(stream_slice={"updated_after": "2021-03-10T23:58:1213"}, next_page_token={'page': 2}) == expected + + +def test_chunk_date_range(request): + commits = request.getfixturevalue("commits") + # start point in future + start_point = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1) + assert not list(commits._chunk_date_range(start_point)) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py new file mode 100644 index 000000000000..bd107e1a16dc --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from source_gitlab.utils import parse_url + + +@pytest.mark.parametrize( + "url, expected", + ( + ("http://example.com", (True, "http", "example.com")), + ("http://example", (True, "http", "example")), + ("test://example.com", (False, "", "")), + ("https://example.com/test/test2", (False, "", "")), + ) +) +def test_parse_url(url, expected): + assert parse_url(url) == expected diff --git a/airbyte-integrations/connectors/source-glassfrog/main.py b/airbyte-integrations/connectors/source-glassfrog/main.py index 22d58701db3d..f063b41ef47f 100644 --- a/airbyte-integrations/connectors/source-glassfrog/main.py +++ b/airbyte-integrations/connectors/source-glassfrog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_glassfrog import SourceGlassfrog +from source_glassfrog.run import run if __name__ == "__main__": - source = SourceGlassfrog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml index 051a5ef1d90c..edf2c6a6fba6 100644 --- a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.glassfrog.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-glassfrog registries: oss: enabled: true @@ -20,7 +24,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/glassfrog tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py index e62098fca78b..531a1c2a7d87 100644 --- a/airbyte-integrations/connectors/source-glassfrog/setup.py +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-glassfrog=source_glassfrog.run:run", + ], + }, name="source_glassfrog", description="Source implementation for Glassfrog.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py new file mode 100644 index 000000000000..618d3873c80c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_glassfrog import SourceGlassfrog + + +def run(): + source = SourceGlassfrog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gnews/main.py b/airbyte-integrations/connectors/source-gnews/main.py index 2d7fef617053..4702ac5fd364 100644 --- a/airbyte-integrations/connectors/source-gnews/main.py +++ b/airbyte-integrations/connectors/source-gnews/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gnews import SourceGnews +from source_gnews.run import run if __name__ == "__main__": - source = SourceGnews() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gnews/metadata.yaml b/airbyte-integrations/connectors/source-gnews/metadata.yaml index 0f8a7ba6b189..3c2eb68602d9 100644 --- a/airbyte-integrations/connectors/source-gnews/metadata.yaml +++ b/airbyte-integrations/connectors/source-gnews/metadata.yaml @@ -8,6 +8,11 @@ data: icon: gnews.svg license: MIT name: GNews + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-gnews registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py index 3e4cf6f2cc28..564eaf258c85 100644 --- a/airbyte-integrations/connectors/source-gnews/setup.py +++ b/airbyte-integrations/connectors/source-gnews/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gnews=source_gnews.run:run", + ], + }, name="source_gnews", description="Source implementation for Gnews.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/run.py b/airbyte-integrations/connectors/source-gnews/source_gnews/run.py new file mode 100644 index 000000000000..c2bf1ff536a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gnews/source_gnews/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gnews import SourceGnews + + +def run(): + source = SourceGnews() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gocardless/main.py b/airbyte-integrations/connectors/source-gocardless/main.py index 0c7fbe5c27ed..b7d51bd717b7 100644 --- a/airbyte-integrations/connectors/source-gocardless/main.py +++ b/airbyte-integrations/connectors/source-gocardless/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gocardless import SourceGocardless +from source_gocardless.run import run if __name__ == "__main__": - source = SourceGocardless() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gocardless/metadata.yaml b/airbyte-integrations/connectors/source-gocardless/metadata.yaml index d49af139e575..3d8b577d174a 100644 --- a/airbyte-integrations/connectors/source-gocardless/metadata.yaml +++ b/airbyte-integrations/connectors/source-gocardless/metadata.yaml @@ -8,6 +8,10 @@ data: icon: gocardless.svg license: MIT name: GoCardless + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gocardless registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-gocardless/setup.py b/airbyte-integrations/connectors/source-gocardless/setup.py index b49d3b8111e0..46bcd590d002 100644 --- a/airbyte-integrations/connectors/source-gocardless/setup.py +++ b/airbyte-integrations/connectors/source-gocardless/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gocardless=source_gocardless.run:run", + ], + }, name="source_gocardless", description="Source implementation for Gocardless.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py b/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py new file mode 100644 index 000000000000..1884dd380ed4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gocardless import SourceGocardless + + +def run(): + source = SourceGocardless() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gong/Dockerfile b/airbyte-integrations/connectors/source-gong/Dockerfile index 8c90d307427e..40e34b1cfb0d 100644 --- a/airbyte-integrations/connectors/source-gong/Dockerfile +++ b/airbyte-integrations/connectors/source-gong/Dockerfile @@ -34,5 +34,5 @@ COPY source_gong ./source_gong ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-gong diff --git a/airbyte-integrations/connectors/source-gong/main.py b/airbyte-integrations/connectors/source-gong/main.py index d30389447551..dc012c0e42c0 100644 --- a/airbyte-integrations/connectors/source-gong/main.py +++ b/airbyte-integrations/connectors/source-gong/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gong import SourceGong +from source_gong.run import run if __name__ == "__main__": - source = SourceGong() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gong/metadata.yaml b/airbyte-integrations/connectors/source-gong/metadata.yaml index 08864fcb3190..dd82f25883b3 100644 --- a/airbyte-integrations/connectors/source-gong/metadata.yaml +++ b/airbyte-integrations/connectors/source-gong/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: 32382e40-3b49-4b99-9c5c-4076501914e7 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-gong githubIssueLabel: source-gong icon: gong.svg license: MIT name: Gong + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gong registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-gong/setup.py b/airbyte-integrations/connectors/source-gong/setup.py index e87d4ea56771..2232e3fe24a1 100644 --- a/airbyte-integrations/connectors/source-gong/setup.py +++ b/airbyte-integrations/connectors/source-gong/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gong=source_gong.run:run", + ], + }, name="source_gong", description="Source implementation for Gong.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gong/source_gong/run.py b/airbyte-integrations/connectors/source-gong/source_gong/run.py new file mode 100644 index 000000000000..95da404269f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-gong/source_gong/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gong import SourceGong + + +def run(): + source = SourceGong() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gong/source_gong/schemas/calls.json b/airbyte-integrations/connectors/source-gong/source_gong/schemas/calls.json index d2488ac806f5..a8b5b7e3e7cb 100644 --- a/airbyte-integrations/connectors/source-gong/source_gong/schemas/calls.json +++ b/airbyte-integrations/connectors/source-gong/source_gong/schemas/calls.json @@ -60,6 +60,9 @@ }, "isPrivate": { "type": ["null", "boolean"] + }, + "calendarEventId": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-gong/source_gong/schemas/users.json b/airbyte-integrations/connectors/source-gong/source_gong/schemas/users.json index f23814c77c14..726bec44117c 100644 --- a/airbyte-integrations/connectors/source-gong/source_gong/schemas/users.json +++ b/airbyte-integrations/connectors/source-gong/source_gong/schemas/users.json @@ -8,6 +8,9 @@ "emailAddress": { "type": ["null", "string"] }, + "trustedEmailAddress": { + "type": ["null", "string"] + }, "created": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md b/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md index 89c07e4750cb..4092d3c7075c 100644 --- a/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md +++ b/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md @@ -9,14 +9,14 @@ The resources are listed [here](https://developers.google.com/google-ads/api/ref When querying data, there are three categories of information that can be fetched: - **Attributes**: These are properties of the various entities in the API e.g: the title or ID of an ad campaign. -- **Metrics**: metrics are statistics related to entities in the API. For example, the number of impressions for an ad or an ad campaign. All available metrics can be found [here](https://developers.google.com/google-ads/api/fields/v11/metrics). +- **Metrics**: metrics are statistics related to entities in the API. For example, the number of impressions for an ad or an ad campaign. All available metrics can be found [here](https://developers.google.com/google-ads/api/fields/v15/metrics). - **Segments**: These are ways to partition metrics returned in the query by particular attributes. For example, one could query for the number of impressions (views of an ad) by running SELECT metrics.impressions FROM campaigns which would return the number of impressions for each campaign e.g: 10k impressions. Or you could query for impressions segmented by device type e.g; SELECT metrics.impressions, segments.device FROM campaigns which would return the number of impressions broken down by device type e.g: 3k iOS and 7k Android. When summing the result across all segments, the sum should be the same (approximately) as when requesting the whole query without segments. This is a useful feature for granular data analysis as an advertiser may for example want to know if their ad is successful with a particular kind of person over the other. See more about segmentation [here](https://developers.google.com/google-ads/api/docs/concepts/retrieving-objects). -If you want to get a representation of the raw resources in the API e.g: just know what are all the ads or campaigns in your google account, you would query only for attributes e.g. SELECT campaign.title FROM campaigns. +If you want to get a representation of the raw resources in the API e.g: just know what are all the ads or campaigns in your Google account, you would query only for attributes e.g. SELECT campaign.title FROM campaigns. But if you wanted to get reports about the data (a common use case is impression data for an ad campaign) then you would query for metrics, potentially with segmentation. diff --git a/airbyte-integrations/connectors/source-google-ads/README.md b/airbyte-integrations/connectors/source-google-ads/README.md index d539875f4dc0..8489101ea912 100644 --- a/airbyte-integrations/connectors/source-google-ads/README.md +++ b/airbyte-integrations/connectors/source-google-ads/README.md @@ -1,116 +1,55 @@ -# Google Ads Source +# Google-Ads source connector -This is the repository for the Google Ads source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-ads). + +This is the repository for the Google-Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/google-ads). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-ads) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_ads/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/google-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_ads/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-ads test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-google-ads spec +poetry run source-google-ads check --config secrets/config.json +poetry run source-google-ads discover --config secrets/config.json +poetry run source-google-ads read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-google-ads build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-ads:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-google-ads:latest -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-google-ads:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-google-ads:dev . -# Running the spec command against your patched connector -docker run airbyte/source-google-ads:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-google-ads:dev spec @@ -119,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-ads:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-google-ads test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-ads test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-ads.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/google-ads.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index 84821f3e1804..b7b7d3ba73a6 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -41,6 +41,8 @@ acceptance_tests: bypass_reason: "Value can be updated by Google Ads" - name: customer.optimization_score bypass_reason: "Value can be updated by Google Ads" + - name: customer.pay_per_conversion_eligibility_failure_reasons + bypass_reason: "Value can be updated by Google Ads" - config_path: "secrets/config_click_view.json" expect_records: path: "integration_tests/expected_records_click.jsonl" @@ -55,21 +57,91 @@ acceptance_tests: - name: "keyword_view" bypass_reason: "No data for this date range, tested in previous config" ignored_fields: + account_performance_report: + - name: metrics.cross_device_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + ad_group: + - name: ad_group.url_custom_parameters + bypass_reason: "Value can be updated by Google Ads" customer: - name: customer.optimization_score_weight bypass_reason: "Value can be updated by Google Ads" - name: customer.optimization_score bypass_reason: "Value can be updated by Google Ads" + - name: customer.pay_per_conversion_eligibility_failure_reasons + bypass_reason: "Value can be updated by Google Ads" campaign_budget: - name: campaign_budget.recommended_budget_estimated_change_weekly_interactions bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cross_device_conversions + bypass_reason: "Value can be updated by Google Ads" campaign: - name: campaign.optimization_score bypass_reason: "Value can be updated by Google Ads" + ad_group_ad_legacy: + - name: metrics.all_conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_current_model_attributed_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.current_model_attributed_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.current_model_attributed_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_current_model_attributed_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cross_device_conversions + bypass_reason: "Value can be updated by Google Ads" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_manager_account.json" incremental: tests: - config_path: "secrets/incremental_config.json" diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl index ed3bf7bf792d..c734ff62715b 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl @@ -1,71 +1,71 @@ -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 27142857.14285714, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 3, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.10714285714285714, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 28, "metrics.interaction_rate": 0.10714285714285714, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 3, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6935849056603773, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2852830188679245, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271281866} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 30000.0, "metrics.average_cpc": 30000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 2500000.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 1, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 30000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.08333333333333333, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 12, "metrics.interaction_rate": 0.08333333333333333, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.7254437869822485, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2603550295857988, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271281869} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 0, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.9001, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271281870} -{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-18"}, "emitted_at": 1697271296345} -{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-19"}, "emitted_at": 1697271296348} -{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-20"}, "emitted_at": 1697271296348} -{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-18"}, "emitted_at": 1697271294773} -{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-19"}, "emitted_at": 1697271294777} -{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-20"}, "emitted_at": 1697271294781} -{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1697271302445} -{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1697271302447} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 19750000.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 4, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.1, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 40, "metrics.interaction_rate": 0.1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 4, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.75, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271276253} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH_PARTNERS", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 0, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 11, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271276263} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 37391304.347826086, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 6, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2608695652173913, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-19", "segments.day_of_week": "THURSDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 23, "metrics.interaction_rate": 0.2608695652173913, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 6, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.7391304347826086, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1697271276267} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1697271313915} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1697271313919} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1697271313920} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 10515001, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data warehouses", "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data warehouses", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10515001", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271316846} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "UNSPECIFIED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 10683521, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "database software", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 0, "ad_group_criterion.effective_cpm_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "database software", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10683521", "ad_group_criterion.status": "REMOVED", "ad_group_criterion.system_serving_status": "UNSPECIFIED", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271316853} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 11100571, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "integration software", "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "integration software", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~11100571", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271316858} -{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1697271324563} -{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1697271324566} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-18"}, "emitted_at": 1697271280535} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-19"}, "emitted_at": 1697271280537} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-20"}, "emitted_at": 1697271280538} -{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271301332} -{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271301334} -{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1697271301335} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10515001", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 10515001, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271322120} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10683521", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 10683521, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271322122} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~11100571", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 11100571, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271322123} -{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1697271307054} -{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1697271307056} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 1, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1697271298841} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 2, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1697271298844} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 3, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1697271298846} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1697271312921} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1697271312925} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1697271312929} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-18", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 15490196.078431372, "metrics.average_cpv": 0.0, "metrics.clicks": 4, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0784313725490196, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 51, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.0784313725490196, "metrics.interactions": 4, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271278680} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-19", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 31851851.85185185, "metrics.average_cpv": 0.0, "metrics.clicks": 6, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2222222222222222, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 27, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.2222222222222222, "metrics.interactions": 6, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271278687} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-20", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 215000.0, "metrics.average_cpc": 215000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 16538461.53846154, "metrics.average_cpv": 0.0, "metrics.clicks": 2, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 430000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.07692307692307693, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 26, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.07692307692307693, "metrics.interactions": 2, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271278692} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271325181} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271325184} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271325185} -{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271300909} -{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271300911} -{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1697271300912} -{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyet", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1697271306644} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-18"}, "emitted_at": 1697271297222} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-19"}, "emitted_at": 1697271297224} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-20"}, "emitted_at": 1697271297224} -{"stream": "display_keyword_view", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 10012000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 1, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10012, "metrics.active_view_measurable_impressions": 1, "metrics.active_view_viewability": 1.0, "ad_group.id": 143992182864, "ad_group.name": "Video Non-skippable - 2022-05-30", "ad_group.status": "ENABLED", "segments.ad_network_type": "YOUTUBE_WATCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 10012000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/143992182864", "campaign.base_campaign": "customers/4651612872/campaigns/17354032686", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "TARGET_CPM", "campaign.id": 17354032686, "campaign.name": "Video Non-skippable - 2022-05-30", "campaign.status": "ENABLED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10012, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.keyword.text": "big data software", "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.day_of_week": "TUESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 26160872903, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: false\n"], "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-30", "segments.year": 2022, "segments.date": "2022-05-31"}, "emitted_at": 1697271287298} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1697271283467} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1697271283469} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1697271283470} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-18"}, "emitted_at": 1697271305208} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-19"}, "emitted_at": 1697271305210} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-20"}, "emitted_at": 1697271305210} -{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "data integration software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 18697003}, "emitted_at": 1697271284967} -{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "informatica software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 3, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 27723800}, "emitted_at": 1697271284970} -{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl extract transform load", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 4, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 439152736}, "emitted_at": 1697271284972} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1697271312107} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1697271312109} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1697271312110} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271308851} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271308851} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271308852} -{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2356, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2356~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271303374} -{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2484, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2484~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271303379} -{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.targeting_location": true, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpm": 36190476.190476194, "metrics.average_cpv": 0.0, "metrics.clicks": 3, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.14285714285714285, "metrics.impressions": 21, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.14285714285714285, "metrics.interactions": 3, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271303382} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 19750000.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 4, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.1, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 40, "metrics.interaction_rate": 0.1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 4, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.75, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1707920792924} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH_PARTNERS", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 0, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 11, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1707920792925} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 37391304.347826086, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 6, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2608695652173913, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-19", "segments.day_of_week": "THURSDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 23, "metrics.interaction_rate": 0.2608695652173913, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 6, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.7391304347826086, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1707920792932} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-18", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 15490196.078431372, "metrics.average_cpv": 0.0, "metrics.clicks": 4, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0784313725490196, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 51, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.0784313725490196, "metrics.interactions": 4, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746549} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-19", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 31851851.85185185, "metrics.average_cpv": 0.0, "metrics.clicks": 6, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2222222222222222, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 27, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.2222222222222222, "metrics.interactions": 6, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746559} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-20", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 215000.0, "metrics.average_cpc": 215000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 16538461.53846154, "metrics.average_cpv": 0.0, "metrics.clicks": 2, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 430000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.07692307692307693, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 26, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.07692307692307693, "metrics.interactions": 2, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746561} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-18"}, "emitted_at": 1704407754204} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-19"}, "emitted_at": 1704407754210} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-20"}, "emitted_at": 1704407754210} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 27142857.14285714, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 3, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.10714285714285714, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 28, "metrics.interaction_rate": 0.10714285714285714, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 3, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6935849056603773, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2852830188679245, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755279} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 30000.0, "metrics.average_cpc": 30000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 2500000.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 1, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 30000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.08333333333333333, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 12, "metrics.interaction_rate": 0.08333333333333333, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.7254437869822485, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2603550295857988, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755286} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 0, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.9001, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755286} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756403} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756404} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756405} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "data integration software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 18697003}, "emitted_at": 1704407757564} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "informatica software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 3, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 27723800}, "emitted_at": 1704407757565} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl extract transform load", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 4, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 439152736}, "emitted_at": 1704407757565} +{"stream": "display_keyword_view", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 10012000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 1, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10012, "metrics.active_view_measurable_impressions": 1, "metrics.active_view_viewability": 1.0, "ad_group.id": 143992182864, "ad_group.name": "Video Non-skippable - 2022-05-30", "ad_group.status": "ENABLED", "segments.ad_network_type": "YOUTUBE", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 10012000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/143992182864", "campaign.base_campaign": "customers/4651612872/campaigns/17354032686", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "TARGET_CPM", "campaign.id": 17354032686, "campaign.name": "Video Non-skippable - 2022-05-30", "campaign.status": "ENABLED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10012, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.keyword.text": "big data software", "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.day_of_week": "TUESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 26160872903, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: false\n"], "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-30", "segments.year": 2022, "segments.date": "2022-05-31"}, "emitted_at": 1704407759165} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-18"}, "emitted_at": 1704407765438} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-19"}, "emitted_at": 1704407765455} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-20"}, "emitted_at": 1704407765456} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 790000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-18"}, "emitted_at": 1704715893659} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 860000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-19"}, "emitted_at": 1704715893662} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 430000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-20"}, "emitted_at": 1704715893662} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-18"}, "emitted_at": 1704407768194} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-19"}, "emitted_at": 1704407768194} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-20"}, "emitted_at": 1704407768195} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 1, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769633} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 2, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769640} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 3, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769646} +{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771173} +{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771175} +{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704407771175} +{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771465} +{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771468} +{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704407771468} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1704407771926} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1704407771929} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2356, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2356~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772615} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2484, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2484~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772615} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.targeting_location": true, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpm": 36190476.190476194, "metrics.average_cpv": 0.0, "metrics.clicks": 3, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.14285714285714285, "metrics.impressions": 21, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.14285714285714285, "metrics.interactions": 3, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772616} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-18"}, "emitted_at": 1704407774246} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-19"}, "emitted_at": 1704407774254} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-20"}, "emitted_at": 1704407774254} +{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyte", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1704407775427} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1704407775721} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1704407775723} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777549} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777550} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777551} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1704407779851} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1704407779852} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1704407779852} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1704407780704} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1704407780707} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1704407780713} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1704407781887} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1704407781888} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1704407781889} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integrations", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integrations", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786207} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integration services", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integration services", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786208} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "cloud data integration", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "cloud data integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786209} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823748} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823749} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823750} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1704407848182} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1704407848186} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849655} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849656} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849656} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl index 347ceb81c084..62c6a529e5bc 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl @@ -1,73 +1,67 @@ -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 322678.1115879828, "metrics.active_view_ctr": 0.001430615164520744, "metrics.active_view_impressions": 699, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 225552, "metrics.active_view_measurable_impressions": 868, "metrics.active_view_viewability": 0.8052995391705069, "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 225552.0, "metrics.average_cpc": 225552.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 259852.53456221198, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 1, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0999, "metrics.content_rank_lost_impression_share": 0.9001, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 225552, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.001152073732718894, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 868, "metrics.interaction_rate": 0.001152073732718894, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271735465} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 849618.904224174, "metrics.active_view_ctr": 0.001589293182768716, "metrics.active_view_impressions": 11955, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10157194, "metrics.active_view_measurable_impressions": 12717, "metrics.active_view_viewability": 0.9400802075961312, "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 534589.1578947369, "metrics.average_cpc": 534589.1578947369, "metrics.average_cpe": 0.0, "metrics.average_cpm": 798709.915860659, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 19, "metrics.content_budget_lost_impression_share": 0.9001, "metrics.content_impression_share": 0.0999, "metrics.content_rank_lost_impression_share": 0.008972127521976223, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10157194, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0014940630651883305, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 12717, "metrics.interaction_rate": 0.0014940630651883305, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 19, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271735468} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 768549.1573033708, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 712, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 547207, "metrics.active_view_measurable_impressions": 740, "metrics.active_view_viewability": 0.9621621621621622, "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 739468.9189189189, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.9001, "metrics.content_impression_share": 0.0999, "metrics.content_rank_lost_impression_share": 0.002987251805929501, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 547207, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 740, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271735469} -{"stream": "ad_group", "data": {"campaign.id": 19410069806, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.campaign": "customers/4651612872/campaigns/19410069806", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 2000000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": true, "ad_group.final_url_suffix": "", "ad_group.id": 144799120517, "ad_group.labels": [], "ad_group.name": "Ad group 1", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/144799120517", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "DISPLAY_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2023-08-31"}, "emitted_at": 1697271745136} -{"stream": "ad_group", "data": {"campaign.id": 19410069806, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.campaign": "customers/4651612872/campaigns/19410069806", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 2000000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": true, "ad_group.final_url_suffix": "", "ad_group.id": 144799120517, "ad_group.labels": [], "ad_group.name": "Ad group 1", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/144799120517", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "DISPLAY_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2023-09-01"}, "emitted_at": 1697271745140} -{"stream": "ad_group", "data": {"campaign.id": 19410069806, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.campaign": "customers/4651612872/campaigns/19410069806", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 2000000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": true, "ad_group.final_url_suffix": "", "ad_group.id": 144799120517, "ad_group.labels": [], "ad_group.name": "Ad group 1", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/144799120517", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "DISPLAY_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2023-09-02"}, "emitted_at": 1697271745143} -{"stream": "ad_group_ad", "data": {"ad_group.id": 144799120517, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/643022056303", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": true, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": true, "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/144799120517~643022056303", "ad_group_ad.status": "ENABLED", "segments.date": "2023-08-31"}, "emitted_at": 1697271744432} -{"stream": "ad_group_ad", "data": {"ad_group.id": 144799120517, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/643022056303", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": true, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": true, "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/144799120517~643022056303", "ad_group_ad.status": "ENABLED", "segments.date": "2023-09-01"}, "emitted_at": 1697271744436} -{"stream": "ad_group_ad", "data": {"ad_group.id": 144799120517, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/643022056303", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": true, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": true, "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/144799120517~643022056303", "ad_group_ad.status": "ENABLED", "segments.date": "2023-09-02"}, "emitted_at": 1697271744439} -{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1697271747853} -{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1697271747855} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 144799120517, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 817743.0046386354, "metrics.active_view_ctr": 0.0014963339817447255, "metrics.active_view_impressions": 13366, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10929953, "metrics.active_view_measurable_impressions": 14325, "metrics.active_view_viewability": 0.9330541012216405, "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 546497.65, "metrics.average_cpc": 546497.65, "metrics.average_cpe": 0.0, "metrics.average_cpm": 762998.4642233857, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 20, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10929953, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0013961605584642235, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 14325, "metrics.interaction_rate": 0.0013961605584642235, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 20, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2023-08-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2023-07-01", "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271733030} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 144799120517, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 801880.5223334146, "metrics.active_view_ctr": 0.00183060776177691, "metrics.active_view_impressions": 16388, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 13141218, "metrics.active_view_measurable_impressions": 17733, "metrics.active_view_viewability": 0.9241527096373992, "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 438040.6, "metrics.average_cpc": 438040.6, "metrics.average_cpe": 0.0, "metrics.average_cpm": 741060.0575198783, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 30, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 13141218, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.001691761123329386, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2023-09-01", "segments.day_of_week": "FRIDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 17733, "metrics.interaction_rate": 0.001691761123329386, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 30, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2023-09-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2023-07-01", "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271733033} -{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 144799120517, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 789581.4073226545, "metrics.active_view_ctr": 0.002059496567505721, "metrics.active_view_impressions": 17480, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 13801883, "metrics.active_view_measurable_impressions": 19049, "metrics.active_view_viewability": 0.917633471573311, "ad_group_ad.ad_group": "customers/4651612872/adGroups/144799120517", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "ad_group_ad.ad_strength": "AVERAGE", "ad_group_ad.ad.type": "RESPONSIVE_DISPLAY_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 383385.6388888889, "metrics.average_cpc": 383385.6388888889, "metrics.average_cpe": 0.0, "metrics.average_cpm": 724546.3278912279, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 36, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 13801883, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0018898629849335923, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2023-09-02", "segments.day_of_week": "SATURDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 643022056303, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 19049, "metrics.interaction_rate": 0.0018898629849335923, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 36, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2023-09-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": true, "ad_group_ad.ad.responsive_display_ad.business_name": "c", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": ["text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\n", "text: \"\u30c6\u30ec\u30d3CM\u3067\u8a71\u984c\"\n", "text: \"modern ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "ALL_FORMATS", "ad_group_ad.ad.responsive_display_ad.headlines": ["text: \"ELT\"\n"], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "text: \"ELT data portability\"\n", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": ["asset: \"customers/4651612872/assets/39906715549\"\n", "asset: \"customers/4651612872/assets/40179016327\"\n"], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": ["asset: \"customers/4651612872/assets/39906715552\"\n", "asset: \"customers/4651612872/assets/40179014782\"\n"], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2023-07-01", "ad_group_ad.ad.responsive_search_ad.descriptions": [], "ad_group_ad.ad.responsive_search_ad.headlines": [], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271733037} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 144799120517, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-08-31"}, "emitted_at": 1697271757536} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 144799120517, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-09-01"}, "emitted_at": 1697271757538} -{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 144799120517, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-09-02"}, "emitted_at": 1697271757539} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 10515001, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data warehouses", "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data warehouses", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10515001", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271760071} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "UNSPECIFIED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 10683521, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "database software", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 0, "ad_group_criterion.effective_cpm_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "database software", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10683521", "ad_group_criterion.status": "REMOVED", "ad_group_criterion.system_serving_status": "UNSPECIFIED", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271760078} -{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 123273719655, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/123273719655", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 11100571, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "integration software", "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "integration software", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~11100571", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271760082} -{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1697271767616} -{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1697271767617} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.date": "2023-08-31"}, "emitted_at": 1697271734552} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.date": "2023-09-01"}, "emitted_at": 1697271734554} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.date": "2023-09-02"}, "emitted_at": 1697271734555} -{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271747340} -{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271747342} -{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1697271747342} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10515001", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 10515001, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271765107} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~10683521", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 10683521, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271765112} -{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/123273719655~11100571", "ad_group.id": 123273719655, "ad_group_criterion.criterion_id": 11100571, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1697271765116} -{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1697271751027} -{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1697271751029} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "DISPLAY", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12199001897", "campaign_budget.amount_micros": 13000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 19410069806, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Brand awareness and reach-Display-1", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "UNKNOWN", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/19410069806", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-12-28", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": [], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 1, "metrics.ctr": 0.0013440860215053765, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 514885, "metrics.impressions": 744, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 762792.5925925926, "metrics.active_view_ctr": 0.0014814814814814814, "metrics.active_view_impressions": 675, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 514885, "metrics.active_view_measurable_impressions": 744, "metrics.active_view_viewability": 0.907258064516129, "metrics.average_cost": 514885.0, "metrics.average_cpc": 514885.0, "metrics.average_cpm": 692049.7311827956, "metrics.interactions": 1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-08-31", "segments.hour": 0, "segments.ad_network_type": "CONTENT"}, "emitted_at": 1697271746262} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "DISPLAY", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12199001897", "campaign_budget.amount_micros": 13000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 19410069806, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Brand awareness and reach-Display-1", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "UNKNOWN", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/19410069806", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-12-28", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": [], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 646461, "metrics.impressions": 916, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 753451.048951049, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 858, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 646461, "metrics.active_view_measurable_impressions": 916, "metrics.active_view_viewability": 0.9366812227074236, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 705743.4497816594, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-08-31", "segments.hour": 1, "segments.ad_network_type": "CONTENT"}, "emitted_at": 1697271746265} -{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "DISPLAY", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12199001897", "campaign_budget.amount_micros": 13000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 19410069806, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Brand awareness and reach-Display-1", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "UNKNOWN", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/19410069806", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-12-28", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": [], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 1, "metrics.ctr": 0.0009746588693957114, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 720408, "metrics.impressions": 1026, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 759924.0506329114, "metrics.active_view_ctr": 0.0010548523206751054, "metrics.active_view_impressions": 948, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 720408, "metrics.active_view_measurable_impressions": 1026, "metrics.active_view_viewability": 0.9239766081871345, "metrics.average_cost": 720408.0, "metrics.average_cpc": 720408.0, "metrics.average_cpm": 702152.0467836257, "metrics.interactions": 1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-08-31", "segments.hour": 2, "segments.ad_network_type": "CONTENT"}, "emitted_at": 1697271746271} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 19410069806, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-08-31"}, "emitted_at": 1697271756998} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 19959839954, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-08-31"}, "emitted_at": 1697271757002} -{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 19410069806, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-09-01"}, "emitted_at": 1697271757005} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 19410069806, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 13000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 12199001897, "campaign_budget.name": "Brand awareness and reach-Display-1", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/12199001897", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2023-08-31", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/19410069806", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 546497.65, "metrics.average_cpc": 546497.65, "metrics.average_cpe": 0.0, "metrics.average_cpm": 762998.4642233857, "metrics.average_cpv": 0.0, "metrics.clicks": 20, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 10929953, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0013961605584642235, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 14325, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.0013961605584642235, "metrics.interactions": 20, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271733660} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 19959839954, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 10000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 12516167002, "campaign_budget.name": "Performance Max-5", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/12516167002", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2023-08-31", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/19959839954", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271733667} -{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 19410069806, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 13000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 12199001897, "campaign_budget.name": "Brand awareness and reach-Display-1", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/12199001897", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2023-09-01", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/19410069806", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 438040.6, "metrics.average_cpc": 438040.6, "metrics.average_cpe": 0.0, "metrics.average_cpm": 741060.0575198783, "metrics.average_cpv": 0.0, "metrics.clicks": 30, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 13141218, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.001691761123329386, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 17733, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.001691761123329386, "metrics.interactions": 30, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271733672} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271768336} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271768339} -{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1697271768341} -{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271746911} -{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1697271746913} -{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1697271746914} -{"stream": "click_view", "data": {"ad_group.name": "Ad group 1", "click_view.gclid": "Cj0KCQjw9MCnBhCYARIsAB1WQVUIRLTdkamVscRiXquP1eII_RaEdkhIWLZH-4pheDHLztk14iU0q6AaAnAqEALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/144799120517~643022056303", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 19410069806, "ad_group.id": 144799120517, "segments.date": "2023-08-31", "customer.id": 4651612872, "campaign.name": "Brand awareness and reach-Display-1", "segments.ad_network_type": "CONTENT", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1697271737196} -{"stream": "click_view", "data": {"ad_group.name": "Ad group 1", "click_view.gclid": "Cj0KCQjw9MCnBhCYARIsAB1WQVUW_kB70LlXrJjshH0iPvI55YQpnWPqPBbZiyCyh3XThG84yXubRJQaArTeEALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/144799120517~643022056303", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 19410069806, "ad_group.id": 144799120517, "segments.date": "2023-08-31", "customer.id": 4651612872, "campaign.name": "Brand awareness and reach-Display-1", "segments.ad_network_type": "CONTENT", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1697271737199} -{"stream": "click_view", "data": {"ad_group.name": "Ad group 1", "click_view.gclid": "Cj0KCQjw9MCnBhCYARIsAB1WQVVEPTT3Df-dDQUH5AFq2jb7BFWVIF1iouyQLuBlmdZOW2jlPn51t8waAn4UEALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/144799120517~643022056303", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 19410069806, "ad_group.id": 144799120517, "segments.date": "2023-08-31", "customer.id": 4651612872, "campaign.name": "Brand awareness and reach-Display-1", "segments.ad_network_type": "CONTENT", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": false, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1697271737201} -{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyet", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1697271750618} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2023-08-31"}, "emitted_at": 1697271745506} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2023-09-01"}, "emitted_at": 1697271745507} -{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7099339, "customer.optimization_score_weight": 12517.84, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2023-09-02"}, "emitted_at": 1697271745508} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2410, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 144799120517, "segments.date": "2023-08-31"}, "emitted_at": 1697271739358} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2410, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 144799120517, "segments.date": "2023-08-31"}, "emitted_at": 1697271739359} -{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2410, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 144799120517, "segments.date": "2023-09-01"}, "emitted_at": 1697271739360} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-12-28", "campaign.end_date": "2037-12-30", "segments.date": "2023-08-31"}, "emitted_at": 1697271749746} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-04-10", "campaign.end_date": "2037-12-30", "segments.date": "2023-08-31"}, "emitted_at": 1697271749748} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-12-28", "campaign.end_date": "2037-12-30", "segments.date": "2023-09-01"}, "emitted_at": 1697271749748} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1697271756219} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1697271756221} -{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1697271756222} -{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~945751797", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 443457.14285714284, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 35, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 15521, "metrics.active_view_measurable_impressions": 44, "metrics.active_view_viewability": 0.7954545454545454, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 352750.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 15521, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Online Communities"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 945751797, "metrics.impressions": 44, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/299", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271741823} -{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543464477", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 422666.6666666667, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 3, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 1268, "metrics.active_view_measurable_impressions": 11, "metrics.active_view_viewability": 0.2727272727272727, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 115272.72727272726, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 1268, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Shopping"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543464477, "metrics.impressions": 11, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/18", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271741829} -{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543465137", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 312102.08816705336, "metrics.active_view_ctr": 0.002320185614849188, "metrics.active_view_impressions": 431, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 134516, "metrics.active_view_measurable_impressions": 509, "metrics.active_view_viewability": 0.8467583497053045, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 134516.0, "metrics.average_cpc": 134516.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 264275.0491159135, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 1, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 134516, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Arts & Entertainment"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0019646365422396855, "segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543465137, "metrics.impressions": 509, "metrics.interaction_rate": 0.0019646365422396855, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2023-08-01", "segments.quarter": "2023-07-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/3", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-08-28", "segments.year": 2023}, "emitted_at": 1697271741832} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271753038} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271753039} -{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1697271753039} -{"stream": "user_location_view", "data": {"segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.month": "2023-08-01", "segments.week": "2023-08-28", "segments.quarter": "2023-07-01", "segments.year": 2023, "segments.ad_network_type": "CONTENT", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2040, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2040~false", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 69333.33333333333, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 208, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271748629} -{"stream": "user_location_view", "data": {"segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.month": "2023-08-01", "segments.week": "2023-08-28", "segments.quarter": "2023-07-01", "segments.year": 2023, "segments.ad_network_type": "CONTENT", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~false", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271748634} -{"stream": "user_location_view", "data": {"segments.date": "2023-08-31", "segments.day_of_week": "THURSDAY", "segments.month": "2023-08-01", "segments.week": "2023-08-28", "segments.quarter": "2023-07-01", "segments.year": 2023, "segments.ad_network_type": "CONTENT", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2344, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2344~false", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 612683.3333333333, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 36761, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 60, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1697271748637} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 155311392438, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 5.657287433333334, "metrics.all_conversions_value": 833.910118, "metrics.all_conversions": 169.718623, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 5602666.666666667, "metrics.average_cpc": 5602666.666666667, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1031165644.1717792, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "ENABLED", "metrics.clicks": 30, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.22079663333333333, "metrics.conversions_value": 662.3899, "metrics.conversions": 6.623899, "metrics.cost_micros": 168080000, "metrics.cost_per_all_conversions": 990345.0607185282, "metrics.cost_per_conversion": 25374783.039415307, "metrics.cost_per_current_model_attributed_conversion": 25374783.039415307, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 1.0, "metrics.ctr": 0.18404907975460122, "metrics.current_model_attributed_conversions_value": 662.3899, "metrics.current_model_attributed_conversions": 6.623899, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 163, "metrics.interaction_rate": 0.18404907975460122, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 30, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2023-12-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2023-10-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.8834355828220859, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 4.9134862353909154, "metrics.value_per_conversion": 100.0, "metrics.value_per_current_model_attributed_conversion": 100.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1707926876037} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 155311392438, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 5.881101891891892, "metrics.all_conversions_value": 1404.284328, "metrics.all_conversions": 217.60077, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 8890000.0, "metrics.average_cpc": 8890000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1279883268.4824903, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "ENABLED", "metrics.clicks": 37, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.4133928378378378, "metrics.conversions_value": 1184.5535, "metrics.conversions": 15.295535, "metrics.cost_micros": 328930000, "metrics.cost_per_all_conversions": 1511621.4892070463, "metrics.cost_per_conversion": 21504968.606851608, "metrics.cost_per_current_model_attributed_conversion": 21504968.606851608, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 3.0, "metrics.ctr": 0.14396887159533073, "metrics.current_model_attributed_conversions_value": 1184.5535, "metrics.current_model_attributed_conversions": 15.295535, "segments.date": "2024-01-01", "segments.day_of_week": "MONDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 257, "metrics.interaction_rate": 0.14396887159533073, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 37, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2024-01-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2024-01-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.9688715953307393, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 6.45348970042707, "metrics.value_per_conversion": 77.44439798934788, "metrics.value_per_current_model_attributed_conversion": 77.44439798934788, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1707926876046} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 150000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 13022493317, "campaign_budget.name": "mm_search_brand", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2023-12-31", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/20643300404", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.average_cost": 5602666.666666667, "metrics.average_cpc": 5602666.666666667, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1031165644.1717792, "metrics.average_cpv": 0.0, "metrics.clicks": 30, "metrics.cost_micros": 168080000, "metrics.ctr": 0.18404907975460122, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 163, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.18404907975460122, "metrics.interactions": 30, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1705322166925} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 150000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 13022493317, "campaign_budget.name": "mm_search_brand", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2024-01-02", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/20643300404", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.average_cost": 2976862.745098039, "metrics.average_cpc": 2976862.745098039, "metrics.average_cpe": 0.0, "metrics.average_cpm": 780565552.6992288, "metrics.average_cpv": 0.0, "metrics.clicks": 102, "metrics.cost_micros": 303640000, "metrics.ctr": 0.2622107969151671, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 389, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.2622107969151671, "metrics.interactions": 102, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1705322166925} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2024-01-03"}, "emitted_at": 1704408105935} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.date": "2024-01-03"}, "emitted_at": 1704408105942} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2024-01-02"}, "emitted_at": 1704408105943} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 2.9861930909090906, "metrics.all_conversions_value": 32.848124, "metrics.all_conversions": 32.848124, "metrics.average_cost": 1398181.8181818181, "metrics.average_cpc": 1398181.8181818181, "metrics.average_cpe": 0.0, "metrics.average_cpm": 640833333.3333334, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 11, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 15380000, "metrics.cost_per_all_conversions": 468215.4755626227, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.4583333333333333, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 24, "metrics.interaction_rate": 0.4583333333333333, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 11, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-12-01", "segments.quarter": "2023-10-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.6666666666666666, "metrics.search_impression_share": 0.6153846153846154, "metrics.search_rank_lost_impression_share": 0.38461538461538464, "metrics.value_per_all_conversions": 1.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1704408106623} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 2, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-12-01", "segments.quarter": "2023-10-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 1.0, "metrics.search_impression_share": 1.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1704408106623} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT0aoRchs-JIhSNfsaUU1GQLPOaNU15XNhGEkNLQ0kpOpYoV_VDNNogaAl-2EALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107339} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT17gRC4RsmoYczHLguLKTaojzCB4bPA0GjBSa3x44kKTbWVCvXEe58aAkeHEALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107340} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT1H36_GC-jRtw1xNj-9Y5IdIZWa-1j-BqhYt5JSB82QzNE5-7OxgB4aAlU4EALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107340} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 155311392438, "segments.date": "2023-12-31"}, "emitted_at": 1704408109676} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 155311392438, "segments.date": "2023-12-31"}, "emitted_at": 1704408109677} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 155311392438, "segments.date": "2024-01-01"}, "emitted_at": 1704408109677} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~945751797", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 264196.96969696967, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 66, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 17437, "metrics.active_view_measurable_impressions": 90, "metrics.active_view_viewability": 0.7333333333333333, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 193744.44444444444, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 17437, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Online Communities"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 945751797, "metrics.impressions": 90, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/299", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113977} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543464477", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 862000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 2, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 1724, "metrics.active_view_measurable_impressions": 4, "metrics.active_view_viewability": 0.5, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 431000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 1724, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Shopping"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543464477, "metrics.impressions": 4, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/18", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113979} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543465137", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 338986.6666666667, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 75, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 25424, "metrics.active_view_measurable_impressions": 104, "metrics.active_view_viewability": 0.7211538461538461, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 244461.53846153844, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 25424, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Arts & Entertainment"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543465137, "metrics.impressions": 104, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/3", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113986} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2023-12-31"}, "emitted_at": 1707927302757} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2024-01-01"}, "emitted_at": 1707927302758} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2024-01-02"}, "emitted_at": 1707927302759} +{"stream": "ad_group", "data": {"campaign.id": 20643300404, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.campaign": "customers/4651612872/campaigns/20643300404", "metrics.cost_micros": 168080000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 155311392438, "ad_group.labels": [], "ad_group.name": "Airbyte", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/155311392438", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airbyte\"\n", "key: \"campaign\"\nvalue: \"mm_search_brand\"\n"], "segments.date": "2023-12-31"}, "emitted_at": 1704717743436} +{"stream": "ad_group", "data": {"campaign.id": 20643300404, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.campaign": "customers/4651612872/campaigns/20643300404", "metrics.cost_micros": 328930000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 155311392438, "ad_group.labels": [], "ad_group.name": "Airbyte", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/155311392438", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airbyte\"\n", "key: \"campaign\"\nvalue: \"mm_search_brand\"\n"], "segments.date": "2024-01-01"}, "emitted_at": 1704717743438} +{"stream": "ad_group", "data": {"campaign.id": 20655886237, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/153930342465", "ad_group.campaign": "customers/4651612872/campaigns/20655886237", "metrics.cost_micros": 27110000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 153930342465, "ad_group.labels": [], "ad_group.name": "Airflow", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/153930342465", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airflow\"\n", "key: \"campaign\"\nvalue: \"mm_search_competitors\"\n"], "segments.date": "2024-01-02"}, "emitted_at": 1704717743440} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2023-12-31"}, "emitted_at": 1704408117407} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2024-01-01"}, "emitted_at": 1704408117408} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2024-01-02"}, "emitted_at": 1704408117408} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.amount_micros": 150000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 20643300404, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "mm_search_brand", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.5967968177178755, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/20643300404", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2023-10-10", "campaign.status": "ENABLED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=mm_search_brand&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=20643300404&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-12-31", "segments.hour": 3, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1708533414290} +{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119170} +{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119172} +{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704408119173} +{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119522} +{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119524} +{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704408119525} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1704408119841} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1704408119844} +{"stream":"user_location_view","data":{"segments.date":"2023-12-31","segments.day_of_week":"SUNDAY","segments.month":"2023-12-01","segments.week":"2023-12-25","segments.quarter":"2023-10-01","segments.year":2023,"segments.ad_network_type":"SEARCH","customer.currency_code":"USD","customer.id":4651612872,"customer.descriptive_name":"Airbyte","customer.time_zone":"America/Los_Angeles","user_location_view.country_criterion_id":2124,"user_location_view.targeting_location":false,"user_location_view.resource_name":"customers/4651612872/userLocationViews/2124~false","campaign.base_campaign":"customers/4651612872/campaigns/20643300404","campaign.id":20643300404,"campaign.name":"mm_search_brand","campaign.status":"ENABLED","ad_group.name":"Airbyte","ad_group.status":"ENABLED","ad_group.base_ad_group":"customers/4651612872/adGroups/155311392438","metrics.all_conversions":0.0,"metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.clicks":0,"metrics.conversions":0.0,"metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.impressions":3,"metrics.interaction_event_types":[],"metrics.interaction_rate":0.0,"metrics.interactions":0,"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0},"emitted_at":1707482532099} +{"stream":"user_location_view","data":{"segments.date":"2023-12-31","segments.day_of_week":"SUNDAY","segments.month":"2023-12-01","segments.week":"2023-12-25","segments.quarter":"2023-10-01","segments.year":2023,"segments.ad_network_type":"SEARCH","customer.currency_code":"USD","customer.id":4651612872,"customer.descriptive_name":"Airbyte","customer.time_zone":"America/Los_Angeles","user_location_view.country_criterion_id":2356,"user_location_view.targeting_location":false,"user_location_view.resource_name":"customers/4651612872/userLocationViews/2356~false","campaign.base_campaign":"customers/4651612872/campaigns/20643300404","campaign.id":20643300404,"campaign.name":"mm_search_brand","campaign.status":"ENABLED","ad_group.name":"Airbyte","ad_group.status":"ENABLED","ad_group.base_ad_group":"customers/4651612872/adGroups/155311392438","metrics.all_conversions":0.0,"metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.clicks":0,"metrics.conversions":0.0,"metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.impressions":1,"metrics.interaction_event_types":[],"metrics.interaction_rate":0.0,"metrics.interactions":0,"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0},"emitted_at":1707482532106} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2023-12-31"}, "emitted_at": 1704408121315} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2024-01-01"}, "emitted_at": 1704408121315} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2024-01-02"}, "emitted_at": 1704408121315} +{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyte", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1704408121936} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1704408122314} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1704408122315} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124247} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124249} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124250} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1704408126496} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1704408126498} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1704408126499} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-12-31"}, "emitted_at": 1704408127194} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-01"}, "emitted_at": 1704408127198} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20637264648, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-02"}, "emitted_at": 1704408127200} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 155311392438, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-12-31"}, "emitted_at": 1704408127574} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 155311392438, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-01"}, "emitted_at": 1704408127581} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 154050719199, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-02"}, "emitted_at": 1704408127583} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integrations", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integrations", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130758} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integration services", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integration services", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130764} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "cloud data integration", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "cloud data integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130766} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168252} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168257} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168258} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1704408192425} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1704408192426} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194062} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194068} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194068} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py index 46660cc99b5d..65f544425f63 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py @@ -7,7 +7,7 @@ import pytest from airbyte_cdk.models import SyncMode -from google.ads.googleads.v13.services.types.google_ads_service import GoogleAdsRow +from google.ads.googleads.v15.services.types.google_ads_service import GoogleAdsRow from source_google_ads.source import SourceGoogleAds @@ -101,11 +101,11 @@ def create_google_ads_row_from_dict(data: dict) -> GoogleAdsRow: "campaign.id": 11112, "campaign.name": "Campaign 1", "campaign.status": "UNKNOWN", - "segments.product_bidding_category_level1": "Electronics", - "segments.product_bidding_category_level2": "Mobile Phones", - "segments.product_bidding_category_level3": "Smartphones", - "segments.product_bidding_category_level4": "Android", - "segments.product_bidding_category_level5": "Samsung", + "segments.product_category_level1": "Electronics", + "segments.product_category_level2": "Mobile Phones", + "segments.product_category_level3": "Smartphones", + "segments.product_category_level4": "Android", + "segments.product_category_level5": "Samsung", "segments.product_channel": "UNSPECIFIED", "segments.product_channel_exclusivity": "SINGLE_CHANNEL", "segments.click_type": "APP_DEEPLINK", @@ -162,11 +162,11 @@ def create_google_ads_row_from_dict(data: dict) -> GoogleAdsRow: "campaign.id": 11112, "campaign.name": "Campaign 1", "campaign.status": "UNKNOWN", - "segments.product_bidding_category_level1": "Electronics", - "segments.product_bidding_category_level2": "Mobile Phones", - "segments.product_bidding_category_level3": "Smartphones", - "segments.product_bidding_category_level4": "Android", - "segments.product_bidding_category_level5": "Samsung", + "segments.product_category_level1": "Electronics", + "segments.product_category_level2": "Mobile Phones", + "segments.product_category_level3": "Smartphones", + "segments.product_category_level4": "Android", + "segments.product_category_level5": "Samsung", "segments.product_channel": "UNSPECIFIED", "segments.product_channel_exclusivity": "SINGLE_CHANNEL", "segments.click_type": "APP_DEEPLINK", diff --git a/airbyte-integrations/connectors/source-google-ads/main.py b/airbyte-integrations/connectors/source-google-ads/main.py index 74d321502526..2824c4955943 100644 --- a/airbyte-integrations/connectors/source-google-ads/main.py +++ b/airbyte-integrations/connectors/source-google-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_ads import SourceGoogleAds +from source_google_ads.run import run if __name__ == "__main__": - source = SourceGoogleAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml index c69732e864ff..ed883a5080af 100644 --- a/airbyte-integrations/connectors/source-google-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-ads/metadata.yaml @@ -7,17 +7,21 @@ data: - accounts.google.com - googleads.googleapis.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 - dockerImageTag: 2.0.4 + dockerImageTag: 3.3.4 dockerRepository: airbyte/source-google-ads documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads githubIssueLabel: source-google-ads icon: google-adwords.svg license: Elv2 name: Google Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-ads registries: cloud: enabled: true @@ -39,6 +43,11 @@ data: API. Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. upgradeDeadline: "2023-11-30" + 3.0.0: + message: Google is deprecating v13 of the Google Ads API in January. + This release upgrades the Google Ads API to the latest version (v15), which causes changes in several schemas. + Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. + upgradeDeadline: "2024-01-12" suggestedStreams: streams: - campaigns diff --git a/airbyte-integrations/connectors/source-google-ads/poetry.lock b/airbyte-integrations/connectors/source-google-ads/poetry.lock new file mode 100644 index 000000000000..f6e98089cef2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/poetry.lock @@ -0,0 +1,1345 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-ads" +version = "22.1.0" +description = "Client library for the Google Ads API" +optional = false +python-versions = ">=3.7, <3.12" +files = [ + {file = "google-ads-22.1.0.tar.gz", hash = "sha256:cfab38b40eb8424a4a514823bd8b911a57ef55dd64e2112cfa46a70d8090de98"}, + {file = "google_ads-22.1.0-py3-none-any.whl", hash = "sha256:6fdd3fb635678fbb3c8f87271afc81f0e139882b83b48505160fc4daacf33ad0"}, +] + +[package.dependencies] +google-api-core = ">=2.8.0,<=3.0.0" +google-auth-oauthlib = ">=0.3.0,<2.0.0" +googleapis-common-protos = ">=1.56.0,<2.0.0" +grpcio = ">=1.38.1,<2.0.0" +grpcio-status = ">=1.38.1,<2.0.0" +proto-plus = ">=1.19.6,<2.0.0" +protobuf = ">=3.12.0,<3.18.dev0 || >=3.20.dev0,<5.0.0" +PyYAML = ">=5.1,<7.0" +setuptools = ">=40.3.0" + +[package.extras] +tests = ["nox (>=2020.12.31,<2022.6)"] + +[[package]] +name = "google-api-core" +version = "2.17.0" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.17.0.tar.gz", hash = "sha256:de7ef0450faec7c75e0aea313f29ac870fdc44cfaec9d6499a9a17305980ef66"}, + {file = "google_api_core-2.17.0-py3-none-any.whl", hash = "sha256:08ed79ed8e93e329de5e3e7452746b734e6bf8438d8d64dd3319d21d3164890c"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.27.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-oauthlib" +version = "1.2.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"}, + {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"}, +] + +[package.dependencies] +google-auth = ">=2.15.0" +requests-oauthlib = ">=0.7.0" + +[package.extras] +tool = ["click (>=6.0.0)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "grpcio" +version = "1.60.1" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, + {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, + {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, + {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, + {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, + {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, + {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, + {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, + {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, + {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, + {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, + {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, + {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, + {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, + {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, + {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, + {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, + {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, + {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, + {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, + {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, + {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, + {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, + {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.60.1)"] + +[[package]] +name = "grpcio-status" +version = "1.60.1" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.60.1.tar.gz", hash = "sha256:61b5aab8989498e8aa142c20b88829ea5d90d18c18c853b9f9e6d407d37bf8b4"}, + {file = "grpcio_status-1.60.1-py3-none-any.whl", hash = "sha256:3034fdb239185b6e0f3169d08c268c4507481e4b8a434c21311a03d9eb5889a0"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.60.1" +protobuf = ">=4.21.6" + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "proto-plus" +version = "1.23.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, + {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "995b656ad5991d34f517389e1e26276ebf1052e6fc4af5f3955b6a5d9c7cd2ef" diff --git a/airbyte-integrations/connectors/source-google-ads/pyproject.toml b/airbyte-integrations/connectors/source-google-ads/pyproject.toml new file mode 100644 index 000000000000..0535b4fa62d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.3.4" +name = "source-google-ads" +description = "Source implementation for Google Ads." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/google-ads" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_google_ads" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +google-ads = "==22.1.0" +protobuf = "==4.25.2" +pendulum = "==2.1.2" +airbyte-cdk = "==0.58.8" + +[tool.poetry.scripts] +source-google-ads = "source_google_ads.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.12.0" +requests-mock = "^1.11.0" +freezegun = "^1.4.0" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-google-ads/requirements.txt b/airbyte-integrations/connectors/source-google-ads/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py deleted file mode 100644 index 581c3b2d0159..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -# pin protobuf==3.20.0 as other versions may cause problems on different architectures -# (see https://github.com/airbytehq/airbyte/issues/13580) -MAIN_REQUIREMENTS = ["airbyte-cdk>=0.51.3", "google-ads==20.0.0", "protobuf", "pendulum"] - -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "freezegun", "requests-mock"] - -setup( - name="source_google_ads", - description="Source implementation for Google Ads.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py new file mode 100644 index 000000000000..be206ee13e62 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.utils import AirbyteTracedException + +from .utils import GAQL + +logger = logging.getLogger("airbyte_logger") + +FULL_REFRESH_CUSTOM_TABLE = [ + "asset", + "asset_group_listing_group_filter", + "custom_audience", + "geo_target_constant", + "change_event", + "change_status", +] + + +class MigrateCustomQuery: + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `custom_queries`. + + Add `segments.date` for all queries where it was previously added by IncrementalCustomQuery class. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + Determines if a configuration requires migration. + + Args: + - config (Mapping[str, Any]): The configuration data to check. + + Returns: + - True: If the configuration requires migration. + - False: Otherwise. + """ + return "custom_queries_array" not in config + + @classmethod + def update_custom_queries(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + """ + Update custom queries with segments.date field. + + Args: + - config (Mapping[str, Any]): The configuration from which the key should be removed. + - source (Source, optional): The data source. Defaults to None. + + Returns: + - Mapping[str, Any]: The configuration after removing the key. + """ + custom_queries = [] + for query in config.get("custom_queries", []): + new_query = query.copy() + try: + query_object = GAQL.parse(query["query"]) + except ValueError: + message = f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + if query_object.resource_name not in FULL_REFRESH_CUSTOM_TABLE and "segments.date" not in query_object.fields: + query_object = query_object.append_field("segments.date") + + new_query["query"] = str(query_object) + custom_queries.append(new_query) + + config["custom_queries_array"] = custom_queries + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Modifies the configuration and then saves it back to the source. + + Args: + - config_path (str): The path where the configuration is stored. + - source (Source): The data source. + - config (Mapping[str, Any]): The current configuration. + + Returns: + - Mapping[str, Any]: The updated configuration. + """ + migrated_config = cls.update_custom_queries(config, source) + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + """ + Emits the control messages related to configuration migration. + + Args: + - migrated_config (Mapping[str, Any]): The migrated configuration. + """ + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the configuration migration process. + + It first checks if the `--config` argument is provided, and if so, + determines whether migration is needed, and then performs the migration + if required. + + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + if cls.should_migrate(config): + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 358290f5b906..4a3ac096cfdd 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -2,12 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from functools import lru_cache from typing import Any, Dict, Mapping from .streams import GoogleAdsStream, IncrementalGoogleAdsStream from .utils import GAQL +DATE_TYPES = ("segments.date", "segments.month", "segments.quarter", "segments.week") + class CustomQueryMixin: def __init__(self, config, **kwargs): @@ -66,7 +69,8 @@ def get_json_schema(self) -> Dict[str, Any]: google_data_type = node.data_type.name field_value = {"type": [google_datatype_mapping.get(google_data_type, "string"), "null"]} - if google_data_type == "DATE": + # Google Ads doesn't differentiate between DATE and DATETIME, so we need to manually check for fields with known type + if google_data_type == "DATE" and field in DATE_TYPES: field_value["format"] = "date" if google_data_type == "ENUM": diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index e21f3906531d..09ebafcd9267 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -5,18 +5,18 @@ import logging from enum import Enum -from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping +from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional import backoff from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.client import GoogleAdsClient -from google.ads.googleads.v13.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse +from google.ads.googleads.v15.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse from google.api_core.exceptions import InternalServerError, ServerError, TooManyRequests from google.auth import exceptions from proto.marshal.collections import Repeated, RepeatedComposite -API_VERSION = "v13" +API_VERSION = "v15" logger = logging.getLogger("airbyte") @@ -27,8 +27,28 @@ def __init__(self, credentials: MutableMapping[str, Any]): # `google-ads` library version `14.0.0` and higher requires an additional required parameter `use_proto_plus`. # More details can be found here: https://developers.google.com/google-ads/api/docs/client-libs/python/protobuf-messages credentials["use_proto_plus"] = True - self.client = self.get_google_ads_client(credentials) - self.ga_service = self.client.get_service("GoogleAdsService") + self.clients = {} + self.ga_services = {} + self.credentials = credentials + + self.clients["default"] = self.get_google_ads_client(credentials) + self.ga_services["default"] = self.clients["default"].get_service("GoogleAdsService") + + self.customer_service = self.clients["default"].get_service("CustomerService") + + def get_client(self, login_customer_id="default"): + if login_customer_id in self.clients: + return self.clients[login_customer_id] + new_creds = self.credentials.copy() + new_creds["login_customer_id"] = login_customer_id + self.clients[login_customer_id] = self.get_google_ads_client(new_creds) + return self.clients[login_customer_id] + + def ga_service(self, login_customer_id="default"): + if login_customer_id in self.ga_services: + return self.ga_services[login_customer_id] + self.ga_services[login_customer_id] = self.clients[login_customer_id].get_service("GoogleAdsService") + return self.ga_services[login_customer_id] @staticmethod def get_google_ads_client(credentials) -> GoogleAdsClient: @@ -38,6 +58,14 @@ def get_google_ads_client(credentials) -> GoogleAdsClient: message = "The authentication to Google Ads has expired. Re-authenticate to restore access to Google Ads." raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) from e + def get_accessible_accounts(self): + customer_resource_names = self.customer_service.list_accessible_customers().resource_names + logger.info(f"Found {len(customer_resource_names)} accessible accounts: {customer_resource_names}") + + for customer_resource_name in customer_resource_names: + customer_id = self.ga_service().parse_customer_path(customer_resource_name)["customer_id"] + yield customer_id + @backoff.on_exception( backoff.expo, (InternalServerError, ServerError, TooManyRequests), @@ -46,13 +74,13 @@ def get_google_ads_client(credentials) -> GoogleAdsClient: ), max_tries=5, ) - def send_request(self, query: str, customer_id: str) -> Iterator[SearchGoogleAdsResponse]: - client = self.client + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default") -> Iterator[SearchGoogleAdsResponse]: + client = self.get_client(login_customer_id) search_request = client.get_type("SearchGoogleAdsRequest") search_request.query = query search_request.page_size = self.DEFAULT_PAGE_SIZE search_request.customer_id = customer_id - return [self.ga_service.search(search_request)] + return [self.ga_service(login_customer_id).search(search_request)] def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: """ @@ -61,8 +89,8 @@ def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: :return dict of fields type info. """ - ga_field_service = self.client.get_service("GoogleAdsFieldService") - request = self.client.get_type("SearchGoogleAdsFieldsRequest") + ga_field_service = self.get_client().get_service("GoogleAdsFieldService") + request = self.get_client().get_type("SearchGoogleAdsFieldsRequest") request.page_size = len(fields) fields_sql = ",".join([f"'{field}'" for field in fields]) request.query = f""" @@ -170,14 +198,16 @@ def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[ elif isinstance(field_value, (Repeated, RepeatedComposite)): field_value = [str(value) for value in field_value] - # Google Ads has a lot of entities inside itself and we cannot process them all separately, because: + # Google Ads has a lot of entities inside itself, and we cannot process them all separately, because: # 1. It will take a long time # 2. We have no way to get data on absolutely all entities to test. # # To prevent JSON from throwing an error during deserialization, we made such a hack. # For example: - # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). - # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). + # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset + # (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). + # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo + # (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). if not isinstance(field_value, (list, int, float, str, bool, dict)) and field_value is not None: field_value = str(field_value) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py index 7b4296b19d5c..7da4ed7c2b9c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py @@ -2,28 +2,34 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from dataclasses import dataclass -from typing import Any, Iterable, Mapping, Union +from typing import Any, Iterable, Mapping -from pendulum import timezone +from pendulum import local_timezone, timezone from pendulum.tz.timezone import Timezone @dataclass class CustomerModel: id: str - time_zone: Union[timezone, str] = "local" + time_zone: timezone = local_timezone() is_manager_account: bool = False + login_customer_id: str = None @classmethod - def from_accounts(cls, accounts: Iterable[Iterable[Mapping[str, Any]]]): + def from_accounts(cls, accounts: Iterable[Mapping[str, Any]]) -> Iterable["CustomerModel"]: data_objects = [] - for account_list in accounts: - for account in account_list: - time_zone_name = account.get("customer.time_zone") - tz = Timezone(time_zone_name) if time_zone_name else "local" + for account in accounts: + time_zone_name = account.get("customer_client.time_zone") + tz = Timezone(time_zone_name) if time_zone_name else local_timezone() - data_objects.append( - cls(id=str(account["customer.id"]), time_zone=tz, is_manager_account=bool(account.get("customer.manager"))) + data_objects.append( + cls( + id=str(account["customer_client.id"]), + time_zone=tz, + is_manager_account=bool(account.get("customer_client.manager")), + login_customer_id=account.get("login_customer_id"), ) + ) return data_objects diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py new file mode 100644 index 000000000000..dd759a035015 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_ads import SourceGoogleAds +from source_google_ads.config_migrations import MigrateCustomQuery + + +def run(): + source = SourceGoogleAds() + MigrateCustomQuery.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json index 87f32300d809..96dbdc94edea 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json @@ -14,6 +14,9 @@ "ad_group.campaign": { "type": ["null", "string"] }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, "ad_group.cpc_bid_micros": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json index 0d205b62f057..fe5efc371589 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json @@ -41,10 +41,10 @@ "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": { "type": ["null", "string"] }, - "ad_group_criterion.listing_group.case_value.product_bidding_category.id": { + "ad_group_criterion.listing_group.case_value.product_category.category_id": { "type": ["null", "integer"] }, - "ad_group_criterion.listing_group.case_value.product_bidding_category.level": { + "ad_group_criterion.listing_group.case_value.product_category.level": { "type": ["null", "string"] }, "ad_group_criterion.listing_group.case_value.product_brand.value": { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json index 0afac0449237..6b7d4f334ca5 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json @@ -170,9 +170,6 @@ "campaign.shopping_setting.merchant_id": { "type": ["null", "integer"] }, - "campaign.shopping_setting.sales_country": { - "type": ["null", "string"] - }, "campaign.start_date": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json new file mode 100644 index 000000000000..efb4bfd93f78 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer_client.client_customer": { + "type": ["null", "boolean"] + }, + "customer_client.level": { + "type": ["null", "string"] + }, + "customer_client.id": { + "type": ["null", "integer"] + }, + "customer_client.manager": { + "type": ["null", "boolean"] + }, + "customer_client.time_zone": { + "type": ["null", "number"] + }, + "customer_client.status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json index 965e5e3857d3..f679be52592c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json @@ -44,19 +44,19 @@ "campaign.status": { "type": ["null", "string"] }, - "segments.product_bidding_category_level1": { + "segments.product_category_level1": { "type": ["null", "string"] }, - "segments.product_bidding_category_level2": { + "segments.product_category_level2": { "type": ["null", "string"] }, - "segments.product_bidding_category_level3": { + "segments.product_category_level3": { "type": ["null", "string"] }, - "segments.product_bidding_category_level4": { + "segments.product_category_level4": { "type": ["null", "string"] }, - "segments.product_bidding_category_level5": { + "segments.product_category_level5": { "type": ["null", "string"] }, "segments.product_channel": { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index aa535bd68535..2402cd18adbe 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -34,6 +34,7 @@ CampaignLabel, ClickView, Customer, + CustomerClient, CustomerLabel, DisplayKeywordView, GeographicView, @@ -47,14 +48,7 @@ ) from .utils import GAQL -FULL_REFRESH_CUSTOM_TABLE = [ - "asset", - "asset_group_listing_group_filter", - "custom_audience", - "geo_target_constant", - "change_event", - "change_status", -] +logger = logging.getLogger("airbyte") class SourceGoogleAds(AbstractSource): @@ -65,24 +59,28 @@ class SourceGoogleAds(AbstractSource): def _validate_and_transform(config: Mapping[str, Any]): if config.get("end_date") == "": config.pop("end_date") - for query in config.get("custom_queries", []): + for query in config.get("custom_queries_array", []): try: query["query"] = GAQL.parse(query["query"]) except ValueError: - message = f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + message = ( + f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. " + "https://developers.google.com/google-ads/api/fields/v15/query_validator" + ) raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + if "customer_id" in config: + config["customer_ids"] = config["customer_id"].split(",") + config.pop("customer_id") + return config @staticmethod def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]: credentials = config["credentials"] - # use_proto_plus is set to True, because setting to False returned wrong value types, which breakes the backward compatibility. + # use_proto_plus is set to True, because setting to False returned wrong value types, which breaks the backward compatibility. # For more info read the related PR's description: https://github.com/airbytehq/airbyte/pull/9996 credentials.update(use_proto_plus=True) - - # https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid - if "login_customer_id" in config and config["login_customer_id"].strip(): - credentials["login_customer_id"] = config["login_customer_id"] return credentials @staticmethod @@ -104,11 +102,45 @@ def get_incremental_stream_config(google_api: GoogleAds, config: Mapping[str, An ) return incremental_stream_config - def get_account_info(self, google_api: GoogleAds, config: Mapping[str, Any]) -> Iterable[Iterable[Mapping[str, Any]]]: - dummy_customers = [CustomerModel(id=_id) for _id in config["customer_id"].split(",")] - accounts_stream = ServiceAccounts(google_api, customers=dummy_customers) - for slice_ in accounts_stream.stream_slices(): - yield accounts_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice_) + def get_all_accounts(self, google_api: GoogleAds, customers: List[CustomerModel], customer_status_filter: List[str]) -> List[str]: + customer_clients_stream = CustomerClient(api=google_api, customers=customers, customer_status_filter=customer_status_filter) + for slice in customer_clients_stream.stream_slices(): + for record in customer_clients_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): + yield record + + def _get_all_connected_accounts( + self, google_api: GoogleAds, customer_status_filter: List[str] + ) -> Iterable[Iterable[Mapping[str, Any]]]: + customer_ids = [customer_id for customer_id in google_api.get_accessible_accounts()] + dummy_customers = [CustomerModel(id=_id, login_customer_id=_id) for _id in customer_ids] + + yield from self.get_all_accounts(google_api, dummy_customers, customer_status_filter) + + def get_customers(self, google_api: GoogleAds, config: Mapping[str, Any]) -> List[CustomerModel]: + customer_status_filter = config.get("customer_status_filter", []) + accounts = self._get_all_connected_accounts(google_api, customer_status_filter) + customers = CustomerModel.from_accounts(accounts) + + # filter duplicates as one customer can be accessible from mutiple connected accounts + unique_customers = [] + seen_ids = set() + for customer in customers: + if customer.id in seen_ids: + continue + seen_ids.add(customer.id) + unique_customers.append(customer) + customers = unique_customers + customers_dict = {customer.id: customer for customer in customers} + + # filter only selected accounts + if config.get("customer_ids"): + customers = [] + for customer_id in config["customer_ids"]: + if customer_id not in customers_dict: + logging.warning(f"Customer with id {customer_id} is not accessible. Skipping it.") + else: + customers.append(customers_dict[customer_id]) + return customers @staticmethod def is_metrics_in_custom_query(query: GAQL) -> bool: @@ -117,30 +149,64 @@ def is_metrics_in_custom_query(query: GAQL) -> bool: return True return False + @staticmethod + def is_custom_query_incremental(query: GAQL) -> bool: + time_segment_in_select, time_segment_in_where = ["segments.date" in clause for clause in [query.fields, query.where]] + return time_segment_in_select and not time_segment_in_where + + def create_custom_query_stream( + self, + google_api: GoogleAds, + single_query_config: Mapping[str, Any], + customers: List[CustomerModel], + non_manager_accounts: List[CustomerModel], + incremental_config: Mapping[str, Any], + non_manager_incremental_config: Mapping[str, Any], + ): + query = single_query_config["query"] + is_incremental = self.is_custom_query_incremental(query) + is_non_manager = self.is_metrics_in_custom_query(query) + + if is_non_manager: + # Skip query with metrics if there are no non-manager accounts + if not non_manager_accounts: + return + + customers = non_manager_accounts + incremental_config = non_manager_incremental_config + + if is_incremental: + return IncrementalCustomQuery(config=single_query_config, **incremental_config) + else: + return CustomQuery(config=single_query_config, api=google_api, customers=customers) + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: config = self._validate_and_transform(config) logger.info("Checking the config") google_api = GoogleAds(credentials=self.get_credentials(config)) - accounts = self.get_account_info(google_api, config) - customers = CustomerModel.from_accounts(accounts) - # Check custom query request validity by sending metric request with non-existant time window + customers = self.get_customers(google_api, config) + logger.info(f"Found {len(customers)} customers: {[customer.id for customer in customers]}") + + # Check custom query request validity by sending metric request with non-existent time window for customer in customers: - for query in config.get("custom_queries", []): + for query in config.get("custom_queries_array", []): query = query["query"] if customer.is_manager_account and self.is_metrics_in_custom_query(query): logger.warning( f"Metrics are not available for manager account {customer.id}. " - f"Please remove metrics fields in your custom query: {query}." + f'Skipping the custom query: "{query}" for manager account.' ) - if query.resource_name not in FULL_REFRESH_CUSTOM_TABLE: - if IncrementalCustomQuery.cursor_field in query.fields: - message = f"Custom query should not contain {IncrementalCustomQuery.cursor_field}" - raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) + continue + + # Add segments.date to where clause of incremental custom queries if they are not present. + # The same will be done during read, but with start and end date from config + if self.is_custom_query_incremental(query): query = IncrementalCustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01") + query = query.set_limit(1) - response = google_api.send_request(str(query), customer_id=customer.id) + response = google_api.send_request(str(query), customer_id=customer.id, login_customer_id=customer.login_customer_id) # iterate over the response otherwise exceptions will not be raised! for _ in response: pass @@ -149,8 +215,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self._validate_and_transform(config) google_api = GoogleAds(credentials=self.get_credentials(config)) - accounts = self.get_account_info(google_api, config) - customers = CustomerModel.from_accounts(accounts) + + customers = self.get_customers(google_api, config) + logger.info(f"Found {len(customers)} customers: {[customer.id for customer in customers]}") + non_manager_accounts = [customer for customer in customers if not customer.is_manager_account] default_config = dict(api=google_api, customers=customers) incremental_config = self.get_incremental_stream_config(google_api, config, customers) @@ -190,17 +258,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: KeywordView(**non_manager_incremental_config), ] ) - for single_query_config in config.get("custom_queries", []): - query = single_query_config["query"] - if self.is_metrics_in_custom_query(query): - if non_manager_accounts: - if query.resource_name in FULL_REFRESH_CUSTOM_TABLE: - streams.append(CustomQuery(config=single_query_config, api=google_api, customers=non_manager_accounts)) - else: - streams.append(IncrementalCustomQuery(config=single_query_config, **non_manager_incremental_config)) - continue - if query.resource_name in FULL_REFRESH_CUSTOM_TABLE: - streams.append(CustomQuery(config=single_query_config, api=google_api, customers=customers)) - else: - streams.append(IncrementalCustomQuery(config=single_query_config, **incremental_config)) + + for single_query_config in config.get("custom_queries_array", []): + query_stream = self.create_custom_query_stream( + google_api, single_query_config, customers, non_manager_accounts, incremental_config, non_manager_incremental_config + ) + if query_stream: + streams.append(query_stream) return streams diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 8a15796874fc..2b84f6bc1beb 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Ads Spec", "type": "object", - "required": ["credentials", "customer_id"], + "required": ["credentials"], "additionalProperties": true, "properties": { "credentials": { @@ -64,6 +64,18 @@ "examples": ["6783948572,5839201945"], "order": 1 }, + "customer_status_filter": { + "title": "Customer Statuses Filter", + "description": "A list of customer statuses to filter on. For detailed info about what each status mean refer to Google Ads documentation.", + "default": [], + "order": 2, + "type": "array", + "items": { + "title": "CustomerStatus", + "description": "An enumeration.", + "enum": ["UNKNOWN", "ENABLED", "CANCELED", "SUSPENDED", "CLOSED"] + } + }, "start_date": { "type": "string", "title": "Start Date", @@ -71,7 +83,7 @@ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-25"], - "order": 2, + "order": 3, "format": "date" }, "end_date": { @@ -81,14 +93,14 @@ "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-30"], - "order": 6, + "order": 4, "format": "date" }, - "custom_queries": { + "custom_queries_array": { "type": "array", "title": "Custom GAQL Queries", "description": "", - "order": 3, + "order": 5, "items": { "type": "object", "required": ["query", "table_name"], @@ -110,15 +122,6 @@ } } }, - "login_customer_id": { - "type": "string", - "title": "Login Customer ID for Managed Accounts", - "description": "If your access to the customer account is through a manager account, this field is required, and must be set to the 10-digit customer ID of the manager account. For more information about this field, refer to Google's documentation.", - "pattern_descriptor": ": 10 digits, with no dashes.", - "pattern": "^([0-9]{10})?$", - "examples": ["7349206847"], - "order": 4 - }, "conversion_window_days": { "title": "Conversion Window", "type": "integer", @@ -127,7 +130,7 @@ "maximum": 1095, "default": 14, "examples": [14], - "order": 5 + "order": 6 } } }, diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index af7bf549b25d..499bceca367e 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from abc import ABC, abstractmethod from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional @@ -13,13 +14,13 @@ from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v13.services.services.google_ads_service.pagers import SearchPager -from google.ads.googleads.v13.services.types.google_ads_service import SearchGoogleAdsResponse -from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests +from google.ads.googleads.v15.services.services.google_ads_service.pagers import SearchPager +from google.ads.googleads.v15.services.types.google_ads_service import SearchGoogleAdsResponse +from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests, Unauthenticated from .google_ads import GoogleAds, logger from .models import CustomerModel -from .utils import ExpiredPageTokenError, chunk_date_range, generator_backoff, get_resource_name, parse_dates, traced_exception +from .utils import ExpiredPageTokenError, chunk_date_range, detached, generator_backoff, get_resource_name, parse_dates, traced_exception class GoogleAdsStream(Stream, ABC): @@ -41,19 +42,36 @@ def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[s def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: for customer in self.customers: - yield {"customer_id": customer.id} + yield {"customer_id": customer.id, "login_customer_id": customer.login_customer_id} + + @generator_backoff( + wait_gen=backoff.constant, + exception=(TimeoutError), + max_tries=5, + on_backoff=lambda details: logger.info( + f"Caught retryable error {details['exception']} after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + interval=1, + ) + @detached(timeout_minutes=5) + def request_records_job(self, customer_id, login_customer_id, query, stream_slice): + response_records = self.google_ads_client.send_request(query=query, customer_id=customer_id, login_customer_id=login_customer_id) + yield from self.parse_records_with_backoff(response_records, stream_slice) def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: if stream_slice is None: return [] customer_id = stream_slice["customer_id"] - try: - response_records = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=customer_id) + login_customer_id = stream_slice["login_customer_id"] - yield from self.parse_records_with_backoff(response_records, stream_slice) - except GoogleAdsException as exception: + try: + yield from self.request_records_job(customer_id, login_customer_id, self.get_query(stream_slice), stream_slice) + except (GoogleAdsException, Unauthenticated) as exception: traced_exception(exception, customer_id, self.CATCH_CUSTOMER_NOT_ENABLED_ERROR) + except TimeoutError as exception: + # Prevent sync failure + logger.warning(f"Timeout: Failed to access {self.name} stream data. {str(exception)}") @generator_backoff( wait_gen=backoff.expo, @@ -133,6 +151,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite ): if chunk: chunk["customer_id"] = customer.id + chunk["login_customer_id"] = customer.login_customer_id yield chunk def _update_state(self, customer_id: str, record: MutableMapping[str, Any]): @@ -200,7 +219,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: class Customer(IncrementalGoogleAdsStream): """ - Customer stream: https://developers.google.com/google-ads/api/fields/v11/customer + Customer stream: https://developers.google.com/google-ads/api/fields/v15/customer """ primary_key = ["customer.id", "segments.date"] @@ -212,9 +231,65 @@ def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[s yield record +class CustomerClient(GoogleAdsStream): + """ + Customer Client stream: https://developers.google.com/google-ads/api/fields/v15/customer_client + """ + + primary_key = ["customer_client.id"] + + def __init__(self, customer_status_filter: List[str], **kwargs): + self.customer_status_filter = customer_status_filter + super().__init__(**kwargs) + + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + table_name = get_resource_name(self.name) + + active_customers_condition = [] + if self.customer_status_filter: + customer_status_filter = ", ".join([f"'{status}'" for status in self.customer_status_filter]) + active_customers_condition = [f"customer_client.status in ({customer_status_filter})"] + + query = GoogleAds.convert_schema_into_query(fields=fields, table_name=table_name, conditions=active_customers_condition) + return query + + def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This method is overridden to avoid using login_customer_id from dummy_customers. + + login_customer_id is used in the stream_slices to pass it to child customers, + but we don't need it here as this class iterate over customers accessible from user creds. + """ + if stream_slice is None: + return [] + + customer_id = stream_slice["customer_id"] + + try: + response_records = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=customer_id) + + yield from self.parse_records_with_backoff(response_records, stream_slice) + except GoogleAdsException as exception: + traced_exception(exception, customer_id, self.CATCH_CUSTOMER_NOT_ENABLED_ERROR) + + def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: + """ + login_cusotmer_id is populated to child customers if they are under managers account + """ + records = [record for record in super().parse_response(response)] + + # read_records get all customers connected to customer_id from stream_slice + # if the result is more than one customer, it's a manager, otherwise it is client account for which we don't need login_customer_id + root_is_manager = len(records) > 1 + for record in records: + record["login_customer_id"] = stream_slice["login_customer_id"] if root_is_manager else "default" + yield record + + class CustomerLabel(GoogleAdsStream): """ - Customer Label stream: https://developers.google.com/google-ads/api/fields/v14/customer_label + Customer Label stream: https://developers.google.com/google-ads/api/fields/v15/customer_label """ primary_key = ["customer_label.resource_name"] @@ -231,7 +306,7 @@ class ServiceAccounts(GoogleAdsStream): class Campaign(IncrementalGoogleAdsStream): """ - Campaign stream: https://developers.google.com/google-ads/api/fields/v11/campaign + Campaign stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -240,7 +315,7 @@ class Campaign(IncrementalGoogleAdsStream): class CampaignBudget(IncrementalGoogleAdsStream): """ - Campaigns stream: https://developers.google.com/google-ads/api/fields/v13/campaign_budget + Campaigns stream: https://developers.google.com/google-ads/api/fields/v15/campaign_budget """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -255,7 +330,7 @@ class CampaignBudget(IncrementalGoogleAdsStream): class CampaignBiddingStrategy(IncrementalGoogleAdsStream): """ - Campaign Bidding Strategy stream: https://developers.google.com/google-ads/api/fields/v14/campaign + Campaign Bidding Strategy stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -264,7 +339,7 @@ class CampaignBiddingStrategy(IncrementalGoogleAdsStream): class CampaignLabel(GoogleAdsStream): """ - Campaign labels stream: https://developers.google.com/google-ads/api/fields/v11/campaign_label + Campaign labels stream: https://developers.google.com/google-ads/api/fields/v15/campaign_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. @@ -273,15 +348,30 @@ class CampaignLabel(GoogleAdsStream): class AdGroup(IncrementalGoogleAdsStream): """ - AdGroup stream: https://developers.google.com/google-ads/api/fields/v11/ad_group + AdGroup stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ primary_key = ["ad_group.id", "segments.date"] + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + # validation that the customer is not a manager + # due to unsupported metrics.cost_micros field and removing it in case custom is a manager + if [customer for customer in self.customers if customer.id == stream_slice["customer_id"]][0].is_manager_account: + fields = [field for field in fields if field != "metrics.cost_micros"] + table_name = get_resource_name(self.name) + start_date, end_date = stream_slice.get("start_date"), stream_slice.get("end_date") + cursor_condition = [f"{self.cursor_field} >= '{start_date}' AND {self.cursor_field} <= '{end_date}'"] + + query = GoogleAds.convert_schema_into_query( + fields=fields, table_name=table_name, conditions=cursor_condition, order_field=self.cursor_field + ) + return query + class AdGroupLabel(GoogleAdsStream): """ - Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_label + Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. @@ -290,7 +380,7 @@ class AdGroupLabel(GoogleAdsStream): class AdGroupBiddingStrategy(IncrementalGoogleAdsStream): """ - Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v14/ad_group + Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -299,7 +389,7 @@ class AdGroupBiddingStrategy(IncrementalGoogleAdsStream): class AdGroupCriterionLabel(GoogleAdsStream): """ - Ad Group Criterion Label stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label + Ad Group Criterion Label stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion_label """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -308,7 +398,7 @@ class AdGroupCriterionLabel(GoogleAdsStream): class AdGroupAd(IncrementalGoogleAdsStream): """ - Ad Group Ad stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad + Ad Group Ad stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad """ primary_key = ["ad_group.id", "ad_group_ad.ad.id", "segments.date"] @@ -316,7 +406,7 @@ class AdGroupAd(IncrementalGoogleAdsStream): class AdGroupAdLabel(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ primary_key = ["ad_group.id", "ad_group_ad.ad.id", "label.id"] @@ -324,7 +414,7 @@ class AdGroupAdLabel(GoogleAdsStream): class AccountPerformanceReport(IncrementalGoogleAdsStream): """ - AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/customer + AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v15/customer Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance """ @@ -333,7 +423,7 @@ class AccountPerformanceReport(IncrementalGoogleAdsStream): class AdGroupAdLegacy(IncrementalGoogleAdsStream): """ - AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad + AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance """ @@ -342,7 +432,7 @@ class AdGroupAdLegacy(IncrementalGoogleAdsStream): class DisplayKeywordView(IncrementalGoogleAdsStream): """ - DisplayKeywordView stream: https://developers.google.com/google-ads/api/fields/v11/display_keyword_view + DisplayKeywordView stream: https://developers.google.com/google-ads/api/fields/v15/display_keyword_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance """ @@ -357,7 +447,7 @@ class DisplayKeywordView(IncrementalGoogleAdsStream): class TopicView(IncrementalGoogleAdsStream): """ - DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/topic_view + DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v15/topic_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance """ @@ -372,14 +462,14 @@ class TopicView(IncrementalGoogleAdsStream): class ShoppingPerformanceView(IncrementalGoogleAdsStream): """ - ShoppingPerformanceView stream: https://developers.google.com/google-ads/api/fields/v11/shopping_performance_view + ShoppingPerformanceView stream: https://developers.google.com/google-ads/api/fields/v15/shopping_performance_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance """ class UserLocationView(IncrementalGoogleAdsStream): """ - UserLocationView stream: https://developers.google.com/google-ads/api/fields/v11/user_location_view + UserLocationView stream: https://developers.google.com/google-ads/api/fields/v15/user_location_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#geo_performance """ @@ -394,7 +484,7 @@ class UserLocationView(IncrementalGoogleAdsStream): class GeographicView(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/geographic_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/geographic_view """ primary_key = ["customer.id", "geographic_view.country_criterion_id", "geographic_view.location_type", "segments.date"] @@ -402,7 +492,7 @@ class GeographicView(IncrementalGoogleAdsStream): class KeywordView(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/keyword_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/keyword_view """ primary_key = ["ad_group.id", "ad_group_criterion.criterion_id", "segments.date"] @@ -410,7 +500,7 @@ class KeywordView(IncrementalGoogleAdsStream): class ClickView(IncrementalGoogleAdsStream): """ - ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v11/ClickView + ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v15/ClickView """ primary_key = ["click_view.gclid", "segments.date", "segments.ad_network_type"] @@ -422,7 +512,7 @@ class ClickView(IncrementalGoogleAdsStream): class UserInterest(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ primary_key = ["user_interest.user_interest_id"] @@ -430,7 +520,7 @@ class UserInterest(GoogleAdsStream): class Audience(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ primary_key = ["customer.id", "audience.id"] @@ -438,7 +528,7 @@ class Audience(GoogleAdsStream): class Label(GoogleAdsStream): """ - Label stream: https://developers.google.com/google-ads/api/fields/v14/label + Label stream: https://developers.google.com/google-ads/api/fields/v15/label """ primary_key = ["label.id"] @@ -446,7 +536,7 @@ class Label(GoogleAdsStream): class ChangeStatus(IncrementalGoogleAdsStream): """ - Change status stream: https://developers.google.com/google-ads/api/fields/v14/change_status + Change status stream: https://developers.google.com/google-ads/api/fields/v15/change_status Stream is only used internally to implement incremental updates for child streams of IncrementalEventsStream """ @@ -463,7 +553,7 @@ def __init__(self, **kwargs): @property def query_limit(self) -> Optional[int]: - "Queries for ChangeStatus resource have to include limit in it" + """Queries for ChangeStatus resource have to include limit in it""" return 10000 def read_records( @@ -492,7 +582,7 @@ def read_records( # if state was not updated before hitting limit - raise error to avoid infinite loop if stream_slice["start_date"] == self.get_current_state(customer_id): raise AirbyteTracedException( - message=f"More then limit {self.query_limit} records with same cursor field. Incremental sync is not possible for this stream.", + message=f"More than limit {self.query_limit} records with same cursor field. Incremental sync is not possible for this stream.", failure_type=FailureType.system_error, ) @@ -537,19 +627,19 @@ def __init__(self, **kwargs): @property @abstractmethod def id_field(self) -> str: - "Name of field used for getting records by id" + """Name of field used for getting records by id""" pass @property @abstractmethod def parent_id_field(self) -> str: - "Field name of id from parent record" + """Field name of id from parent record""" pass @property @abstractmethod def resource_type(self) -> str: - "Resource type used for filtering parent records" + """Resource type used for filtering parent records""" pass @property @@ -573,7 +663,13 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite yield from slices_generator else: for customer in self.customers: - yield {"customer_id": customer.id, "updated_ids": set(), "deleted_ids": set(), "record_changed_time_map": dict()} + yield { + "customer_id": customer.id, + "login_customer_id": customer.login_customer_id, + "updated_ids": set(), + "deleted_ids": set(), + "record_changed_time_map": dict(), + } def _process_parent_record(self, parent_record: MutableMapping[str, Any], child_slice: MutableMapping[str, Any]) -> bool: """Process a single parent_record and update the child_slice.""" @@ -597,7 +693,13 @@ def read_parent_stream( sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state.get(self.parent_stream_name) ): customer_id = parent_slice.get("customer_id") - child_slice = {"customer_id": customer_id, "updated_ids": set(), "deleted_ids": set(), "record_changed_time_map": dict()} + child_slice = { + "customer_id": customer_id, + "updated_ids": set(), + "deleted_ids": set(), + "record_changed_time_map": dict(), + "login_customer_id": parent_slice.get("login_customer_id"), + } if not self.get_current_state(customer_id): yield child_slice continue @@ -637,7 +739,8 @@ def _read_deleted_records(self, stream_slice: MutableMapping[str, Any] = None): for deleted_record_id in stream_slice.get("deleted_ids", []): yield {self.id_field: deleted_record_id, "deleted_at": stream_slice["record_changed_time_map"].get(deleted_record_id)} - def _split_slice(self, child_slice: MutableMapping[str, Any], chunk_size: int = 10000) -> Iterable[Mapping[str, Any]]: + @staticmethod + def _split_slice(child_slice: MutableMapping[str, Any], chunk_size: int = 10000) -> Iterable[Mapping[str, Any]]: """ Splits a child slice into smaller chunks based on the chunk_size. @@ -656,13 +759,20 @@ def _split_slice(self, child_slice: MutableMapping[str, Any], chunk_size: int = record_changed_time_map = child_slice["record_changed_time_map"] customer_id = child_slice["customer_id"] + login_customer_id = child_slice["login_customer_id"] # Split the updated_ids into chunks and yield them for i in range(0, len(updated_ids), chunk_size): chunk_ids = set(updated_ids[i : i + chunk_size]) chunk_time_map = {k: record_changed_time_map[k] for k in chunk_ids} - yield {"updated_ids": chunk_ids, "record_changed_time_map": chunk_time_map, "customer_id": customer_id, "deleted_ids": set()} + yield { + "updated_ids": chunk_ids, + "record_changed_time_map": chunk_time_map, + "customer_id": customer_id, + "deleted_ids": set(), + "login_customer_id": login_customer_id, + } def read_records( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_slice: MutableMapping[str, Any] = None, **kwargs @@ -700,7 +810,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: class AdGroupCriterion(IncrementalEventsStream): """ - Ad Group Criterion stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion + Ad Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -713,7 +823,7 @@ class AdGroupCriterion(IncrementalEventsStream): class AdListingGroupCriterion(AdGroupCriterion): """ - Ad Listing Group Criterion stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion + Ad Listing Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion While this stream utilizes the same resource as the AdGroupCriterions, it specifically targets the listing group and has distinct schemas. """ @@ -721,7 +831,7 @@ class AdListingGroupCriterion(AdGroupCriterion): class CampaignCriterion(IncrementalEventsStream): """ - Campaign Criterion stream: https://developers.google.com/google-ads/api/fields/v14/campaign_criterion + Campaign Criterion stream: https://developers.google.com/google-ads/api/fields/v15/campaign_criterion """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index 1b7f938f7748..3085343c9278 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -2,7 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import functools +import queue import re +import threading import time from dataclasses import dataclass from datetime import datetime @@ -12,10 +15,11 @@ from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v13.errors.types.authentication_error import AuthenticationErrorEnum -from google.ads.googleads.v13.errors.types.authorization_error import AuthorizationErrorEnum -from google.ads.googleads.v13.errors.types.quota_error import QuotaErrorEnum -from google.ads.googleads.v13.errors.types.request_error import RequestErrorEnum +from google.ads.googleads.v15.errors.types.authentication_error import AuthenticationErrorEnum +from google.ads.googleads.v15.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v15.errors.types.quota_error import QuotaErrorEnum +from google.ads.googleads.v15.errors.types.request_error import RequestErrorEnum +from google.api_core.exceptions import Unauthenticated from source_google_ads.google_ads import logger @@ -50,12 +54,19 @@ def is_error_type(error_value, target_enum_value): return int(error_value) == int(target_enum_value) -def traced_exception(ga_exception: GoogleAdsException, customer_id: str, catch_disabled_customer_error: bool): +def traced_exception(ga_exception: Union[GoogleAdsException, Unauthenticated], customer_id: str, catch_disabled_customer_error: bool): """Add user-friendly message for GoogleAdsException""" messages = [] raise_exception = AirbyteTracedException failure_type = FailureType.config_error + if isinstance(ga_exception, Unauthenticated): + message = ( + f"Authentication failed for the customer '{customer_id}'. " + f"Please try to Re-authenticate your credentials on set up Google Ads page." + ) + raise raise_exception.from_exception(failure_type=failure_type, exc=ga_exception, message=message) from ga_exception + for error in ga_exception.failure.errors: # Get error codes authorization_error = error.error_code.authorization_error @@ -179,6 +190,123 @@ def wrapper(*args, **kwargs) -> Generator: return decorator +class RunAsThread: + """ + The `RunAsThread` decorator is designed to run a generator function in a separate thread with a specified timeout. + This is particularly useful when dealing with functions that involve potentially time-consuming operations, + and you want to enforce a time limit for their execution. + """ + + def __init__(self, timeout_minutes): + """ + :param timeout_minutes: The maximum allowed time (in minutes) for the generator function to idle. + If the timeout is reached, a TimeoutError is raised. + """ + self._timeout_seconds = timeout_minutes * 60 + + def __call__(self, generator_func): + @functools.wraps(generator_func) + def wrapper(*args, **kwargs): + """ + The wrapper function sets up threading components, starts a separate thread to run the generator function. + It uses events and a queue for communication and synchronization between the main thread and the thread running the generator function. + """ + # Event and Queue initialization + write_event = threading.Event() + exit_event = threading.Event() + the_queue = queue.Queue() + + # Thread initialization and start + thread = threading.Thread( + target=self.target, args=(the_queue, write_event, exit_event, generator_func, args, kwargs), daemon=True + ) + thread.start() + + # Records the starting time for the timeout calculation. + start_time = time.time() + while thread.is_alive() or not the_queue.empty(): + # The main thread waits for the `write_event` to be set or until the specified timeout. + if the_queue.empty(): + write_event.wait(self._timeout_seconds) + try: + # The main thread yields the result obtained from reading the queue. + yield self.read(the_queue) + # The timer is reset since a new result has been received, preventing the timeout from occurring. + start_time = time.time() + except queue.Empty: + # If exit_event is set it means that the generator function in the thread has completed its execution. + if exit_event.is_set(): + break + # Check if the timeout has been reached without new results. + if time.time() - start_time > self._timeout_seconds: + # The thread may continue to run for some time after reaching a timeout and even come to life and continue working. + # That is why the exit event is set to signal the generator function to stop producing data. + exit_event.set() + raise TimeoutError(f"Method '{generator_func.__name__}' timed out after {self._timeout_seconds / 60.0} minutes") + # The write event is cleared to reset it for the next iteration. + write_event.clear() + + return wrapper + + def target(self, the_queue, write_event, exit_event, func, args, kwargs): + """ + This is a target function for the thread. + It runs the actual generator function, writing its results to a queue. + Exceptions raised during execution are also written to the queue. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param write_event: An event signaling the availability of new data in the queue. + :param exit_event: An event indicating whether the generator function should stop producing data due to a timeout. + :param func: The generator function to be executed. + :param args: Positional arguments for the generator function. + :param kwargs: Keyword arguments for the generator function. + :return: None + """ + try: + for value in func(*args, **kwargs): + # If the timeout has been reached we must stop producing any data + if exit_event.is_set(): + break + self.write(the_queue, value, write_event) + else: + # Notify the main thread that the generator function has completed its execution. + exit_event.set() + # Notify the main thread (even if the generator didn't produce any data) to prevent waiting for no reason. + if not write_event.is_set(): + write_event.set() + except Exception as e: + self.write(the_queue, e, write_event) + + @staticmethod + def write(the_queue, value, write_event): + """ + Puts a value into the queue and sets a write event to notify the main thread that new data is available. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param value: The value to be put into the communication queue. + This can be any type of data produced by the generator function, including results or exceptions. + :param write_event: An event signaling the availability of new data in the queue. + :return: None + """ + the_queue.put(value) + write_event.set() + + @staticmethod + def read(the_queue, timeout=0.001): + """ + Retrieves a value from the queue, handling the case where the value is an exception, and raising it. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param timeout: A time in seconds to wait for a value to be available in the queue. + If the timeout is reached and no new data is available, a `queue.Empty` exception is raised. + :return: a value retrieved from the queue + """ + value = the_queue.get(block=True, timeout=timeout) + if isinstance(value, Exception): + raise value + return value + + +detached = RunAsThread + + def parse_dates(stream_slice): start_date = pendulum.parse(stream_slice["start_date"]) end_date = pendulum.parse(stream_slice["end_date"]) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py index 1c0af02e0b2d..b2bff404d6e2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py @@ -2,14 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.errors.types.authorization_error import AuthorizationErrorEnum -from google.ads.googleads.v13 import GoogleAdsFailure -from google.ads.googleads.v13.errors.types.authentication_error import AuthenticationErrorEnum -from google.ads.googleads.v13.errors.types.query_error import QueryErrorEnum -from google.ads.googleads.v13.errors.types.quota_error import QuotaErrorEnum +from google.ads.googleads.v15 import GoogleAdsFailure +from google.ads.googleads.v15.errors.types.authentication_error import AuthenticationErrorEnum +from google.ads.googleads.v15.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v15.errors.types.query_error import QueryErrorEnum +from google.ads.googleads.v15.errors.types.quota_error import QuotaErrorEnum class MockSearchRequest: @@ -43,9 +44,12 @@ def get_service(self, service): def load_from_dict(config, version=None): return MockGoogleAdsClient(config) - def send_request(self, query, customer_id): + def send_request(self, query, customer_id, login_customer_id="none"): yield from () + def get_accessible_accounts(self): + yield from ["fake_customer_id", "fake_customer_id_2"] + class MockGoogleAdsFieldService: _instance = None diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py index 9e2287b951d3..7054284da5db 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest from source_google_ads.models import CustomerModel @@ -18,7 +19,7 @@ def test_config(): "customer_id": "123", "start_date": "2021-01-01", "conversion_window_days": 14, - "custom_queries": [ + "custom_queries_array": [ { "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", "primary_key": None, @@ -53,3 +54,8 @@ def mock_oauth_call(requests_mock): @pytest.fixture def customers(config): return [CustomerModel(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + + +@pytest.fixture +def customers_manager(config): + return [CustomerModel(id=_id, time_zone="local", is_manager_account=True) for _id in config["customer_id"].split(",")] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..4ca91bc77892 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_ads.config_migrations import MigrateCustomQuery +from source_google_ads.source import SourceGoogleAds + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_config.json" +NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleAds() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("custom_queries_array") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateCustomQuery() + original_config = load_config() + original_config_queries = original_config["custom_queries"].copy() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "custom_queries_array" in test_migrated_config + assert "segments.date" in test_migrated_config["custom_queries_array"][0]["query"] + # check the old property is in place + assert "custom_queries" in test_migrated_config + assert test_migrated_config["custom_queries"] == original_config_queries + assert "segments.date" not in test_migrated_config["custom_queries"][0]["query"] + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + new_config_queries = test_migrated_config["custom_queries_array"].copy() + new_config_queries[0]["query"] = new_config_queries[0]["query"].replace(", segments.date", "") + print(f"{original_config=} \n {test_migrated_config=}") + assert original_config["custom_queries"] == new_config_queries + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "custom_queries_array" not in test_config + # check the old property is still there + assert "custom_queries" in test_config + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + migration_instance = MigrateCustomQuery() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py index a2f0fd380c54..862324d3c237 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from unittest.mock import MagicMock from source_google_ads.custom_query_stream import CustomQueryMixin, IncrementalCustomQuery @@ -35,9 +36,10 @@ def test_get_json_schema(): "d": Obj(data_type=Obj(name="MESSAGE"), is_repeated=True), "e": Obj(data_type=Obj(name="STRING"), is_repeated=False), "f": Obj(data_type=Obj(name="DATE"), is_repeated=False), + "segments.month": Obj(data_type=Obj(name="DATE"), is_repeated=False), } ) - instance = CustomQueryMixin(config={"query": Obj(fields=["a", "b", "c", "d", "e", "f"])}) + instance = CustomQueryMixin(config={"query": Obj(fields=["a", "b", "c", "d", "e", "f", "segments.month"])}) instance.cursor_field = None instance.google_ads_client = Obj(get_fields_metadata=query_object) schema = instance.get_json_schema() @@ -52,6 +54,7 @@ def test_get_json_schema(): "c": {"type": ["string", "null"]}, "d": {"type": ["null", "array"], "items": {"type": ["string", "null"]}}, "e": {"type": ["string", "null"]}, - "f": {"type": ["string", "null"], "format": "date"}, + "f": {"type": ["string", "null"]}, + "segments.month": {"type": ["string", "null"], "format": "date"}, }, } diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_empty_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_empty_streams.py new file mode 100644 index 000000000000..1eddca1f39ab --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_empty_streams.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +from source_google_ads.google_ads import GoogleAds +from source_google_ads.streams import CustomerLabel, ShoppingPerformanceView + + +def test_query_customer_label_stream(customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + + stream_config = dict( + api=api, + customers=customers, + ) + stream = CustomerLabel(**stream_config) + assert ( + stream.get_query(stream_slice={"customer_id": "123"}) + == "SELECT customer_label.resource_name, customer_label.customer, customer.id, customer_label.label FROM customer_label" + ) + + +def test_query_shopping_performance_view_stream(customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + + stream_config = dict( + api=api, + start_date="2023-01-01 00:00:00.000000", + conversion_window_days=0, + customers=customers, + ) + stream = ShoppingPerformanceView(**stream_config) + stream_slice = { + "start_date": "2023-01-01 00:00:00.000000", + "end_date": "2023-09-19 00:00:00.000000", + "resource_type": "SOME_RESOURCE_TYPE", + "login_customer_id": "default", + } + expected_query = "SELECT customer.descriptive_name, ad_group.id, ad_group.name, ad_group.status, segments.ad_network_type, segments.product_aggregator_id, metrics.all_conversions_from_interactions_rate, metrics.all_conversions_value, metrics.all_conversions, metrics.average_cpc, segments.product_brand, campaign.id, campaign.name, campaign.status, segments.product_category_level1, segments.product_category_level2, segments.product_category_level3, segments.product_category_level4, segments.product_category_level5, segments.product_channel, segments.product_channel_exclusivity, segments.click_type, metrics.clicks, metrics.conversions_from_interactions_rate, metrics.conversions_value, metrics.conversions, metrics.cost_micros, metrics.cost_per_all_conversions, metrics.cost_per_conversion, segments.product_country, metrics.cross_device_conversions, metrics.ctr, segments.product_custom_attribute0, segments.product_custom_attribute1, segments.product_custom_attribute2, segments.product_custom_attribute3, segments.product_custom_attribute4, segments.date, segments.day_of_week, segments.device, customer.id, metrics.impressions, segments.product_language, segments.product_merchant_id, segments.month, segments.product_item_id, segments.product_condition, segments.product_title, segments.product_type_l1, segments.product_type_l2, segments.product_type_l3, segments.product_type_l4, segments.product_type_l5, segments.quarter, segments.product_store_id, metrics.value_per_all_conversions, metrics.value_per_conversion, segments.week, segments.year FROM shopping_performance_view WHERE segments.date >= '2023-01-01 00:00:00.000000' AND segments.date <= '2023-09-19 00:00:00.000000' ORDER BY segments.date ASC" + assert stream.get_query(stream_slice=stream_slice) == expected_query diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py index e333713f92b8..e71263296007 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from contextlib import nullcontext as does_not_raise from unittest.mock import Mock @@ -9,11 +10,21 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.utils import AirbyteTracedException from source_google_ads.google_ads import GoogleAds +from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds from source_google_ads.streams import AdGroupLabel, Label, ServiceAccounts from .common import MockGoogleAdsClient, mock_google_ads_request_failure + +@pytest.fixture +def mock_get_customers(mocker): + mocker.patch( + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=False, time_zone="Europe/Berlin", id="123")]), + ) + + params = [ ( ["USER_PERMISSION_DENIED"], @@ -23,15 +34,6 @@ ["CUSTOMER_NOT_FOUND"], "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", ), - ( - ["CUSTOMER_NOT_ENABLED"], - ( - "The customer account '123' hasn't finished signup or has been deactivated. " - "Sign in to the Google Ads UI to verify its status. " - "For reactivating deactivated accounts, refer to: " - "https://support.google.com/google-ads/answer/2375392." - ), - ), (["QUERY_ERROR"], "Incorrect custom query. Error in query: unexpected end of query."), ( ["RESOURCE_EXHAUSTED"], @@ -50,6 +52,10 @@ @pytest.mark.parametrize(("exception", "error_message"), params) def test_expected_errors(mocker, config, exception, error_message): mock_google_ads_request_failure(mocker, exception) + mocker.patch( + "source_google_ads.google_ads.GoogleAds.get_accessible_accounts", + Mock(return_value=["123", "12345"]), + ) source = SourceGoogleAds() with pytest.raises(AirbyteTracedException) as exception: status_ok, error = source.check_connection(AirbyteLogger(), config) @@ -73,7 +79,7 @@ def test_read_record_error_handling(mocker, config, customers, cls, raise_expect context = pytest.raises(AirbyteTracedException) if raise_expected else does_not_raise() with context as exception: - for _ in stream.read_records(sync_mode=Mock(), stream_slice={"customer_id": "1234567890"}): + for _ in stream.read_records(sync_mode=Mock(), stream_slice={"customer_id": "1234567890", "login_customer_id": "default"}): pass if raise_expected: @@ -98,9 +104,9 @@ def test_read_record_error_handling(mocker, config, customers, cls, raise_expect True, None, ( - "Metrics are not available for manager account 8765. Please remove metrics " - "fields in your custom query: SELECT campaign.accessible_bidding_strategy, " - "metrics.clicks FROM campaigns." + "Metrics are not available for manager account 8765. " + 'Skipping the custom query: "SELECT campaign.accessible_bidding_strategy, ' + 'metrics.clicks FROM campaigns" for manager account.' ), ), ( @@ -122,16 +128,16 @@ def test_read_record_error_handling(mocker, config, customers, cls, raise_expect "table_name": "unhappytable", }, False, - "Custom query should not contain segments.date", + None, None, ), ], ) def test_check_custom_queries(mocker, config, custom_query, is_manager_account, error_message, warning): - config["custom_queries"] = [custom_query] + config["custom_queries_array"] = [custom_query] mocker.patch( - "source_google_ads.source.SourceGoogleAds.get_account_info", - Mock(return_value=[[{"customer.manager": is_manager_account, "customer.time_zone": "Europe/Berlin", "customer.id": "8765"}]]), + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=is_manager_account, time_zone="Europe/Berlin", id="8765")]), ) mocker.patch("source_google_ads.google_ads.GoogleAdsClient", return_value=MockGoogleAdsClient) source = SourceGoogleAds() diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index ac27340254b7..3f66564846f4 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from datetime import date import pendulum @@ -168,7 +169,7 @@ def test_get_fields_metadata(mocker): response = google_ads_client.get_fields_metadata(fields) # Get the mock service to check the request query - mock_service = google_ads_client.client.get_service("GoogleAdsFieldService") + mock_service = google_ads_client.get_client().get_service("GoogleAdsFieldService") # Assert the constructed request query expected_query = """ diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py index b136cee2cb1c..8ddf8bd80fba 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from copy import deepcopy from unittest.mock import DEFAULT, MagicMock, Mock, call @@ -53,7 +54,7 @@ class MockGoogleAds(GoogleAds): def parse_single_result(self, schema, result): return result - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): if query == "query_parent": return mock_response_parent() else: @@ -63,7 +64,7 @@ def send_request(self, query: str, customer_id: str): def test_change_status_stream(config, customers): """ """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id} + stream_slice = {"customer_id": customer_id, "login_customer_id": "default"} google_api = MockGoogleAds(credentials=config["credentials"]) @@ -77,7 +78,7 @@ def test_change_status_stream(config, customers): ) assert len(result) == 4 assert stream.get_query.call_count == 1 - stream.get_query.assert_called_with({"customer_id": customer_id}) + stream.get_query.assert_called_with({"customer_id": customer_id, "login_customer_id": "default"}) def test_child_incremental_events_read(config, customers): @@ -88,7 +89,7 @@ def test_child_incremental_events_read(config, customers): It shouldn't read records on 2021-01-01, 2021-01-02 """ customer_id = next(iter(customers)).id - parent_stream_slice = {"customer_id": customer_id, "resource_type": "CAMPAIGN_CRITERION"} + parent_stream_slice = {"customer_id": customer_id, "resource_type": "CAMPAIGN_CRITERION", "login_customer_id": "default"} stream_state = {"change_status": {customer_id: {"change_status.last_change_date_time": "2023-08-16 13:20:01.003295"}}} google_api = MockGoogleAds(credentials=config["credentials"]) @@ -120,6 +121,7 @@ def test_child_incremental_events_read(config, customers): "3": "2023-06-13 12:36:03.772447", "4": "2023-06-13 12:36:04.772447", }, + "login_customer_id": "default", } ] @@ -220,7 +222,7 @@ class MockGoogleAdsLimit(GoogleAds): def parse_single_result(self, schema, result): return result - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): self.count += 1 if self.count == 1: return mock_response_1() @@ -254,7 +256,12 @@ def test_query_limit_hit(config, customers): This test simulates a scenario where the limit is hit and slice start_date is updated with latest record cursor """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2023-06-13 11:35:04.772447", "end_date": "2023-06-13 13:36:04.772447"} + stream_slice = { + "customer_id": customer_id, + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } google_api = MockGoogleAdsLimit(credentials=config["credentials"]) stream_config = dict( @@ -274,16 +281,37 @@ def test_query_limit_hit(config, customers): assert stream.get_query.call_count == 3 get_query_calls = [ - call({"customer_id": "123", "start_date": "2023-06-13 11:35:04.772447", "end_date": "2023-06-13 13:36:04.772447"}), - call({"customer_id": "123", "start_date": "2023-06-13 12:36:02.772447", "end_date": "2023-06-13 13:36:04.772447"}), - call({"customer_id": "123", "start_date": "2023-06-13 12:36:04.772447", "end_date": "2023-06-13 13:36:04.772447"}), + call( + { + "customer_id": "123", + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), + call( + { + "customer_id": "123", + "start_date": "2023-06-13 12:36:02.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), + call( + { + "customer_id": "123", + "start_date": "2023-06-13 12:36:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), ] get_query_mock.assert_has_calls(get_query_calls) class MockGoogleAdsLimitException(MockGoogleAdsLimit): - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): self.count += 1 if self.count == 1: return mock_response_1() @@ -301,7 +329,12 @@ def test_query_limit_hit_exception(config, customers): then error will be raised """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2023-06-13 11:35:04.772447", "end_date": "2023-06-13 13:36:04.772447"} + stream_slice = { + "customer_id": customer_id, + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } google_api = MockGoogleAdsLimitException(credentials=config["credentials"]) stream_config = dict( @@ -319,7 +352,7 @@ def test_query_limit_hit_exception(config, customers): ) ) - expected_message = "More then limit 2 records with same cursor field. Incremental sync is not possible for this stream." + expected_message = "More than limit 2 records with same cursor field. Incremental sync is not possible for this stream." assert e.value.message == expected_message @@ -341,6 +374,7 @@ def test_change_status_get_query(mocker, config, customers): "start_date": "2023-01-01 00:00:00.000000", "end_date": "2023-09-19 00:00:00.000000", "resource_type": "SOME_RESOURCE_TYPE", + "login_customer_id": "default", } # Call the get_query method with the stream_slice @@ -401,6 +435,7 @@ def test_incremental_events_stream_get_query(mocker, config, customers): "customers/1234567890/adGroupCriteria/111111111111~4": "2023-09-18 08:56:59.165599", "customers/1234567890/adGroupCriteria/111111111111~5": "2023-09-18 08:56:59.165599", }, + "login_customer_id": "default", } # Call the get_query method with the stream_slice @@ -430,6 +465,7 @@ def test_read_records_with_slice_splitting(mocker, config): "record_changed_time_map": {i: f"time_{i}" for i in range(15000)}, "customer_id": "sample_customer_id", "deleted_ids": set(), + "login_customer_id": "default", } # Create a mock instance of the CampaignCriterion stream @@ -454,12 +490,14 @@ def test_read_records_with_slice_splitting(mocker, config): "record_changed_time_map": {i: f"time_{i}" for i in range(10000)}, "customer_id": "sample_customer_id", "deleted_ids": set(), + "login_customer_id": "default", } expected_second_slice = { "updated_ids": set(range(10000, 15000)), "record_changed_time_map": {i: f"time_{i}" for i in range(10000, 15000)}, "customer_id": "sample_customer_id", "deleted_ids": set(), + "login_customer_id": "default", } # Verify the arguments passed to the parent's read_records method for both calls diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json new file mode 100644 index 000000000000..2ce005d03ec0 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json @@ -0,0 +1,18 @@ +{ + "credentials": { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "customer_id": "1234567890", + "start_date": "2023-09-04", + "conversion_window_days": 14, + "custom_queries": [ + { + "query": "SELECT campaign.name, metrics.clicks FROM campaign", + "primary_key": null, + "table_name": "test_query" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json new file mode 100644 index 000000000000..7d8097055f09 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "customer_id": "1234567890", + "start_date": "2023-09-04", + "conversion_window_days": 14, + "custom_queries_array": [] +} diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py index 9546e70b83af..7606a76bc7bf 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py @@ -2,20 +2,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from unittest.mock import Mock + import pytest +from pendulum.tz.timezone import Timezone from source_google_ads.models import CustomerModel -def test_time_zone(): - mock_account_info = [[{"customer.id": "8765"}]] +def test_time_zone(mocker): + mocker.patch("source_google_ads.models.local_timezone", Mock(return_value=Timezone("Europe/Riga"))) + + mock_account_info = [{"customer_client.id": "8765"}] customers = CustomerModel.from_accounts(mock_account_info) for customer in customers: - assert customer.time_zone == "local" + assert customer.time_zone.name == Timezone("Europe/Riga").name @pytest.mark.parametrize("is_manager_account", (True, False)) def test_manager_account(is_manager_account): - mock_account_info = [[{"customer.manager": is_manager_account, "customer.id": "8765"}]] + mock_account_info = [{"customer_client.manager": is_manager_account, "customer_client.id": "8765"}] customers = CustomerModel.from_accounts(mock_account_info) for customer in customers: assert customer.is_manager_account is is_manager_account diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index 22f4efc95994..6394817edd99 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import re from collections import namedtuple -from unittest.mock import Mock +from unittest.mock import Mock, call import pendulum import pytest @@ -13,6 +14,7 @@ from pendulum import today from source_google_ads.custom_query_stream import IncrementalCustomQuery from source_google_ads.google_ads import GoogleAds +from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds from source_google_ads.streams import AdGroupAdLegacy, chunk_date_range from source_google_ads.utils import GAQL @@ -21,10 +23,10 @@ @pytest.fixture -def mock_account_info(mocker): +def mock_get_customers(mocker): mocker.patch( - "source_google_ads.source.SourceGoogleAds.get_account_info", - Mock(return_value=[[{"customer.manager": False, "customer.time_zone": "Europe/Berlin", "customer.id": "8765"}]]), + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=False, time_zone="Europe/Berlin", id="8765")]), ) @@ -112,14 +114,15 @@ def test_chunk_date_range(): ] == slices -def test_streams_count(config, mock_account_info): +def test_streams_count(config, mock_get_customers): source = SourceGoogleAds() streams = source.streams(config) expected_streams_number = 30 + print(f"{config=} \n{streams=}") assert len(streams) == expected_streams_number -def test_read_missing_stream(config, mock_account_info): +def test_read_missing_stream(config, mock_get_customers): source = SourceGoogleAds() catalog = ConfiguredAirbyteCatalog( @@ -390,7 +393,7 @@ def test_check_connection_should_pass_when_config_valid(mocker): "customer_id": "fake_customer_id", "start_date": "2022-01-01", "conversion_window_days": 14, - "custom_queries": [ + "custom_queries_array": [ { "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", "primary_key": None, @@ -435,8 +438,84 @@ def test_stream_slices(config, customers): ) slices = list(stream.stream_slices()) assert slices == [ - {"start_date": "2020-12-18", "end_date": "2021-01-01", "customer_id": "123"}, - {"start_date": "2021-01-02", "end_date": "2021-01-16", "customer_id": "123"}, - {"start_date": "2021-01-17", "end_date": "2021-01-31", "customer_id": "123"}, - {"start_date": "2021-02-01", "end_date": "2021-02-10", "customer_id": "123"}, + {"start_date": "2020-12-18", "end_date": "2021-01-01", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-01-02", "end_date": "2021-01-16", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-01-17", "end_date": "2021-01-31", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-02-01", "end_date": "2021-02-10", "customer_id": "123", "login_customer_id": None}, ] + + +def mock_send_request(query: str, customer_id: str, login_customer_id: str = "default"): + print(query, customer_id, login_customer_id) + if customer_id == "123": + if "WHERE customer_client.status in ('active')" in query: + return [ + [ + {"customer_client.id": "123", "customer_client.status": "active"}, + ] + ] + else: + return [ + [ + {"customer_client.id": "123", "customer_client.status": "active"}, + {"customer_client.id": "456", "customer_client.status": "disabled"}, + ] + ] + else: + return [ + [ + {"customer_client.id": "789", "customer_client.status": "active"}, + ] + ] + + +@pytest.mark.parametrize( + "customer_status_filter, expected_ids, send_request_calls", + [ + ( + [], + ["123", "456", "789"], + [ + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client", + customer_id="123", + ), + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client", + customer_id="789", + ), + ], + ), # Empty filter, expect all customers + ( + ["active"], + ["123", "789"], + [ + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client WHERE customer_client.status in ('active')", + customer_id="123", + ), + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client WHERE customer_client.status in ('active')", + customer_id="789", + ), + ], + ), # Non-empty filter, expect filtered customers + ], +) +def test_get_customers(mocker, customer_status_filter, expected_ids, send_request_calls): + mock_google_api = Mock() + + mock_google_api.get_accessible_accounts.return_value = ["123", "789"] + mock_google_api.send_request.side_effect = mock_send_request + mock_google_api.parse_single_result.side_effect = lambda schema, result: result + + mock_config = {"customer_status_filter": customer_status_filter, "customer_ids": ["123", "456", "789"]} + + source = SourceGoogleAds() + + customers = source.get_customers(mock_google_api, mock_config) + + mock_google_api.send_request.assert_has_calls(send_request_calls) + + assert len(customers) == len(expected_ids) + assert {customer.id for customer in customers} == set(expected_ids) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index de8a42a4da28..a171b869d3d2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -2,18 +2,19 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from unittest.mock import Mock import pytest from airbyte_cdk.models import SyncMode from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure -from google.ads.googleads.v11.errors.types.request_error import RequestErrorEnum -from google.api_core.exceptions import DataLoss, InternalServerError, ResourceExhausted, TooManyRequests +from google.ads.googleads.v15.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure +from google.ads.googleads.v15.errors.types.request_error import RequestErrorEnum +from google.api_core.exceptions import DataLoss, InternalServerError, ResourceExhausted, TooManyRequests, Unauthenticated from grpc import RpcError from source_google_ads.google_ads import GoogleAds -from source_google_ads.streams import ClickView, Customer +from source_google_ads.streams import AdGroup, ClickView, Customer, CustomerLabel # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired. exception = GoogleAdsException( @@ -50,7 +51,7 @@ class MockGoogleAds(GoogleAds): def parse_single_result(self, schema, result): return result - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): self.count += 1 if self.count == 1: return mock_response_1() @@ -66,7 +67,7 @@ def test_page_token_expired_retry_succeeds(config, customers): It shouldn't read records on 2021-01-01, 2021-01-02 """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15", "login_customer_id": customer_id} google_api = MockGoogleAds(credentials=config["credentials"]) incremental_stream_config = dict( @@ -83,7 +84,9 @@ def test_page_token_expired_retry_succeeds(config, customers): result = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) assert len(result) == 9 assert stream.get_query.call_count == 2 - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15"}) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15", "login_customer_id": customer_id} + ) def mock_response_fails_1(): @@ -109,7 +112,7 @@ def mock_response_fails_2(): class MockGoogleAdsFails(MockGoogleAds): - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): self.count += 1 if self.count == 1: return mock_response_fails_1() @@ -123,7 +126,7 @@ def test_page_token_expired_retry_fails(config, customers): because Google Ads API doesn't allow filter by datetime. """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15", "login_customer_id": customer_id} google_api = MockGoogleAdsFails(credentials=config["credentials"]) incremental_stream_config = dict( @@ -144,7 +147,9 @@ def test_page_token_expired_retry_fails(config, customers): "Please contact the Airbyte team with the link of your connection for assistance." ) - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15"}) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15", "login_customer_id": customer_id} + ) assert stream.get_query.call_count == 2 @@ -160,7 +165,7 @@ def mock_response_fails_one_date(): class MockGoogleAdsFailsOneDate(MockGoogleAds): - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): return mock_response_fails_one_date() @@ -171,7 +176,7 @@ def test_page_token_expired_it_should_fail_date_range_1_day(config, customers): Minimum date range is 1 day. """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": customer_id} google_api = MockGoogleAdsFailsOneDate(credentials=config["credentials"]) incremental_stream_config = dict( @@ -191,17 +196,21 @@ def test_page_token_expired_it_should_fail_date_range_1_day(config, customers): "Page token has expired during processing response. " "Please contact the Airbyte team with the link of your connection for assistance." ) - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"}) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": customer_id} + ) assert stream.get_query.call_count == 1 @pytest.mark.parametrize("error_cls", (ResourceExhausted, TooManyRequests, InternalServerError, DataLoss)) def test_retry_transient_errors(mocker, config, customers, error_cls): + customer_id = next(iter(customers)).id + mocker.patch("time.sleep") credentials = config["credentials"] credentials.update(use_proto_plus=True) api = GoogleAds(credentials=credentials) - mocked_search = mocker.patch.object(api.ga_service, "search", side_effect=error_cls("Error message")) + mocked_search = mocker.patch.object(api.ga_services["default"], "search", side_effect=error_cls("Error message")) incremental_stream_config = dict( api=api, conversion_window_days=config["conversion_window_days"], @@ -210,8 +219,7 @@ def test_retry_transient_errors(mocker, config, customers, error_cls): customers=customers, ) stream = ClickView(**incremental_stream_config) - customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": "default"} records = [] with pytest.raises(error_cls) as exception: records = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) @@ -260,3 +268,33 @@ def test_parse_response(mocker, customers, config): ] assert output == expected_output + + +def test_read_records_unauthenticated(mocker, customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + + mocker.patch.object(api, "parse_single_result", side_effect=Unauthenticated(message="Unauthenticated")) + + stream_config = dict( + api=api, + customers=customers, + ) + stream = CustomerLabel(**stream_config) + with pytest.raises(AirbyteTracedException) as exc_info: + list(stream.read_records(SyncMode.full_refresh, {"customer_id": "customer_id", "login_customer_id": "default"})) + + assert exc_info.value.message == ( + "Authentication failed for the customer 'customer_id'. " "Please try to Re-authenticate your credentials on set up Google Ads page." + ) + + +def test_ad_group_stream_query_removes_metrics_field_for_manager(customers_manager, customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + stream_config = dict(api=api, customers=customers_manager, start_date="2020-01-01", conversion_window_days=10) + stream = AdGroup(**stream_config) + assert "metrics" not in stream.get_query(stream_slice={"customer_id": "123"}) + stream_config = dict(api=api, customers=customers, start_date="2020-01-01", conversion_window_days=10) + stream = AdGroup(**stream_config) + assert "metrics" in stream.get_query(stream_slice={"customer_id": "123"}) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py index 78be0956a1dc..7c42cd50360e 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from datetime import datetime from unittest.mock import Mock @@ -83,16 +84,16 @@ def test_parse_GAQL_ok(): @pytest.mark.parametrize( "config", [ - {"custom_queries": [{"query": "SELECT field1, field2 FROM x_Table2", "table_name": "test_table"}]}, - {"custom_queries": [{"query": "SELECT field1, field2 FROM x_Table WHERE ", "table_name": "test_table"}]}, - {"custom_queries": [{"query": "SELECT field1, , field2 FROM table", "table_name": "test_table"}]}, - {"custom_queries": [{"query": "SELECT fie ld1, field2 FROM table", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT field1, field2 FROM x_Table2", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT field1, field2 FROM x_Table WHERE ", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT field1, , field2 FROM table", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT fie ld1, field2 FROM table", "table_name": "test_table"}]}, ], ) def test_parse_GAQL_fail(config): with pytest.raises(AirbyteTracedException) as e: SourceGoogleAds._validate_and_transform(config) - expected_message = "The custom GAQL query test_table failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + expected_message = "The custom GAQL query test_table failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v15/query_validator" assert e.value.message == expected_message diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/README.md b/airbyte-integrations/connectors/source-google-analytics-data-api/README.md index 2ca089c74cd0..85ddc7e2f335 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/README.md +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/README.md @@ -1,118 +1,55 @@ -# Google Analytics Data Api Source +# Google-Analytics-Data-Api source connector -This is the repository for the Google Analytics Data Api source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-analytics-data-api). + +This is the repository for the Google-Analytics-Data-Api source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/google-analytics-data-api). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-analytics-data-api) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_data_api/spec.{yaml,json}` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_data_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-analytics-data-api test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-google-analytics-data-api spec +poetry run source-google-analytics-data-api check --config secrets/config.json +poetry run source-google-analytics-data-api discover --config secrets/config.json +poetry run source-google-analytics-data-api read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-google-analytics-data-api build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-analytics-data-api:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-google-analytics-data-api:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-google-analytics-data-api:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-google-analytics-data-api:dev . -# Running the spec command against your patched connector -docker run airbyte/source-google-analytics-data-api:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-google-analytics-data-api:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-data- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-analytics-data-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-google-analytics-data-api test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-analytics-data-api test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-data-api.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-data-api.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml index 56c4350f3bd8..4e07a4a4ce28 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml @@ -7,9 +7,8 @@ acceptance_tests: tests: - spec_path: "source_google_analytics_data_api/spec.json" backward_compatibility_tests_config: - # changed the structure of `custom_reports` - # from `json string` to `list[reports]` - disable_for_version: 1.5.1 + # changed the structure of `custom_reports` -> `cohortSpec` + disable_for_version: 2.1.0 connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py index ae4135b1396b..93839ed0e51a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi -from source_google_analytics_data_api.config_migrations import MigrateCustomReports, MigratePropertyID +from source_google_analytics_data_api.run import run if __name__ == "__main__": - source = SourceGoogleAnalyticsDataApi() - MigratePropertyID.migrate(sys.argv[1:], source) - MigrateCustomReports.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml index 23678df88e96..e2eeabd0d679 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml @@ -8,17 +8,21 @@ data: - www.googleapis.com - analyticsdata.googleapis.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a - dockerImageTag: 2.0.3 + dockerImageTag: 2.4.1 dockerRepository: airbyte/source-google-analytics-data-api documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api githubIssueLabel: source-google-analytics-data-api icon: google-analytics.svg license: Elv2 name: Google Analytics 4 (GA4) + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-analytics-data-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/poetry.lock b/airbyte-integrations/connectors/source-google-analytics-data-api/poetry.lock new file mode 100644 index 000000000000..fc68d9d8fe8f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/poetry.lock @@ -0,0 +1,1320 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.61.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.61.0.tar.gz", hash = "sha256:8beda008c5a177041ac02860a431ce7b1ecd00062a4a8f31fe6ac446cbed3e70"}, + {file = "airbyte_cdk-0.61.0-py3-none-any.whl", hash = "sha256:3f989bfe692c9519d61f9120ddb744ab82c432c2caf25374d4d6f5cdc374a1e9"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "37.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-37.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884"}, + {file = "cryptography-37.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6"}, + {file = "cryptography-37.0.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046"}, + {file = "cryptography-37.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5"}, + {file = "cryptography-37.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b"}, + {file = "cryptography-37.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8"}, + {file = "cryptography-37.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280"}, + {file = "cryptography-37.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3"}, + {file = "cryptography-37.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59"}, + {file = "cryptography-37.0.4-cp36-abi3-win32.whl", hash = "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157"}, + {file = "cryptography-37.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327"}, + {file = "cryptography-37.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b"}, + {file = "cryptography-37.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9"}, + {file = "cryptography-37.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67"}, + {file = "cryptography-37.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d"}, + {file = "cryptography-37.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282"}, + {file = "cryptography-37.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa"}, + {file = "cryptography-37.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441"}, + {file = "cryptography-37.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596"}, + {file = "cryptography-37.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a"}, + {file = "cryptography-37.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab"}, + {file = "cryptography-37.0.4.tar.gz", hash = "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.2.0" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyjwt" +version = "2.4.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyJWT-2.4.0-py3-none-any.whl", hash = "sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf"}, + {file = "PyJWT-2.4.0.tar.gz", hash = "sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba"}, +] + +[package.extras] +crypto = ["cryptography (>=3.3.1)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1)", "mypy", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "e30751a43a938ad9b4dc31ecc79afd469721b19b982cf9b1470b4a45ea707b8e" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/pyproject.toml b/airbyte-integrations/connectors/source-google-analytics-data-api/pyproject.toml new file mode 100644 index 000000000000..b3a3b51f287c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.4.1" +name = "source-google-analytics-data-api" +description = "Source implementation for Google Analytics Data Api." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/google-analytics-data-api" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_google_analytics_data_api" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +cryptography = "==37.0.4" +requests = "==2.31.0" +airbyte-cdk = "==0.61.0" +PyJWT = "==2.4.0" +pandas = "==2.2.0" + +[tool.poetry.scripts] +source-google-analytics-data-api = "source_google_analytics_data_api.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +requests-mock = "^1.11.0" +freezegun = "^1.4.0" +pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt b/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py deleted file mode 100644 index b11a793a8d5f..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT==2.4.0", "cryptography==37.0.4", "requests", "pandas"] - -TEST_REQUIREMENTS = [ - "freezegun", - "pytest~=6.1", - "pytest-mock~=3.6.1", - "requests-mock", -] - -setup( - name="source_google_analytics_data_api", - description="Source implementation for Google Analytics Data Api.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py index 23b9fcd5f4f4..621b5bbafcaf 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py @@ -6,6 +6,7 @@ import logging from typing import Any, List, Mapping +import dpath.util from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository @@ -33,9 +34,9 @@ class MigratePropertyID: @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: """ - This method determines whether config require migration. + This method determines whether config requires migration. Returns: - > True, if the transformation is neccessary + > True, if the transformation is necessary > False, otherwise. """ if cls.migrate_from_key in config: @@ -72,7 +73,7 @@ def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: """ This method checks the input args, should the config be migrated, - transform if neccessary and emit the CONTROL message. + transform if necessary and emit the CONTROL message. """ # get config path config_path = AirbyteEntrypoint(source).extract_config(args) @@ -104,7 +105,7 @@ class MigrateCustomReports: @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: """ - This method determines whether or not the config should be migrated to have the new structure for the `custom_reports`, + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, based on the source spec. Returns: > True, if the transformation is necessary @@ -126,7 +127,7 @@ def _should_migrate(cls, config: Mapping[str, Any]) -> bool: def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGoogleAnalyticsDataApi = None) -> Mapping[str, Any]: # assign old values to new property that will be used within the new version config[cls.migrate_to_key] = config[cls.migrate_from_key] - # transfom `json_str` to `list` of objects + # transform `json_str` to `list` of objects return source._validate_custom_reports(config) @classmethod @@ -150,7 +151,77 @@ def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: """ This method checks the input args, should the config be migrated, - transform if neccessary and emit the CONTROL message. + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) + + +class MigrateCustomReportsCohortSpec: + """ + This class stands for migrating the config at runtime, + Specifically, starting from `2.1.0`; the `cohortSpec` property will be added tp `custom_reports_array` with flag `enabled`: + > List([{name: my_report, "cohortSpec": { "enabled": "true" } }, ...]) + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `cohortSpec` inside `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + + return not dpath.util.search(config, "custom_reports_array/**/cohortSpec/enabled") + + @classmethod + def _transform_custom_reports_cohort_spec( + cls, + config: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Assign `enabled` property that will be used within the new version""" + for report in config.get("custom_reports_array", []): + if report.get("cohortSpec"): + report["cohortSpec"]["enabled"] = "true" + else: + report.setdefault("cohortSpec", {})["enabled"] = "false" + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGoogleAnalyticsDataApi, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_custom_reports_cohort_spec(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. """ # get config path config_path = AirbyteEntrypoint(source).extract_config(args) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py new file mode 100644 index 000000000000..ed4ec25e9250 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.config_migrations import MigrateCustomReports, MigrateCustomReportsCohortSpec, MigratePropertyID + + +def run(): + source = SourceGoogleAnalyticsDataApi() + MigratePropertyID.migrate(sys.argv[1:], source) + MigrateCustomReports.migrate(sys.argv[1:], source) + MigrateCustomReportsCohortSpec.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 41024fcdef5e..b84a91cfafc6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -22,7 +22,13 @@ from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError from source_google_analytics_data_api import utils -from source_google_analytics_data_api.utils import DATE_FORMAT, WRONG_DIMENSIONS, WRONG_JSON_SYNTAX, WRONG_METRICS +from source_google_analytics_data_api.utils import ( + DATE_FORMAT, + WRONG_CUSTOM_REPORT_CONFIG, + WRONG_DIMENSIONS, + WRONG_JSON_SYNTAX, + WRONG_METRICS, +) from .api_quota import GoogleAnalyticsApiQuota from .utils import ( @@ -37,8 +43,8 @@ transform_json, ) -# set the quota handler globaly since limitations are the same for all streams -# the initial values should be saved once and tracked for each stream, inclusivelly. +# set the quota handler globally since limitations are the same for all streams +# the initial values should be saved once and tracked for each stream, inclusively. GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() LOOKBACK_WINDOW = datetime.timedelta(days=2) @@ -157,6 +163,11 @@ def add_metrics(metrics, metric_types, row) -> dict: def _metric_type_to_python(metric_data: Tuple[str, str]) -> Any: metric_name, metric_value = metric_data python_type = metrics_type_to_python(metric_types[metric_name]) + + # Google Analytics sometimes returns float for integer metrics. + # So this is a workaround for this issue: https://github.com/airbytehq/oncall/issues/4130 + if python_type == int: + return metric_name, round(float(metric_value)) return metric_name, python_type(metric_value) return dict(map(_metric_type_to_python, zip(metrics, [v["value"] for v in row["metricValues"]]))) @@ -202,6 +213,12 @@ def get_json_schema(self) -> Mapping[str, Any]: } ) + # change the type of `conversions:*` metrics from int to float: https://github.com/airbytehq/oncall/issues/4130 + if self.config.get("convert_conversions_event", False): + for schema_field in schema["properties"]: + if schema_field.startswith("conversions:"): + schema["properties"][schema_field]["type"] = ["null", "float"] + return schema def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -240,6 +257,12 @@ def parse_response( metrics = [h.get("name") for h in r.get("metricHeaders", [{}])] metrics_type_map = {h.get("name"): h.get("type") for h in r.get("metricHeaders", [{}]) if "name" in h} + # change the type of `conversions:*` metrics from int to float: https://github.com/airbytehq/oncall/issues/4130 + if self.config.get("convert_conversions_event", False): + for schema_field in metrics_type_map: + if schema_field.startswith("conversions:"): + metrics_type_map[schema_field] = "TYPE_FLOAT" + for row in r.get("rows", []): record = { "property_id": self.config["property_id"], @@ -292,6 +315,7 @@ def request_body_json( "returnPropertyQuota": True, "offset": str(0), "limit": str(self.page_size), + "keepEmptyRows": self.config.get("keep_empty_rows", False), } dimension_filter = self.config.get("dimensionFilter") @@ -519,8 +543,14 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> report_stream = self.instantiate_report_class(report, False, _config, page_size=100) # check if custom_report dimensions + metrics can be combined and report generated - stream_slice = next(report_stream.stream_slices(sync_mode=SyncMode.full_refresh)) - next(report_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), None) + try: + stream_slice = next(report_stream.stream_slices(sync_mode=SyncMode.full_refresh)) + next(report_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), None) + except HTTPError as e: + error_response = "" + if e.response.status_code == HTTPStatus.BAD_REQUEST: + error_response = e.response.json().get("error", {}).get("message", "") + return False, WRONG_CUSTOM_REPORT_CONFIG.format(report=report["name"], error_response=error_response) return True, None @@ -545,7 +575,7 @@ def instantiate_report_streams(self, report: dict, config: Mapping[str, Any], ** def instantiate_report_class( report: dict, add_name_suffix: bool, config: Mapping[str, Any], **extra_kwargs ) -> GoogleAnalyticsDataApiBaseStream: - cohort_spec = report.get("cohortSpec") + cohort_spec = report.get("cohortSpec", {}) pivots = report.get("pivots") stream_config = { **config, @@ -558,7 +588,7 @@ def instantiate_report_class( if pivots: stream_config["pivots"] = pivots report_class_tuple = (PivotReport,) - if cohort_spec: + if cohort_spec.pop("enabled", "") == "true": stream_config["cohort_spec"] = cohort_spec report_class_tuple = (CohortReportMixin, *report_class_tuple) name = report["name"] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json index 1b3c6077bd4a..c2bdf1211498 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json @@ -2095,6 +2095,132 @@ "required": ["field_name", "filter"] } ] + }, + "cohortSpec": { + "title": "Cohort Reports", + "description": "Cohort reports creates a time series of user retention for the cohort.", + "type": "object", + "order": 5, + "oneOf": [ + { + "title": "Disabled", + "type": "object", + "properties": { + "enabled": { + "type": "string", + "const": "false" + } + } + }, + { + "title": "Enabled", + "type": "object", + "properties": { + "enabled": { + "type": "string", + "const": "true" + }, + "cohorts": { + "name": "Cohorts", + "order": 0, + "type": "array", + "always_show": true, + "items": { + "title": "Cohorts", + "type": "object", + "required": ["dimension", "dateRange"], + "properties": { + "name": { + "title": "Name", + "type": "string", + "always_show": true, + "pattern": "^(?!(cohort_|RESERVED_)).*$", + "description": "Assigns a name to this cohort. If not set, cohorts are named by their zero based index cohort_0, cohort_1, etc.", + "order": 0 + }, + "dimension": { + "title": "Dimension", + "description": "Dimension used by the cohort. Required and only supports `firstSessionDate`", + "type": "string", + "enum": ["firstSessionDate"], + "order": 1 + }, + "dateRange": { + "type": "object", + "required": ["startDate", "endDate"], + "properties": { + "startDate": { + "title": "Start Date", + "type": "string", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2021-01-01"], + "order": 2 + }, + "endDate": { + "title": "End Date", + "type": "string", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2021-01-01"], + "order": 3 + } + } + } + } + } + }, + "cohortsRange": { + "type": "object", + "order": 1, + "required": ["granularity", "endOffset"], + "properties": { + "granularity": { + "title": "Granularity", + "description": "The granularity used to interpret the startOffset and endOffset for the extended reporting date range for a cohort report.", + "type": "string", + "enum": [ + "GRANULARITY_UNSPECIFIED", + "DAILY", + "WEEKLY", + "MONTHLY" + ], + "order": 0 + }, + "startOffset": { + "title": "Start Offset", + "description": "Specifies the start date of the extended reporting date range for a cohort report.", + "type": "integer", + "minimum": 0, + "order": 1 + }, + "endOffset": { + "title": "End Offset", + "description": "Specifies the end date of the extended reporting date range for a cohort report.", + "type": "integer", + "minimum": 0, + "order": 2 + } + } + }, + "cohortReportSettings": { + "type": "object", + "title": "Cohort Report Settings", + "description": "Optional settings for a cohort report.", + "properties": { + "accumulate": { + "always_show": true, + "title": "Accumulate", + "description": "If true, accumulates the result from first touch day to the end day", + "type": "boolean" + } + } + } + } + } + ] } }, "required": ["name", "dimensions", "metrics"] @@ -2109,6 +2235,20 @@ "maximum": 364, "default": 1, "order": 5 + }, + "keep_empty_rows": { + "type": "boolean", + "title": "Keep Empty Rows", + "description": "If false, each row with all metrics equal to 0 will not be returned. If true, these rows will be returned if they are not separately removed by a filter. More information is available in the documentation.", + "default": false, + "order": 6 + }, + "convert_conversions_event": { + "type": "boolean", + "title": "Convert `conversions:*` Metrics to Float", + "description": "Enables conversion of `conversions:*` event metrics from integers to floats. This is beneficial for preventing data rounding when the API returns float values for any `conversions:*` fields.", + "default": false, + "order": 7 } } }, diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py index f40cdf08da64..5a77a16f9a32 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py @@ -71,6 +71,7 @@ WRONG_METRICS = "The custom report {report_name} entered contains invalid metrics: {fields}. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." WRONG_PIVOTS = "The custom report {report_name} entered contains invalid pivots: {fields}. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)." API_LIMIT_PER_HOUR = "Your API key has reached its limit for the hour. Wait until the quota refreshes in an hour to retry." +WRONG_CUSTOM_REPORT_CONFIG = "Please check configuration for custom report {report}. {error_response}" def datetime_to_secs(dt: datetime.datetime) -> int: diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py index 6abb31990bd2..fcd8e1b879be 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py @@ -49,6 +49,7 @@ def config(one_year_ago): "screenPageViewsPerSession", "bounceRate", ], + "keep_empty_rows": True, "custom_reports": json.dumps( [ { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json new file mode 100644 index 000000000000..245a01f07016 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json @@ -0,0 +1,59 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "" + }, + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_ids": "314186564", + "custom_reports_array": [ + { + "name": "cohort_report", + "dimensions": ["cohort", "cohortNthDay"], + "metrics": ["cohortActiveUsers"], + "cohortSpec": { + "cohorts": [ + { + "dimension": "firstSessionDate", + "dateRange": { + "startDate": "2023-04-24", + "endDate": "2023-04-24" + } + } + ], + "cohortsRange": { + "endOffset": 100, + "granularity": "DAILY" + }, + "cohortReportSettings": { + "accumulate": false + } + } + }, + { + "name": "pivot_report", + "dateRanges": [ + { + "startDate": "2020-09-01", + "endDate": "2020-09-15" + } + ], + "dimensions": ["browser", "country", "language"], + "metrics": ["sessions"], + "pivots": [ + { + "fieldNames": ["browser"], + "limit": 5 + }, + { + "fieldNames": ["country"], + "limit": 250 + }, + { + "fieldNames": ["language"], + "limit": 15 + } + ] + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py new file mode 100644 index 000000000000..de76bda4e8a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import os +from typing import Any, Mapping + +import dpath.util +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_analytics_data_api.config_migrations import MigrateCustomReportsCohortSpec +from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" +NEW_TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleAnalyticsDataApi() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def test_migrate_config(capsys): + migration_instance = MigrateCustomReportsCohortSpec() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][0]["cohortSpec"]["enabled"] == "true" + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][1]["cohortSpec"]["enabled"] == "false" + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + assert not MigrateCustomReportsCohortSpec._should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json new file mode 100644 index 000000000000..fd7ddcd7ce9f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json @@ -0,0 +1,63 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "" + }, + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_ids": "314186564", + "custom_reports_array": [ + { + "name": "cohort_report", + "dimensions": ["cohort", "cohortNthDay"], + "metrics": ["cohortActiveUsers"], + "cohortSpec": { + "cohorts": [ + { + "dimension": "firstSessionDate", + "dateRange": { + "startDate": "2023-04-24", + "endDate": "2023-04-24" + } + } + ], + "cohortsRange": { + "endOffset": 100, + "granularity": "DAILY" + }, + "cohortReportSettings": { + "accumulate": false + }, + "enable": "true" + } + }, + { + "name": "pivot_report", + "dateRanges": [ + { + "startDate": "2020-09-01", + "endDate": "2020-09-15" + } + ], + "dimensions": ["browser", "country", "language"], + "metrics": ["sessions"], + "pivots": [ + { + "fieldNames": ["browser"], + "limit": 5 + }, + { + "fieldNames": ["country"], + "limit": 250 + }, + { + "fieldNames": ["language"], + "limit": 15 + } + ], + "cohortSpec": { + "enabled": "false" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index 8d8460cacf50..631b2a1d8683 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -107,12 +107,39 @@ def test_check_failure(requests_mock, config_gen): @pytest.mark.parametrize( - "status_code", - [ - (403), - (401), - ], + ("status_code", "expected_message"), + ( + (403, "Please check configuration for custom report cohort_report. "), + (400, "Please check configuration for custom report cohort_report. Granularity in the cohortsRange is required."), + ), ) +def test_check_incorrect_custom_reports_config(requests_mock, config_gen, status_code, expected_message): + requests_mock.register_uri( + "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} + ) + requests_mock.register_uri( + "GET", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369/metadata", + json={ + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + ) + requests_mock.register_uri( + "POST", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369:runReport", + status_code=status_code, + json={"error": {"message": "Granularity in the cohortsRange is required."}}, + ) + config = {"custom_reports_array": '[{"name": "cohort_report", "dimensions": ["date"], "metrics": ["totalUsers"]}]'} + source = SourceGoogleAnalyticsDataApi() + logger = MagicMock() + status, message = source.check_connection(logger, config_gen(**config)) + assert status is False + assert message == expected_message + + +@pytest.mark.parametrize("status_code", (403, 401)) def test_missing_metadata(requests_mock, status_code): # required for MetadataDescriptor $instance input class TestConfig: diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 393492730b29..92b57153f5c8 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -80,6 +80,7 @@ def test_request_body_json(patch_base_class): {"name": "operatingSystem"}, {"name": "browser"}, ], + "keepEmptyRows": True, "dateRanges": [request_body_params["stream_slice"]], "returnPropertyQuota": True, "offset": str(0), @@ -163,8 +164,8 @@ def test_parse_response(patch_base_class): { "dimensionValues": [{"value": "20220731"}, {"value": "desktop"}, {"value": "Macintosh"}, {"value": "Chrome"}], "metricValues": [ - {"value": "344"}, - {"value": "169"}, + {"value": "344.234"}, # This is a float will be converted to int + {"value": "169.345345"}, # This is a float will be converted to int {"value": "420"}, {"value": "1.2209302325581395"}, {"value": "194.76313766428572"}, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/.dockerignore b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/.dockerignore new file mode 100644 index 000000000000..e3ebf60f6c58 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_google_analytics_v4_service_account_only +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/README.md b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/README.md new file mode 100644 index 000000000000..2c931f8d643f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/README.md @@ -0,0 +1,149 @@ +# Google Analytics V4 (Service Account Only) Source + +This is the repository for the Google Analytics V4 source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-analytics-v4). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-analytics-v4) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_v4/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-analytics-v4-service-account-only test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-google-analytics-v4-service-account-only build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-analytics-v4-service-account-only:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-google-analytics-v4-service-account-only:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-google-analytics-v4-service-account-only:dev . +# Running the spec command against your patched connector +docker run airbyte/source-google-analytics-v4-service-account-only:dev spec +``` +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-google-analytics-v4-service-account-only:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-v4-service-account-only:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-v4-service-account-only:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-analytics-v4-service-account-only:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-analytics-v4 test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-analytics-v4-service-account-only test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-v4-service-account-only.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/acceptance-test-config.yml new file mode 100644 index 000000000000..6a4357b097a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/acceptance-test-config.yml @@ -0,0 +1,35 @@ +connector_image: airbyte/source-google-analytics-v4-service-account-only:dev +test_strictness_level: high +acceptance_tests: + spec: + tests: + - spec_path: source_google_analytics_v4_service_account_only/spec.json + discovery: + tests: + - config_path: secrets/config.json + connection: + tests: + - config_path: secrets/config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: exception + basic_read: + tests: + - config_path: secrets/config.json + empty_streams: + - name: users_per_city + bypass_reason: no records in the stream + expect_records: + path: integration_tests/expected_records.jsonl + timeout_seconds: 1800 + full_refresh: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + incremental: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + timeout_seconds: 2400 + future_state: + future_state_path: integration_tests/abnormal_state.json diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/icon.svg b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/icon.svg new file mode 100644 index 000000000000..94dfa7142701 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/__init__.py new file mode 100644 index 000000000000..9db886e0930f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/__init__.py @@ -0,0 +1,23 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..6886688610f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/abnormal_state.json @@ -0,0 +1,86 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "website_overview" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "traffic_sources" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "pages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "locations" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "monthly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "four_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "two_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "daily_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "devices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "users_per_day" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "new_users_per_day" } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py new file mode 100644 index 000000000000..d49b55882333 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/catalog.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/catalog.json new file mode 100644 index 000000000000..f5cbf5205f5f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/catalog.json @@ -0,0 +1,123 @@ +{ + "streams": [ + { + "stream": { + "name": "website_overview", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "traffic_sources", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_source": { + "type": ["string"] + }, + "ga_medium": { + "type": ["string"] + }, + "ga_socialNetwork": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..71b19aca3eaa --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog.json @@ -0,0 +1,125 @@ +{ + "streams": [ + { + "stream": { + "name": "website_overview", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "traffic_sources", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "locations", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "monthly_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "four_weekly_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "two_weekly_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "weekly_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "daily_active_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "devices", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "new_users_per_day", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog_segment_filters.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog_segment_filters.json new file mode 100644 index 000000000000..2c34edf6530d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/configured_catalog_segment_filters.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "new_users_per_day", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["ga_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..a3ad03e595d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/expected_records.jsonl @@ -0,0 +1,22 @@ +{"stream": "website_overview", "data": {"ga_date": "2023-05-23", "ga_users": 1, "ga_newUsers": 1, "ga_sessions": 1, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 1, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023935006} +{"stream": "website_overview", "data": {"ga_date": "2023-05-24", "ga_users": 4, "ga_newUsers": 3, "ga_sessions": 4, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 4, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023935007} +{"stream": "traffic_sources", "data": {"ga_date": "2023-05-24", "ga_source": "(direct)", "ga_medium": "(none)", "ga_socialNetwork": "(not set)", "ga_users": 3, "ga_newUsers": 3, "ga_sessions": 3, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 3, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023943712} +{"stream": "traffic_sources", "data": {"ga_date": "2023-05-24", "ga_source": "api.surveymonkey.com", "ga_medium": "referral", "ga_socialNetwork": "(not set)", "ga_users": 1, "ga_newUsers": 0, "ga_sessions": 1, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 1, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023943713} +{"stream": "pages", "data": {"ga_date": "2023-05-24", "ga_hostname": "de.surveymonkey.com", "ga_pagePath": "/apps/NKI5TOTqk4tS5BZyJXU9YQ_3D_3D/details/", "ga_pageviews": 1, "ga_uniquePageviews": 1, "ga_avgTimeOnPage": 0.0, "ga_entrances": 1, "ga_entranceRate": 100.0, "ga_bounceRate": 100.0, "ga_exits": 1, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023951462} +{"stream": "pages", "data": {"ga_date": "2023-05-24", "ga_hostname": "www.surveymonkey.com", "ga_pagePath": "/apps/NKI5TOTqk4tS5BZyJXU9YQ_3D_3D/details/", "ga_pageviews": 3, "ga_uniquePageviews": 3, "ga_avgTimeOnPage": 0.0, "ga_entrances": 3, "ga_entranceRate": 100.0, "ga_bounceRate": 100.0, "ga_exits": 3, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023951463} +{"stream": "locations", "data": {"ga_date": "2023-05-24", "ga_continent": "Americas", "ga_subContinent": "Northern America", "ga_country": "United States", "ga_region": "New York", "ga_metro": "New York, NY", "ga_city": "New York", "ga_users": 1, "ga_newUsers": 1, "ga_sessions": 1, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 1, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023959587} +{"stream": "locations", "data": {"ga_date": "2023-05-24", "ga_continent": "Europe", "ga_subContinent": "Western Europe", "ga_country": "Germany", "ga_region": "Hessen", "ga_metro": "(not set)", "ga_city": "Frankfurt", "ga_users": 1, "ga_newUsers": 1, "ga_sessions": 1, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 1, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023959588} +{"stream": "monthly_active_users", "data": {"ga_date": "2023-05-24", "ga_30dayUsers": 32, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023967774} +{"stream": "monthly_active_users", "data": {"ga_date": "2023-05-25", "ga_30dayUsers": 32, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023968394} +{"stream": "four_weekly_active_users", "data": {"ga_date": "2023-05-24", "ga_28dayUsers": 30, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023975150} +{"stream": "four_weekly_active_users", "data": {"ga_date": "2023-05-25", "ga_28dayUsers": 28, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023976478} +{"stream": "two_weekly_active_users", "data": {"ga_date": "2023-05-24", "ga_14dayUsers": 17, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023983198} +{"stream": "two_weekly_active_users", "data": {"ga_date": "2023-05-25", "ga_14dayUsers": 16, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023983753} +{"stream": "weekly_active_users", "data": {"ga_date": "2023-05-24", "ga_7dayUsers": 10, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023990571} +{"stream": "weekly_active_users", "data": {"ga_date": "2023-05-25", "ga_7dayUsers": 10, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023991040} +{"stream": "daily_active_users", "data": {"ga_date": "2023-05-23", "ga_1dayUsers": 1, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023998149} +{"stream": "daily_active_users", "data": {"ga_date": "2023-05-24", "ga_1dayUsers": 4, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685023998151} +{"stream": "devices", "data": {"ga_date": "2023-05-24", "ga_deviceCategory": "desktop", "ga_operatingSystem": "Macintosh", "ga_browser": "Safari", "ga_users": 2, "ga_newUsers": 2, "ga_sessions": 2, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 2, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685024005565} +{"stream": "devices", "data": {"ga_date": "2023-05-24", "ga_deviceCategory": "desktop", "ga_operatingSystem": "Windows", "ga_browser": "Chrome", "ga_users": 1, "ga_newUsers": 0, "ga_sessions": 1, "ga_sessionsPerUser": 1.0, "ga_avgSessionDuration": 0.0, "ga_pageviews": 1, "ga_pageviewsPerSession": 1.0, "ga_avgTimeOnPage": 0.0, "ga_bounceRate": 100.0, "ga_exitRate": 100.0, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685024005566} +{"stream": "new_users_per_day", "data": {"ga_date": "2023-05-24", "ga_country": "Nigeria", "ga_region": "Lagos", "ga_newUsers": 1, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685024012689} +{"stream": "new_users_per_day", "data": {"ga_date": "2023-05-24", "ga_country": "United States", "ga_region": "New York", "ga_newUsers": 1, "view_id": "211669975", "isDataGolden": true}, "emitted_at": 1685024012690} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/invalid_config.json new file mode 100644 index 000000000000..0ab7ad4763b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/invalid_config.json @@ -0,0 +1,10 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "None" + }, + "view_id": "211669975", + "start_date": "2021-02-11", + "window_in_days": 1, + "custom_reports": "[{\"name\": \"users_per_day\", \"dimensions\": [\"ga:date\"], \"metrics\": [\"ga:users\", \"ga:newUsers\"]}, {\"name\": \"sessions_per_country_day\", \"dimensions\": [\"ga:date\", \"ga:country\"], \"metrics\": [\"ga:sessions\", \"ga:sessionsPerUser\", \"ga:avgSessionDuration\"]}]" +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_config.json new file mode 100644 index 000000000000..831183f5c793 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "view_id": "1234567", + "start_date": "2021-01-01", + "window_in_days": 1, + "custom_reports": "custom_reports" +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_state.json new file mode 100644 index 000000000000..0c8625660e07 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/sample_state.json @@ -0,0 +1,86 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "website_overview" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "traffic_sources" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "pages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "locations" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "monthly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "four_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "two_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "daily_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "devices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "users_per_day" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "new_users_per_day" } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py new file mode 100644 index 000000000000..b91a0b49b694 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_analytics_v4_service_account_only import SourceGoogleAnalyticsV4ServiceAccountOnly + +if __name__ == "__main__": + source = SourceGoogleAnalyticsV4ServiceAccountOnly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml new file mode 100644 index 000000000000..79cbba174479 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml @@ -0,0 +1,32 @@ +data: + ab_internal: + ql: 400 + sl: 100 + allowedHosts: + hosts: + - oauth2.googleapis.com + - www.googleapis.com + - analyticsdata.googleapis.com + - analyticsreporting.googleapis.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 9e28a926-8f3c-4911-982d-a2e1c378b59c + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-google-analytics-v4-service-account-only + documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4-service-account-only + githubIssueLabel: source-google-analytics-v4-service-account-only + icon: google-analytics.svg + license: Elv2 + name: Google Analytics (Universal Analytics) + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + supportLevel: community + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bing-ads/requirements.txt b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-bing-ads/requirements.txt rename to airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/requirements.txt diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py new file mode 100644 index 000000000000..1c9b47c2ec4e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path + +from setuptools import find_packages, setup + + +def local_dependency(name: str) -> str: + """Returns a path to a local package.""" + return f"{name} @ file://{Path.cwd().parent / name}" + + +MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT", "cryptography", "requests", local_dependency("source-google-analytics-v4")] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "requests-mock", + "pytest-mock", + "freezegun", +] + +setup( + name="source_google_analytics_v4_service_account_only", + description="Source implementation for Google Analytics V4.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/__init__.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/__init__.py new file mode 100644 index 000000000000..d3028f55635d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/__init__.py @@ -0,0 +1,28 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from .source import SourceGoogleAnalyticsV4ServiceAccountOnly + +__all__ = ["SourceGoogleAnalyticsV4ServiceAccountOnly"] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/source.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/source.py new file mode 100644 index 000000000000..af0201aac566 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/source.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import source_google_analytics_v4 + + +class SourceGoogleAnalyticsV4ServiceAccountOnly(source_google_analytics_v4.SourceGoogleAnalyticsV4): + """Updating of default source logic + This connector shouldn't work with OAuth authentication method. + The base logic of this connector is implemented in the "source-source_google_analytics_v4" connector. + """ diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/spec.json new file mode 100644 index 000000000000..a4be0d1c2ea8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/source_google_analytics_v4_service_account_only/spec.json @@ -0,0 +1,79 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-v4-service-account-only", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Google Analytics (V4) Spec", + "type": "object", + "required": ["view_id", "start_date"], + "additionalProperties": true, + "properties": { + "credentials": { + "order": 0, + "type": "object", + "title": "Credentials", + "description": "Credentials for the service", + "oneOf": [ + { + "type": "object", + "title": "Service Account Key Authentication", + "required": ["credentials_json"], + "properties": { + "auth_type": { + "type": "string", + "const": "Service", + "order": 0 + }, + "credentials_json": { + "title": "Service Account JSON Key", + "type": "string", + "description": "The JSON key of the service account to use for authorization", + "examples": [ + "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" + ], + "airbyte_secret": true + } + } + } + ] + }, + "start_date": { + "order": 1, + "type": "string", + "title": "Replication Start Date", + "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated.", + "examples": ["2020-06-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^$|[\\s\\S]+$", + "format": "date" + }, + "view_id": { + "order": 2, + "type": "string", + "title": "View ID", + "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the Google Analytics Account Explorer." + }, + "end_date": { + "order": 3, + "type": "string", + "title": "Replication End Date", + "description": "The date in the format YYYY-MM-DD. Any data after this date will not be replicated.", + "examples": ["2020-06-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^$|[\\s\\S]+$", + "format": "date" + }, + "custom_reports": { + "order": 4, + "type": "string", + "title": "Custom Reports", + "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." + }, + "window_in_days": { + "type": "integer", + "title": "Data request time increment in days", + "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. ", + "examples": [30, 60, 90, 120, 200, 364], + "default": 1, + "order": 5 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/README.md b/airbyte-integrations/connectors/source-google-analytics-v4/README.md index 4530bcf8a465..4a399dcb19d5 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/README.md +++ b/airbyte-integrations/connectors/source-google-analytics-v4/README.md @@ -1,118 +1,55 @@ -# Google Analytics V4 Source +# Google-Analytics-V4 source connector -This is the repository for the Google Analytics V4 source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-analytics-v4). + +This is the repository for the Google-Analytics-V4 source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/google-analytics-v4). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-analytics-v4) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_v4/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/google-analytics-v4) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_v4/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-analytics-v4 test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-google-analytics-v4 spec +poetry run source-google-analytics-v4 check --config secrets/config.json +poetry run source-google-analytics-v4 discover --config secrets/config.json +poetry run source-google-analytics-v4 read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-google-analytics-v4 build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-analytics-v4:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-google-analytics-v4:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-google-analytics-v4:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-google-analytics-v4:dev . -# Running the spec command against your patched connector -docker run airbyte/source-google-analytics-v4:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-google-analytics-v4:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-v4:de docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-analytics-v4:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-google-analytics-v4 test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-analytics-v4 test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-v4.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-v4.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/main.py b/airbyte-integrations/connectors/source-google-analytics-v4/main.py index 45b902bd6ced..3fd58bc1d5f6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_analytics_v4 import SourceGoogleAnalyticsV4 +from source_google_analytics_v4.run import run if __name__ == "__main__": - source = SourceGoogleAnalyticsV4() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml index e294b1b63b6f..5f36a33ce227 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml @@ -13,13 +13,17 @@ data: connectorSubtype: api connectorType: source definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-google-analytics-v4 documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4 githubIssueLabel: source-google-analytics-v4 icon: google-analytics.svg license: Elv2 name: Google Analytics (Universal Analytics) + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-analytics-v4 registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/poetry.lock b/airbyte-integrations/connectors/source-google-analytics-v4/poetry.lock new file mode 100644 index 000000000000..e5efccf12550 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/poetry.lock @@ -0,0 +1,1185 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.59.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.59.2.tar.gz", hash = "sha256:cd8a2b679ddd01ac1db9d42e4326c4b4e815dcaf7e1654cbe327cfce7654f07b"}, + {file = "airbyte_cdk-0.59.2-py3-none-any.whl", hash = "sha256:90f9144f1519e0c66e260b68be94bb9b8f87130276353073c416df2e62ce7c7e"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "41.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "71852fc65c462f57b22c7c55c2df234d3c27e121649a777a65dd74fd75c9d91b" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml b/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml new file mode 100644 index 000000000000..b0e774e072a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.2.5" +name = "source-google-analytics-v4" +description = "Source implementation for Google Analytics V4." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/google-analytics-v4" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_google_analytics_v4" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.59.2" +PyJWT = "==2.8.0" +requests = "==2.31.0" +cryptography = "==41.0.4" + +[tool.poetry.scripts] +source-google-analytics-v4 = "source_google_analytics_v4.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +requests-mock = "^1.11.0" +pytest-mock = "^3.12.0" +freezegun = "^1.4.0" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt b/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py deleted file mode 100644 index c37ee40da749..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT", "cryptography", "requests"] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "requests-mock", - "pytest-mock", - "freezegun", -] - -setup( - name="source_google_analytics_v4", - description="Source implementation for Google Analytics V4.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py new file mode 100644 index 000000000000..ebb414319fab --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_analytics_v4 import SourceGoogleAnalyticsV4 + + +def run(): + source = SourceGoogleAnalyticsV4() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index d607904310ce..3d81036fa5d1 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -102,6 +102,7 @@ class GoogleAnalyticsV4Stream(HttpStream, ABC): def __init__(self, config: MutableMapping): super().__init__(authenticator=config["authenticator"]) self.start_date = config["start_date"] + self.end_date = config.get("end_date") self.window_in_days: int = config.get("window_in_days", 1) self.view_id = config["view_id"] self.metrics = config["metrics"] @@ -255,7 +256,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs: Any) - ...] """ - end_date = pendulum.now().date() + end_date = (pendulum.parse(self.end_date) if self.end_date else pendulum.now()).date() start_date = pendulum.parse(self.start_date).date() if stream_state: prev_end_date = pendulum.parse(stream_state.get(self.cursor_field)).date() diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index 33fefa97a7ca..c4a8078bfbd0 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -91,9 +91,18 @@ "title": "View ID", "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the Google Analytics Account Explorer." }, - "custom_reports": { + "end_date": { "order": 3, "type": "string", + "title": "Replication End Date", + "description": "The date in the format YYYY-MM-DD. Any data after this date will not be replicated.", + "examples": ["2020-06-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^$|[\\s\\S]+$", + "format": "date" + }, + "custom_reports": { + "order": 4, + "type": "string", "title": "Custom Reports", "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." }, @@ -103,7 +112,7 @@ "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. ", "examples": [30, 60, 90, 120, 200, 364], "default": 1, - "order": 4 + "order": 5 } } }, diff --git a/airbyte-integrations/connectors/source-google-directory/main.py b/airbyte-integrations/connectors/source-google-directory/main.py index 97076817e3a7..fa60e31af90e 100644 --- a/airbyte-integrations/connectors/source-google-directory/main.py +++ b/airbyte-integrations/connectors/source-google-directory/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_directory import SourceGoogleDirectory +from source_google_directory.run import run if __name__ == "__main__": - source = SourceGoogleDirectory() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-directory/metadata.yaml b/airbyte-integrations/connectors/source-google-directory/metadata.yaml index 9db4b72e67e8..eedf71a553af 100644 --- a/airbyte-integrations/connectors/source-google-directory/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-directory/metadata.yaml @@ -8,6 +8,10 @@ data: icon: googledirectory.svg license: MIT name: Google Directory + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-directory registries: cloud: dockerImageTag: 0.2.1 diff --git a/airbyte-integrations/connectors/source-google-directory/setup.py b/airbyte-integrations/connectors/source-google-directory/setup.py index a4dbf5967f65..ac1950ff3ea1 100644 --- a/airbyte-integrations/connectors/source-google-directory/setup.py +++ b/airbyte-integrations/connectors/source-google-directory/setup.py @@ -20,13 +20,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-directory=source_google_directory.run:run", + ], + }, name="source_google_directory", description="Source implementation for Google Directory.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py new file mode 100644 index 000000000000..d7110346906b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_directory import SourceGoogleDirectory + + +def run(): + source = SourceGoogleDirectory() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl index e3df40fb7e17..005c1cb48aa7 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl @@ -2,8 +2,8 @@ {"stream": "test", "data": {"x": 9999, "_ab_source_file_last_modified": "2023-10-16T06:16:06.000000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} {"stream": "test", "data": {"y": 9999, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} {"stream": "test", "data": {"y": 123, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_docx.docx", "_ab_source_file_last_modified": "2023-10-27T00:45:54.000000Z", "_ab_source_file_url": "testdoc_docx.docx"}, "emitted_at": 1698400261867} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:45:58.000000Z", "_ab_source_file_url": "testdoc_pdf.pdf"}, "emitted_at": 1698400264556} -{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_ocr_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:46:04.000000Z", "_ab_source_file_url": "testdoc_ocr_pdf.pdf"}, "emitted_at": 1698400267184} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_google", "_ab_source_file_last_modified": "2023-11-10T13:46:18.551000Z", "_ab_source_file_url": "testdoc_google"}, "emitted_at": 1698400261074} -{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_presentation", "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z", "_ab_source_file_url": "testdoc_presentation"}, "emitted_at": 1698402779268} \ No newline at end of file +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_docx.docx", "_ab_source_file_last_modified": "2023-10-27T00:45:54.000000Z", "_ab_source_file_url": "testdoc_docx.docx", "_ab_source_file_parse_error": null}, "emitted_at": 1698400261867} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:45:58.000000Z", "_ab_source_file_url": "testdoc_pdf.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698400264556} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_ocr_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:46:04.000000Z", "_ab_source_file_url": "testdoc_ocr_pdf.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698400267184} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_google", "_ab_source_file_last_modified": "2023-11-10T13:46:18.551000Z", "_ab_source_file_url": "testdoc_google", "_ab_source_file_parse_error": null}, "emitted_at": 1698400261074} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_presentation", "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z", "_ab_source_file_url": "testdoc_presentation", "_ab_source_file_parse_error": null}, "emitted_at": 1698402779268} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json index 0acb2776ca40..e1341d1bbe27 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json @@ -52,7 +52,7 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", "type": "string", "airbyte_hidden": true }, @@ -275,12 +275,46 @@ "const": "unstructured", "type": "string" }, - "skip_unprocessable_file_types": { + "skip_unprocessable_files": { "type": "boolean", "default": true, - "title": "Skip Unprocessable File Types", - "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] } }, "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", diff --git a/airbyte-integrations/connectors/source-google-drive/main.py b/airbyte-integrations/connectors/source-google-drive/main.py index 4d051be2ff54..606e4f7641e8 100644 --- a/airbyte-integrations/connectors/source-google-drive/main.py +++ b/airbyte-integrations/connectors/source-google-drive/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk import AirbyteEntrypoint -from airbyte_cdk.entrypoint import launch -from source_google_drive import SourceGoogleDrive +from source_google_drive.run import run if __name__ == "__main__": - args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(args) - source = SourceGoogleDrive(catalog_path) - launch(source, args) + run() diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index e652b04b3994..c1352111e894 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,12 +7,16 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.4 + dockerImageTag: 0.0.9 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg license: ELv2 name: Google Drive + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-drive registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index cd6d4c7c95fc..20f6da8ae909 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.55.5", + "airbyte-cdk[file-based]>=0.61.0", "google-api-python-client==2.104.0", "google-auth-httplib2==0.1.1", "google-auth-oauthlib==1.1.0", @@ -25,8 +25,25 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-google-drive=source_google_drive.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py new file mode 100644 index 000000000000..d9d56d62740b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_google_drive import SourceGoogleDrive + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + source = SourceGoogleDrive( + SourceGoogleDrive.read_catalog(catalog_path) if catalog_path else None, + SourceGoogleDrive.read_config(config_path) if config_path else None, + SourceGoogleDrive.read_state(state_path) if state_path else None, + ) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py index fe49fba7fe8c..479711be83ee 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py @@ -1,21 +1,24 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any +from typing import Any, Mapping, Optional -from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.models import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.source import TState from source_google_drive.spec import SourceGoogleDriveSpec from source_google_drive.stream_reader import SourceGoogleDriveStreamReader class SourceGoogleDrive(FileBasedSource): - def __init__(self, catalog_path: str): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState]): super().__init__( stream_reader=SourceGoogleDriveStreamReader(), spec_class=SourceGoogleDriveSpec, - catalog_path=catalog_path, + catalog=catalog, + config=config, + state=state, cursor_cls=DefaultFileBasedCursor, ) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py index 00a360e0640b..4bc354dbf4d5 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -67,11 +67,6 @@ class Config: def documentation_url(cls) -> str: return "https://docs.airbyte.com/integrations/sources/google-drive" - @staticmethod - def remove_discriminator(schema: dict) -> None: - """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" - dpath.util.delete(schema, "properties/*/discriminator") - @classmethod def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: """ @@ -79,10 +74,12 @@ def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: """ schema = super().schema(*args, **kwargs) - cls.remove_discriminator(schema) - # Remove legacy settings dpath.util.delete(schema, "properties/streams/items/properties/legacy_prefix") dpath.util.delete(schema, "properties/streams/items/properties/format/oneOf/*/properties/inference_type") + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + return schema diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py index 8265a27b1edb..956a0e47d3cb 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_pagespeed_insights import SourceGooglePagespeedInsights +from source_google_pagespeed_insights.run import run if __name__ == "__main__": - source = SourceGooglePagespeedInsights() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml index 68f96b0db8e7..2bfe25c5eeae 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml @@ -8,6 +8,10 @@ data: icon: google-pagespeed-insights.svg license: MIT name: Google PageSpeed Insights + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-pagespeed-insights registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py index 07d5d4a738ff..e1c998d1d69d 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-pagespeed-insights=source_google_pagespeed_insights.run:run", + ], + }, name="source_google_pagespeed_insights", description="Source implementation for Google Pagespeed Insights.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py new file mode 100644 index 000000000000..e4bb2c99def7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_pagespeed_insights import SourceGooglePagespeedInsights + + +def run(): + source = SourceGooglePagespeedInsights() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-search-console/README.md b/airbyte-integrations/connectors/source-google-search-console/README.md index 3fc8aacd2bd3..0ff6251cecd7 100755 --- a/airbyte-integrations/connectors/source-google-search-console/README.md +++ b/airbyte-integrations/connectors/source-google-search-console/README.md @@ -1,118 +1,55 @@ -# Google Search Console Source +# Google-Search-Console source connector -This is the repository for the Google Search Console source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-search-console). + +This is the repository for the Google-Search-Console source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/google-search-console). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-search-console) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_search_console/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/google-search-console) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_search_console/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-search-console test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-google-search-console spec +poetry run source-google-search-console check --config secrets/config.json +poetry run source-google-search-console discover --config secrets/config.json +poetry run source-google-search-console read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-google-search-console build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-search-console:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-google-search-console:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-google-search-console:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-google-search-console:dev . -# Running the spec command against your patched connector -docker run airbyte/source-google-search-console:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-google-search-console:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-search-console: docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-search-console:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-google-search-console test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-search-console test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-search-console.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/google-search-console.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py index 1174b079d6b3..4e39115533b4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py @@ -20,7 +20,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-search-console/main.py b/airbyte-integrations/connectors/source-google-search-console/main.py index 117df652ca76..845383457bb7 100755 --- a/airbyte-integrations/connectors/source-google-search-console/main.py +++ b/airbyte-integrations/connectors/source-google-search-console/main.py @@ -2,16 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_search_console import SourceGoogleSearchConsole -from source_google_search_console.config_migrations import MigrateCustomReports +from source_google_search_console.run import run if __name__ == "__main__": - source = SourceGoogleSearchConsole() - # migrate config at runtime - MigrateCustomReports.migrate(sys.argv[1:], source) - # run the connector - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml index e3d321483fd5..5328b530325e 100644 --- a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml @@ -10,13 +10,18 @@ data: connectorSubtype: api connectorType: source definitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 - dockerImageTag: 1.3.6 + dockerImageTag: 1.3.7 dockerRepository: airbyte/source-google-search-console documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console githubIssueLabel: source-google-search-console icon: googlesearchconsole.svg license: Elv2 name: Google Search Console + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-google-search-console registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-search-console/poetry.lock b/airbyte-integrations/connectors/source-google-search-console/poetry.lock new file mode 100644 index 000000000000..88c8ca379b0f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/poetry.lock @@ -0,0 +1,1290 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.52.4" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.52.4.tar.gz", hash = "sha256:d2d5d2c3a988259ed3e270b4d77ea7d6c0ca1a9f57aec8ae54ff64b99ad9b2e8"}, + {file = "airbyte_cdk-0.52.4-py3-none-any.whl", hash = "sha256:94219a67d125e80924a81bb809be90b045359159904c4905c38f68f69c8fd723"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx] (==0.10.19)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx] (==0.10.19)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-api-core" +version = "2.17.0" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.17.0.tar.gz", hash = "sha256:de7ef0450faec7c75e0aea313f29ac870fdc44cfaec9d6499a9a17305980ef66"}, + {file = "google_api_core-2.17.0-py3-none-any.whl", hash = "sha256:08ed79ed8e93e329de5e3e7452746b734e6bf8438d8d64dd3319d21d3164890c"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-api-python-client" +version = "2.105.0" +description = "Google API Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-python-client-2.105.0.tar.gz", hash = "sha256:0a8b32cfc2d9b3c1868ae6faef7ee1ab9c89a6cec30be709ea9c97f9a3e5902d"}, + {file = "google_api_python_client-2.105.0-py2.py3-none-any.whl", hash = "sha256:571ce7c41e53415e385aab5a955725f71780550683ffcb71596f5809677d40b7"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.19.0,<3.0.0.dev0" +google-auth-httplib2 = ">=0.1.0" +httplib2 = ">=0.15.0,<1.dev0" +uritemplate = ">=3.0.1,<5" + +[[package]] +name = "google-auth" +version = "2.23.3" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.23.3.tar.gz", hash = "sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3"}, + {file = "google_auth-2.23.3-py2.py3-none-any.whl", hash = "sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +description = "Google Authentication Library: httplib2 transport" +optional = false +python-versions = "*" +files = [ + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, +] + +[package.dependencies] +google-auth = "*" +httplib2 = ">=0.19.0" + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "httplib2" +version = "0.22.0" +description = "A comprehensive HTTP client library." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, + {file = "httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, +] + +[package.dependencies] +pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyparsing" +version = "3.1.1" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, + {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-lazy-fixture" +version = "0.6.3" +description = "It helps to use fixtures in pytest.mark.parametrize" +optional = false +python-versions = "*" +files = [ + {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"}, + {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"}, +] + +[package.dependencies] +pytest = ">=3.2.5" + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "uritemplate" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" +optional = false +python-versions = ">=3.6" +files = [ + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "5b3037c68d5c9a557b84ace020c5f76a13cd928610dfb0fbc2434d344695180b" diff --git a/airbyte-integrations/connectors/source-google-search-console/pyproject.toml b/airbyte-integrations/connectors/source-google-search-console/pyproject.toml new file mode 100644 index 000000000000..d4c8989d1930 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.3.7" +name = "source-google-search-console" +description = "Source implementation for Google Search Console." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/google-search-console" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_google_search_console" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +google-api-python-client = "==2.105.0" +airbyte-cdk = "==0.52.4" +google-auth = "==2.23.3" + +[tool.poetry.scripts] +source-google-search-console = "source_google_search_console.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-lazy-fixture = "^0.6.3" +pytest = "^6.1" +pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-google-search-console/requirements.txt b/airbyte-integrations/connectors/source-google-search-console/requirements.txt deleted file mode 100755 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-google-search-console/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-google-search-console/setup.py b/airbyte-integrations/connectors/source-google-search-console/setup.py deleted file mode 100755 index fd73d6450d37..000000000000 --- a/airbyte-integrations/connectors/source-google-search-console/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "google-api-python-client", - "google-auth", -] - -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "pytest-lazy-fixture", - "requests-mock", -] - -setup( - name="source_google_search_console", - description="Source implementation for Google Search Console.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py new file mode 100755 index 000000000000..3de91fb3cc50 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_search_console import SourceGoogleSearchConsole +from source_google_search_console.config_migrations import MigrateCustomReports + + +def run(): + source = SourceGoogleSearchConsole() + # migrate config at runtime + MigrateCustomReports.migrate(sys.argv[1:], source) + # run the connector + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-sheets/README.md b/airbyte-integrations/connectors/source-google-sheets/README.md index 313109d6eff3..5ee60ccc3888 100644 --- a/airbyte-integrations/connectors/source-google-sheets/README.md +++ b/airbyte-integrations/connectors/source-google-sheets/README.md @@ -1,67 +1,91 @@ -# Pypi Source +# Google-Sheets source connector -This is the repository for the Pypi configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pypi). + +This is the repository for the Google-Sheets source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/google-sheets). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pypi) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pypi/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pypi test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name source-pypi build +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/google-sheets) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-google-sheets spec +poetry run source-google-sheets check --config secrets/config.json +poetry run source-google-sheets discover --config secrets/config.json +poetry run source-google-sheets read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -An image will be built with the tag `airbyte/source-pypi:dev`. +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -**Via `docker build`:** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-pypi:dev . +airbyte-ci connectors --name=source-google-sheets build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-google-sheets:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-pypi:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pypi:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-google-sheets:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-sheets:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-sheets:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-sheets:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-google-sheets test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-sheets test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-sheets.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/google-sheets.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index 63c87ce3137f..d0e00aadd280 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -6,6 +6,9 @@ acceptance_tests: - config_path: secrets/service_config.json expect_records: path: integration_tests/expected_records.txt + file_types: + skip_test: true + bypass_reason: "The source only supports Google Sheets" connection: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-google-sheets/main.py b/airbyte-integrations/connectors/source-google-sheets/main.py index 4aaa9a106d9b..806ac60fbefe 100644 --- a/airbyte-integrations/connectors/source-google-sheets/main.py +++ b/airbyte-integrations/connectors/source-google-sheets/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_sheets import SourceGoogleSheets +from source_google_sheets.run import run if __name__ == "__main__": - source = SourceGoogleSheets() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml index eaf059bbbcf9..cc4c1f2a0388 100644 --- a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: file connectorType: source definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 - dockerImageTag: 0.3.11 + dockerImageTag: 0.3.16 dockerRepository: airbyte/source-google-sheets documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets githubIssueLabel: source-google-sheets icon: google-sheets.svg license: Elv2 name: Google Sheets + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-sheets registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-sheets/poetry.lock b/airbyte-integrations/connectors/source-google-sheets/poetry.lock new file mode 100644 index 000000000000..099dc9c3eed6 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/poetry.lock @@ -0,0 +1,1285 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.8.tar.gz", hash = "sha256:0f327408ea5d9e913dcd8601ba937489270366f23a2323b13a27bfd49360b371"}, + {file = "airbyte_cdk-0.51.8-py3-none-any.whl", hash = "sha256:ac841fbf20fcadd7b5d7ff4f0872dd70e56c951d63ab8ad02175f756ab0fb541"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.0" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.9.2,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.0" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.0-py3-none-any.whl", hash = "sha256:e6a31fcd237504198a678d02c0040a8798f281c39203da61a5abce67842c5360"}, + {file = "airbyte_protocol_models-0.4.0.tar.gz", hash = "sha256:518736015c29ac60b6b8964a1b0d9b52e40020bcbd89e2545cc781f0b37d0f2b"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-api-core" +version = "2.16.2" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.16.2.tar.gz", hash = "sha256:032d37b45d1d6bdaf68fb11ff621e2593263a239fa9246e2e94325f9c47876d2"}, + {file = "google_api_core-2.16.2-py3-none-any.whl", hash = "sha256:449ca0e3f14c179b4165b664256066c7861610f70b6ffe54bb01a04e9b466929"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-api-python-client" +version = "2.114.0" +description = "Google API Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-python-client-2.114.0.tar.gz", hash = "sha256:e041bbbf60e682261281e9d64b4660035f04db1cccba19d1d68eebc24d1465ed"}, + {file = "google_api_python_client-2.114.0-py2.py3-none-any.whl", hash = "sha256:690e0bb67d70ff6dea4e8a5d3738639c105a478ac35da153d3b2a384064e9e1a"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.19.0,<3.0.0.dev0" +google-auth-httplib2 = ">=0.1.0" +httplib2 = ">=0.15.0,<1.dev0" +uritemplate = ">=3.0.1,<5" + +[[package]] +name = "google-auth" +version = "2.27.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +description = "Google Authentication Library: httplib2 transport" +optional = false +python-versions = "*" +files = [ + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, +] + +[package.dependencies] +google-auth = "*" +httplib2 = ">=0.19.0" + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "httplib2" +version = "0.22.0" +description = "A comprehensive HTTP client library." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, + {file = "httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, +] + +[package.dependencies] +pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pydantic" +version = "1.9.2" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyparsing" +version = "3.1.1" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, + {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, +] + +[[package]] +name = "unidecode" +version = "1.3.8" +description = "ASCII transliterations of Unicode text" +optional = false +python-versions = ">=3.5" +files = [ + {file = "Unidecode-1.3.8-py3-none-any.whl", hash = "sha256:d130a61ce6696f8148a3bd8fe779c99adeb4b870584eeb9526584e9aa091fd39"}, + {file = "Unidecode-1.3.8.tar.gz", hash = "sha256:cfdb349d46ed3873ece4586b96aa75258726e2fa8ec21d6f00a591d98806c2f4"}, +] + +[[package]] +name = "uritemplate" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" +optional = false +python-versions = ">=3.6" +files = [ + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "07b6fe5e724aeac85999fbabc5f09f42f26e4037ec9abd1849dcd56883263b5a" diff --git a/airbyte-integrations/connectors/source-google-sheets/pyproject.toml b/airbyte-integrations/connectors/source-google-sheets/pyproject.toml new file mode 100644 index 000000000000..b3e27e7ba6dd --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.16" +name = "source-google-sheets" +description = "Source implementation for Google Sheets." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/google-sheets" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +packages = [ { include = "source_google_sheets"}] + +[tool.poetry.dependencies] +python = "^3.9" +requests = "==2.31.0" +backoff = "==2.2.1" +google-auth-httplib2 = "==0.2.0" +Unidecode = "==1.3.8" +pydantic = "==1.9.2" +airbyte-cdk = "==0.51.8" +google-api-python-client = "==2.114.0" +PyYAML = "==6.0.1" + +[tool.poetry.scripts] +source-google-sheets = "source_google_sheets.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-google-sheets/requirements.txt b/airbyte-integrations/connectors/source-google-sheets/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py deleted file mode 100644 index 1dd377a9357e..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "backoff", - "requests", - "google-auth-httplib2", - "google-api-python-client", - "PyYAML~=6.0", - "pydantic~=1.9.2", - "Unidecode", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", -] - -setup( - name="source_google_sheets", - description="Source implementation for Google Sheets.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py index d8f367baddb6..1d74091561e0 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py @@ -52,7 +52,7 @@ def headers_to_airbyte_stream(logger: AirbyteLogger, sheet_name: str, header_row """ fields, duplicate_fields = Helpers.get_valid_headers_and_duplicates(header_row_values) if duplicate_fields: - logger.warn(f"Duplicate headers found in {sheet_name}. Ignoring them :{duplicate_fields}") + logger.warn(f"Duplicate headers found in {sheet_name}. Ignoring them: {duplicate_fields}") sheet_json_schema = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -85,8 +85,8 @@ def get_valid_headers_and_duplicates(header_row_values: List[str]) -> (List[str] @staticmethod def get_formatted_row_values(row_data: RowData) -> List[str]: """ - Gets the formatted values of all cell data in this row. A formatted value is the final value a user sees in a spreadsheet. It can be a raw - string input by the user, or the result of a sheets function call. + Gets the formatted values of all cell data in this row. A formatted value is the final value a user sees in a spreadsheet. + It can be a raw string input by the user, or the result of a sheets function call. """ return [value.formattedValue for value in row_data.values] @@ -151,6 +151,9 @@ def get_available_sheets_to_column_index_to_name( first_row = Helpers.get_first_row(client, spreadsheet_id, sheet) if names_conversion: first_row = [safe_name_conversion(h) for h in first_row] + # When performing names conversion, they won't match what is listed in catalog for the majority of cases, + # so they should be cast here in order to have them in records + columns = {safe_name_conversion(c) for c in columns} # Find the column index of each header value idx = 0 for cell_value in first_row: diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py new file mode 100644 index 000000000000..a34dfe611d01 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceGoogleSheets + + +def run(): + source = SourceGoogleSheets() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py index 689d9856a8d7..0e2168c4aff2 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py @@ -12,7 +12,7 @@ DEFAULT_SEPARATOR = "_" -def name_conversion(text): +def name_conversion(text: str) -> str: """ convert name using a set of rules, for example: '1MyName' -> '_1_my_name' """ @@ -36,7 +36,9 @@ def name_conversion(text): return text -def safe_name_conversion(text): +def safe_name_conversion(text: str) -> str: + if not text: + return text new = name_conversion(text) if not new: raise Exception(f"initial string '{text}' converted to empty") diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index 3743f3e46513..1a6b15be8a22 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -65,7 +65,7 @@ def test_headers_to_airbyte_stream(self): actual_stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_values) self.assertEqual(expected_stream, actual_stream) - def test_duplicate_headers_retrived(self): + def test_duplicate_headers_retrieved(self): header_values = ["h1", "h1", "h3"] expected_valid_header_values = ["h3"] @@ -266,10 +266,36 @@ def mock_client_call(spreadsheetId, includeGridData, ranges=None): with patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes: None): sheet_client = GoogleSheetsClient({"fake": "credentials"}, ["auth_scopes"]) sheet_client.client = client + + expected = {sheet1: {0: "1", 1: "2", 2: "3", 3: "4"}} + + # names_conversion = False actual = Helpers.get_available_sheets_to_column_index_to_name( - sheet_client, spreadsheet_id, {sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])} + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + ) + self.assertEqual(expected, actual) + + # names_conversion = False, with null header cell + sheet1_first_row = ["1", "2", "3", "4", None] + expected = {sheet1: {0: "1", 1: "2", 2: "3", 3: "4", 4: None}} + actual = Helpers.get_available_sheets_to_column_index_to_name( + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + ) + self.assertEqual(expected, actual) + + # names_conversion = True, with null header cell + sheet1_first_row = ["AB", "Some Header", "Header", "4", "1MyName", None] + expected = {sheet1: {0: "ab", 1: "some_header", 2: "header", 3: "_4", 4: "_1_my_name", 5: None}} + actual = Helpers.get_available_sheets_to_column_index_to_name( + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + names_conversion=True, ) - expected = {sheet1: {0: "1", 1: "2", 2: "3", 3: "4"}} self.assertEqual(expected, actual) diff --git a/airbyte-integrations/connectors/source-google-webfonts/main.py b/airbyte-integrations/connectors/source-google-webfonts/main.py index 99b5b45a2ba9..70cd774d0a01 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/main.py +++ b/airbyte-integrations/connectors/source-google-webfonts/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_webfonts import SourceGoogleWebfonts +from source_google_webfonts.run import run if __name__ == "__main__": - source = SourceGoogleWebfonts() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml index 58d9d4bd572f..b60374d751b9 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml @@ -8,6 +8,10 @@ data: icon: googleworkpace.svg license: MIT name: Google Webfonts + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-webfonts registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-webfonts/setup.py b/airbyte-integrations/connectors/source-google-webfonts/setup.py index 863003482ff1..1b50e26184ce 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/setup.py +++ b/airbyte-integrations/connectors/source-google-webfonts/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-webfonts=source_google_webfonts.run:run", + ], + }, name="source_google_webfonts", description="Source implementation for Google Webfonts.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py new file mode 100644 index 000000000000..fbce9176984d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_webfonts import SourceGoogleWebfonts + + +def run(): + source = SourceGoogleWebfonts() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py index 010d9559e95b..1e88c29a963d 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_workspace_admin_reports import SourceGoogleWorkspaceAdminReports +from source_google_workspace_admin_reports.run import run if __name__ == "__main__": - source = SourceGoogleWorkspaceAdminReports() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml index a0b5a4ca2007..bfc8f0f6421e 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml @@ -8,6 +8,11 @@ data: icon: googleworkpace.svg license: MIT name: Google Workspace Admin Reports + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-google-workspace-admin-reports registries: cloud: dockerImageTag: 0.1.4 diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py index 7170103bf496..9d85298d1b15 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py @@ -21,13 +21,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-workspace-admin-reports=source_google_workspace_admin_reports.run:run", + ], + }, name="source_google_workspace_admin_reports", description="Source implementation for Google Workspace Admin Reports.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py new file mode 100644 index 000000000000..b5ecb1fabd63 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_workspace_admin_reports import SourceGoogleWorkspaceAdminReports + + +def run(): + source = SourceGoogleWorkspaceAdminReports() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-greenhouse/.coveragerc b/airbyte-integrations/connectors/source-greenhouse/.coveragerc new file mode 100644 index 000000000000..aceb412b78cd --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_greenhouse/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-greenhouse/README.md b/airbyte-integrations/connectors/source-greenhouse/README.md index 7c78083f0569..5061a5ae3f3f 100644 --- a/airbyte-integrations/connectors/source-greenhouse/README.md +++ b/airbyte-integrations/connectors/source-greenhouse/README.md @@ -1,116 +1,55 @@ -# Firebolt Source +# Greenhouse source connector -This is the repository for the Firebolt source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/greenhouse). + +This is the repository for the Greenhouse source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/greenhouse). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/greenhouse) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_greenhouse/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/greenhouse) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_greenhouse/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source greenhouse test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-greenhouse spec +poetry run source-greenhouse check --config secrets/config.json +poetry run source-greenhouse discover --config secrets/config.json +poetry run source-greenhouse read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: - -```bash -airbyte-ci connectors --name source-greenhouse build +### Running unit tests +To run unit tests locally, from the connector directory run: ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-greenhouse:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +poetry run pytest unit_tests ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-greenhouse build +``` -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-greenhouse:latest +An image will be available on your host with the tag `airbyte/source-greenhouse:dev`. -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-greenhouse:dev . -# Running the spec command against your patched connector -docker run airbyte/source-greenhouse:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-greenhouse:dev spec @@ -119,28 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-greenhouse:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-greenhouse:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-greenhouse test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-greenhouse test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/greenhouse.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/greenhouse.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-greenhouse/main.py b/airbyte-integrations/connectors/source-greenhouse/main.py index 55e53e344062..e08a14b429fd 100644 --- a/airbyte-integrations/connectors/source-greenhouse/main.py +++ b/airbyte-integrations/connectors/source-greenhouse/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_greenhouse import SourceGreenhouse +from source_greenhouse.run import run if __name__ == "__main__": - source = SourceGreenhouse() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml index e732c0f18fa6..341baccb65ea 100644 --- a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 - dockerImageTag: 0.4.4 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-greenhouse documentationUrl: https://docs.airbyte.com/integrations/sources/greenhouse githubIssueLabel: source-greenhouse icon: greenhouse.svg license: MIT name: Greenhouse + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-greenhouse registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-greenhouse/poetry.lock b/airbyte-integrations/connectors/source-greenhouse/poetry.lock new file mode 100644 index 000000000000..d00402201707 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/poetry.lock @@ -0,0 +1,1053 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "dataclasses-jsonschema" +version = "2.15.1" +description = "JSON schema generation from dataclasses" +optional = false +python-versions = "*" +files = [ + {file = "dataclasses-jsonschema-2.15.1.tar.gz", hash = "sha256:e3726a76b3d24b6c1f2198982be9278c14fdec84b8652294037558403b0aa5bb"}, + {file = "dataclasses_jsonschema-2.15.1-py3-none-any.whl", hash = "sha256:baa7c5414fb24e103ed131263e95622e3842909481d59ca42a4e639d12faa017"}, +] + +[package.dependencies] +jsonschema = "*" +python-dateutil = "*" + +[package.extras] +apispec = ["apispec"] +fast-validation = ["fastjsonschema"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "ad41b784b46cb6a10f61c59bcdea7023d6326bd7d735a29e673ac23cf4ef9e0e" diff --git a/airbyte-integrations/connectors/source-greenhouse/pyproject.toml b/airbyte-integrations/connectors/source-greenhouse/pyproject.toml new file mode 100644 index 000000000000..2da2664aef98 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.5.0" +name = "source-greenhouse" +description = "Source implementation for Greenhouse." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/greenhouse" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_greenhouse" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.63.2" +dataclasses-jsonschema = "==2.15.1" + +[tool.poetry.scripts] +source-greenhouse = "source_greenhouse.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.6" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-greenhouse/requirements.txt b/airbyte-integrations/connectors/source-greenhouse/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py deleted file mode 100644 index 73945258c42b..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", -] - -setup( - name="source_greenhouse", - description="Source implementation for Greenhouse.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=["airbyte-cdk>=0.44.1", "dataclasses-jsonschema==2.15.1"], - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py new file mode 100644 index 000000000000..d82109b4b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_greenhouse import SourceGreenhouse + + +def run(): + source = SourceGreenhouse() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py index a7a9adaf7202..3adfc4cb9efc 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py @@ -138,7 +138,7 @@ def test_parse_response_expected_response(applications_stream): ] """ response._content = response_content - parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}, records_schema={}) records = [dict(record) for record in parsed_response] assert records == json.loads(response_content) @@ -148,7 +148,7 @@ def test_parse_response_empty_content(applications_stream): response = requests.Response() response.status_code = 200 response._content = b"[]" - parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}, records_schema={}) records = [record for record in parsed_response] assert records == [] @@ -164,7 +164,7 @@ def test_ignore_403(applications_stream): response = requests.Response() response.status_code = 403 response._content = b"" - parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}, records_schema={}) records = [record for record in parsed_response] assert records == [] diff --git a/airbyte-integrations/connectors/source-gridly/main.py b/airbyte-integrations/connectors/source-gridly/main.py index 1e999fa0c2d0..307be6500faf 100644 --- a/airbyte-integrations/connectors/source-gridly/main.py +++ b/airbyte-integrations/connectors/source-gridly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gridly import SourceGridly +from source_gridly.run import run if __name__ == "__main__": - source = SourceGridly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gridly/metadata.yaml b/airbyte-integrations/connectors/source-gridly/metadata.yaml index 1c6747986c8c..11a5a90d789f 100644 --- a/airbyte-integrations/connectors/source-gridly/metadata.yaml +++ b/airbyte-integrations/connectors/source-gridly/metadata.yaml @@ -8,6 +8,10 @@ data: icon: gridly.svg license: MIT name: Gridly + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gridly registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gridly/setup.py b/airbyte-integrations/connectors/source-gridly/setup.py index dc08caa562c5..2b6e0bb2cc20 100644 --- a/airbyte-integrations/connectors/source-gridly/setup.py +++ b/airbyte-integrations/connectors/source-gridly/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gridly=source_gridly.run:run", + ], + }, name="source_gridly", description="Source implementation for Gridly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/run.py b/airbyte-integrations/connectors/source-gridly/source_gridly/run.py new file mode 100644 index 000000000000..2cddadbd477d --- /dev/null +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gridly import SourceGridly + + +def run(): + source = SourceGridly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gutendex/main.py b/airbyte-integrations/connectors/source-gutendex/main.py index 0fd65db37ce0..8304e254daa4 100644 --- a/airbyte-integrations/connectors/source-gutendex/main.py +++ b/airbyte-integrations/connectors/source-gutendex/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gutendex import SourceGutendex +from source_gutendex.run import run if __name__ == "__main__": - source = SourceGutendex() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gutendex/metadata.yaml b/airbyte-integrations/connectors/source-gutendex/metadata.yaml index 01e6a71bbaed..6624a7e221bd 100644 --- a/airbyte-integrations/connectors/source-gutendex/metadata.yaml +++ b/airbyte-integrations/connectors/source-gutendex/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-gutendex license: MIT name: Gutendex + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gutendex registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-gutendex/setup.py b/airbyte-integrations/connectors/source-gutendex/setup.py index b96103135cdf..6759689d84b5 100644 --- a/airbyte-integrations/connectors/source-gutendex/setup.py +++ b/airbyte-integrations/connectors/source-gutendex/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gutendex=source_gutendex.run:run", + ], + }, name="source_gutendex", description="Source implementation for Gutendex.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py new file mode 100644 index 000000000000..ba4bcb6755ac --- /dev/null +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gutendex import SourceGutendex + + +def run(): + source = SourceGutendex() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py index b323465b96c8..a33c09315382 100644 --- a/airbyte-integrations/connectors/source-harness/main.py +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_harness import SourceHarness +from source_harness.run import run if __name__ == "__main__": - source = SourceHarness() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-harness/metadata.yaml b/airbyte-integrations/connectors/source-harness/metadata.yaml index 857504fc2505..59b15a6a1b61 100644 --- a/airbyte-integrations/connectors/source-harness/metadata.yaml +++ b/airbyte-integrations/connectors/source-harness/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.harness.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-harness registries: oss: enabled: false @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/harness tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-harness/setup.py b/airbyte-integrations/connectors/source-harness/setup.py index 6bef3ce1447c..170c960c9500 100644 --- a/airbyte-integrations/connectors/source-harness/setup.py +++ b/airbyte-integrations/connectors/source-harness/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-harness=source_harness.run:run", + ], + }, name="source_harness", description="Source implementation for Harness.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-harness/source_harness/run.py b/airbyte-integrations/connectors/source-harness/source_harness/run.py new file mode 100644 index 000000000000..544daa9407a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harness import SourceHarness + + +def run(): + source = SourceHarness() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harvest/.coveragerc b/airbyte-integrations/connectors/source-harvest/.coveragerc new file mode 100644 index 000000000000..7abb63521095 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_harvest/run.py diff --git a/airbyte-integrations/connectors/source-harvest/README.md b/airbyte-integrations/connectors/source-harvest/README.md index dbf16de24033..a4e109d1f20c 100644 --- a/airbyte-integrations/connectors/source-harvest/README.md +++ b/airbyte-integrations/connectors/source-harvest/README.md @@ -1,116 +1,55 @@ -# Harvest Source +# Harvest source connector + This is the repository for the Harvest source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/harvest). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/harvest). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/harvest) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harvest/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/harvest) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harvest/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source harvest test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-harvest spec +poetry run source-harvest check --config secrets/config.json +poetry run source-harvest discover --config secrets/config.json +poetry run source-harvest read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-harvest build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-harvest:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-harvest:latest -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-harvest:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-harvest:dev . -# Running the spec command against your patched connector -docker run airbyte/source-harvest:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-harvest:dev spec @@ -119,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harvest:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-harvest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-harvest test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-harvest test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/harvest.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/harvest.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl index 4b23bf6a6cea..131e7fe6b2eb 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl @@ -6,8 +6,8 @@ {"stream": "company", "data": {"base_uri": "https://airbyte.harvestapp.com", "full_domain": "airbyte.harvestapp.com", "name": "Airbyte", "is_active": true, "week_start_day": "Monday", "wants_timestamp_timers": false, "time_format": "hours_minutes", "date_format": "%m/%d/%Y", "plan_type": "simple-v4", "expense_feature": true, "invoice_feature": true, "estimate_feature": true, "team_feature": true, "weekly_capacity": 144000, "approval_feature": true, "clock": "12h", "currency": "USD", "currency_code_display": "iso_code_none", "currency_symbol_display": "symbol_before", "decimal_symbol": ".", "thousands_separator": ",", "color_scheme": "orange"}, "emitted_at": 1690884271497} {"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "payment_options": [], "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1690884271995} {"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "payment_options": [], "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1690884271995} -{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273321} -{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273322} +{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": false, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "include_link_to_client_invoice": false, "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1708017738014} +{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": false, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "include_link_to_client_invoice": false, "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1708017738015} {"stream": "invoice_payments", "data": {"id": 21857618, "amount": 500.0, "paid_at": "2021-05-26T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "", "transaction_id": null, "created_at": "2021-05-26T09:07:06Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": "2021-05-26", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174545}, "emitted_at": 1690884275279} {"stream": "invoice_payments", "data": {"id": 21857615, "amount": 76.9, "paid_at": "2021-05-25T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "Payed", "transaction_id": null, "created_at": "2021-05-26T09:06:37Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174531}, "emitted_at": 1690884276439} {"stream": "invoice_item_categories", "data": {"id": 2732435, "name": "Product", "use_as_service": false, "use_as_expense": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884276919} diff --git a/airbyte-integrations/connectors/source-harvest/main.py b/airbyte-integrations/connectors/source-harvest/main.py index 3fa9904a1ea6..e00a49b587fd 100644 --- a/airbyte-integrations/connectors/source-harvest/main.py +++ b/airbyte-integrations/connectors/source-harvest/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_harvest import SourceHarvest +from source_harvest.run import run if __name__ == "__main__": - source = SourceHarvest() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-harvest/metadata.yaml b/airbyte-integrations/connectors/source-harvest/metadata.yaml index 0bb7f453aea5..842b3f385b13 100644 --- a/airbyte-integrations/connectors/source-harvest/metadata.yaml +++ b/airbyte-integrations/connectors/source-harvest/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 - dockerImageTag: 0.1.21 + dockerImageTag: 0.1.23 dockerRepository: airbyte/source-harvest documentationUrl: https://docs.airbyte.com/integrations/sources/harvest githubIssueLabel: source-harvest icon: harvest.svg license: MIT name: Harvest + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-harvest registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-harvest/poetry.lock b/airbyte-integrations/connectors/source-harvest/poetry.lock new file mode 100644 index 000000000000..1ab272398503 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "c286c47fbc557061975b5d48e9b98b439952f4a2678139b5584efa12f9735165" diff --git a/airbyte-integrations/connectors/source-harvest/pyproject.toml b/airbyte-integrations/connectors/source-harvest/pyproject.toml new file mode 100644 index 000000000000..e44d0c488796 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.22" +name = "source-harvest" +description = "Source implementation for Harvest." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/harvest" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_harvest" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = ">=0.62.1" + +[tool.poetry.scripts] +source-harvest = "source_harvest.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-harvest/requirements.txt b/airbyte-integrations/connectors/source-harvest/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-harvest/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py deleted file mode 100644 index a208606a02a8..000000000000 --- a/airbyte-integrations/connectors/source-harvest/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] - -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "requests-mock", -] - -setup( - name="source_harvest", - description="Source implementation for Harvest.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/run.py b/airbyte-integrations/connectors/source-harvest/source_harvest/run.py new file mode 100644 index 000000000000..53406b411f86 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harvest import SourceHarvest + + +def run(): + source = SourceHarvest() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index 3e7d1b2e5617..a91007344077 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -75,7 +75,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalHarvestStream(HarvestStream, ABC): cursor_field = "updated_at" - def __init__(self, replication_start_date: pendulum.datetime = None, **kwargs): + def __init__(self, replication_start_date: Optional[pendulum.DateTime] = None, **kwargs) -> None: super().__init__(**kwargs) self._replication_start_date = replication_start_date @@ -96,7 +96,12 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - replication_start_date = stream_state.get(self.cursor_field) or self._replication_start_date + + replication_start_date = None + if stream_state.get(self.cursor_field): + replication_start_date = stream_state.get(self.cursor_field) + elif self._replication_start_date: + replication_start_date = self._replication_start_date.format("YYYY-MM-DDTHH:mm:ssZ") params.update({"updated_since": replication_start_date}) return params diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py new file mode 100644 index 000000000000..a4f55f7898bf --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "account_id": "an account id", + "replication_start_date": "2021-01-01T00:00:00Z", + "credentials": { + "api_token": "an api key" + } + } + + def with_account_id(self, account_id: str) -> "ConfigBuilder": + self._config["account_id"] = account_id + return self + + def with_replication_start_date(self, replication_start_date: datetime) -> "ConfigBuilder": + self._config["start_date"] = replication_start_date.isoformat()[:-13]+"Z" + return self + + def with_api_token(self, api_token: str) -> "ConfigBuilder": + self._config["credentials"]["api_token"] = api_token + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py new file mode 100644 index 000000000000..093c80610432 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py @@ -0,0 +1,103 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from source_harvest import SourceHarvest + +_A_REPLICATION_START_DATE = "2021-01-01T00:00:00+00:00" +_AN_ACCOUNT_ID = "1209384" +_AN_API_KEY = "harvestapikey" +_AN_INVOICE_ID = "an-invoice-id" +_STREAM_NAME = "invoice_messages" +_TEMPLATE_NAME = "invoice_messages" +_INVOICES_TEMPLATE_NAME = "invoices" +_RECORDS_PATH = FieldPath("invoice_messages") +_INVOICES_RECORDS_PATH = FieldPath("invoices") + + +def _a_message() -> RecordBuilder: + return create_record_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _invoice_messages_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _an_invoice() -> RecordBuilder: + return create_record_builder( + find_template(_INVOICES_TEMPLATE_NAME, __file__), + _INVOICES_RECORDS_PATH, + record_id_path=FieldPath("id"), + ) + + +def _invoices_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_INVOICES_TEMPLATE_NAME, __file__), + _INVOICES_RECORDS_PATH, + ) + + +def _read( + config_builder: ConfigBuilder, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + return read( + SourceHarvest(), + config_builder.build(), + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), + state, + expecting_exception + ) + + +class InvoicesTest(TestCase): + @HttpMocker() + def test_given_replication_start_date_when_read_then_request_is_created_properly(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://api.harvestapp.com/v2/invoices", + query_params={ + "per_page": "50", + }, + ), + _invoices_response().with_record(_an_invoice().with_id(_AN_INVOICE_ID)).build() + ) + http_mocker.get( + HttpRequest( + url=f"https://api.harvestapp.com/v2/invoices/{_AN_INVOICE_ID}/messages", + query_params={ + "per_page": "50", + "updated_since": _A_REPLICATION_START_DATE, + }, + ), + _invoices_response().with_record(_a_message()).build() + ) + + _read(ConfigBuilder().with_account_id(_AN_ACCOUNT_ID).with_api_token(_AN_API_KEY).with_replication_start_date(datetime.fromisoformat(_A_REPLICATION_START_DATE))) + + # endpoint is called diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py new file mode 100644 index 000000000000..835490a917d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from source_harvest import SourceHarvest + +_A_REPLICATION_START_DATE = "2021-01-01T00:00:00+00:00" +_AN_ACCOUNT_ID = "1209384" +_AN_API_KEY = "harvestapikey" +_STREAM_NAME = "invoices" +_TEMPLATE_NAME = "invoices" +_RECORDS_PATH = FieldPath("invoices") + + +def _an_invoice() -> RecordBuilder: + return create_record_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _invoices_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _read( + config_builder: ConfigBuilder, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + return read( + SourceHarvest(), + config_builder.build(), + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), + state, + expecting_exception + ) + + +class InvoicesTest(TestCase): + @HttpMocker() + def test_given_replication_start_date_when_read_then_request_is_created_properly(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://api.harvestapp.com/v2/invoices", + query_params={ + "per_page": "50", + "updated_since": _A_REPLICATION_START_DATE, + }, + headers={ + "Authorization": f"Bearer {_AN_API_KEY}", + "Harvest-Account-ID": _AN_ACCOUNT_ID, + } + ), + _invoices_response().build() + ) + + _read(ConfigBuilder().with_account_id(_AN_ACCOUNT_ID).with_api_token(_AN_API_KEY).with_replication_start_date(datetime.fromisoformat(_A_REPLICATION_START_DATE))) + + # endpoint is called diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json new file mode 100644 index 000000000000..fb4cedccb177 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json @@ -0,0 +1,68 @@ +{ + "invoice_messages": [ + { + "id": 27835209, + "sent_by": "Bob Powell", + "sent_by_email": "bobpowell@example.com", + "sent_from": "Bob Powell", + "sent_from_email": "bobpowell@example.com", + "include_link_to_client_invoice": false, + "send_me_a_copy": false, + "thank_you": false, + "reminder": false, + "send_reminder_on": null, + "created_at": "2017-08-23T22:15:06Z", + "updated_at": "2017-08-23T22:15:06Z", + "attach_pdf": true, + "event_type": null, + "recipients": [ + { + "name": "Richard Roe", + "email": "richardroe@example.com" + } + ], + "subject": "Past due invoice reminder: #1001 from API Examples", + "body": "Dear Customer,\r\n\r\nThis is a friendly reminder to let you know that Invoice 1001 is 144 days past due. If you have already sent the payment, please disregard this message. If not, we would appreciate your prompt attention to this matter.\r\n\r\nThank you for your business.\r\n\r\nCheers,\r\nAPI Examples" + }, + { + "id": 27835207, + "sent_by": "Bob Powell", + "sent_by_email": "bobpowell@example.com", + "sent_from": "Bob Powell", + "sent_from_email": "bobpowell@example.com", + "include_link_to_client_invoice": false, + "send_me_a_copy": true, + "thank_you": false, + "reminder": false, + "send_reminder_on": null, + "created_at": "2017-08-23T22:14:49Z", + "updated_at": "2017-08-23T22:14:49Z", + "attach_pdf": true, + "event_type": null, + "recipients": [ + { + "name": "Richard Roe", + "email": "richardroe@example.com" + }, + { + "name": "Bob Powell", + "email": "bobpowell@example.com" + } + ], + "subject": "Invoice #1001 from API Examples", + "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1001\r\nIssue Date: 04/01/2017\r\nClient: 123 Industries\r\nP.O. Number: \r\nAmount: €288.90\r\nDue: 04/01/2017 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------" + } + ], + "per_page": 2000, + "total_pages": 1, + "total_entries": 2, + "next_page": null, + "previous_page": null, + "page": 1, + "links": { + "first": "https://api.harvestapp.com/api/v2/invoices/13150403/messages?page=1&per_page=2000", + "next": null, + "previous": null, + "last": "https://api.harvestapp.com/v2/invoices/13150403/messages?page=1&per_page=2000" + } +} diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json new file mode 100644 index 000000000000..3be9cee64a0c --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json @@ -0,0 +1,148 @@ +{ + "invoices": [ + { + "id": 13150403, + "client_key": "21312da13d457947a217da6775477afee8c2eba8", + "number": "1001", + "purchase_order": "", + "amount": 288.9, + "due_amount": 288.9, + "tax": 5, + "tax_amount": 13.5, + "tax2": 2, + "tax2_amount": 5.4, + "discount": 10, + "discount_amount": 30, + "subject": "Online Store - Phase 1", + "notes": "Some notes about the invoice.", + "state": "open", + "period_start": "2017-03-01", + "period_end": "2017-03-01", + "issue_date": "2017-04-01", + "due_date": "2017-04-01", + "payment_term": "upon receipt", + "sent_at": "2017-08-23T22:25:59Z", + "paid_at": null, + "paid_date": null, + "closed_at": null, + "recurring_invoice_id": null, + "created_at": "2017-06-27T16:27:16Z", + "updated_at": "2017-08-23T22:25:59Z", + "currency": "EUR", + "payment_options": ["credit_card"], + "client": { + "id": 5735776, + "name": "123 Industries" + }, + "estimate": null, + "retainer": null, + "creator": { + "id": 1782884, + "name": "Bob Powell" + }, + "line_items": [ + { + "id": 53341602, + "kind": "Service", + "description": "03/01/2017 - Project Management: [9:00am - 11:00am] Planning meetings", + "quantity": 2, + "unit_price": 100, + "amount": 200, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + }, + { + "id": 53341603, + "kind": "Service", + "description": "03/01/2017 - Programming: [1:00pm - 2:00pm] Importing products", + "quantity": 1, + "unit_price": 100, + "amount": 100, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + } + ] + }, + { + "id": 13150378, + "client_key": "9e97f4a65c5b83b1fc02f54e5a41c9dc7d458542", + "number": "1000", + "purchase_order": "1234", + "amount": 10700.0, + "due_amount": 0.0, + "tax": 5.0, + "tax_amount": 500.0, + "tax2": 2.0, + "tax2_amount": 200.0, + "discount": null, + "discount_amount": 0.0, + "subject": "Online Store - Phase 1", + "notes": "Some notes about the invoice.", + "state": "paid", + "period_start": null, + "period_end": null, + "issue_date": "2017-02-01", + "due_date": "2017-03-03", + "payment_term": "custom", + "sent_at": "2017-02-01T07:00:00Z", + "paid_at": "2017-02-21T00:00:00Z", + "paid_date": "2017-02-21", + "closed_at": null, + "recurring_invoice_id": null, + "created_at": "2017-06-27T16:24:30Z", + "updated_at": "2017-06-27T16:24:57Z", + "currency": "USD", + "client": { + "id": 5735776, + "name": "123 Industries" + }, + "estimate": { + "id": 1439814 + }, + "retainer": null, + "creator": { + "id": 1782884, + "name": "Bob Powell" + }, + "line_items": [ + { + "id": 53341450, + "kind": "Service", + "description": "50% of Phase 1 of the Online Store", + "quantity": 100.0, + "unit_price": 100.0, + "amount": 10000.0, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + } + ] + } + ], + "per_page": 2000, + "total_pages": 1, + "total_entries": 2, + "next_page": null, + "previous_page": null, + "page": 1, + "links": { + "first": "https://api.harvestapp.com/v2/invoices?page=1&per_page=2000", + "next": null, + "previous": null, + "last": "https://api.harvestapp.com/v2/invoices?page=1&per_page=2000" + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/main.py b/airbyte-integrations/connectors/source-hellobaton/main.py index 1c9ad3053e5a..17a946f7180e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/main.py +++ b/airbyte-integrations/connectors/source-hellobaton/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hellobaton import SourceHellobaton +from source_hellobaton.run import run if __name__ == "__main__": - source = SourceHellobaton() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml index f01eb8e06ffe..183bf69ea8ad 100644 --- a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml +++ b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${company}.hellobaton.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hellobaton registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-hellobaton/setup.py b/airbyte-integrations/connectors/source-hellobaton/setup.py index 91ef2d7cc5f0..d11d669bfcad 100644 --- a/airbyte-integrations/connectors/source-hellobaton/setup.py +++ b/airbyte-integrations/connectors/source-hellobaton/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-hellobaton=source_hellobaton.run:run", + ], + }, name="source_hellobaton", description="Source implementation for Hellobaton.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py new file mode 100644 index 000000000000..4d005af85b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hellobaton import SourceHellobaton + + +def run(): + source = SourceHellobaton() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubplanner/main.py b/airbyte-integrations/connectors/source-hubplanner/main.py index 23d01f5e10a6..aa973ab41009 100644 --- a/airbyte-integrations/connectors/source-hubplanner/main.py +++ b/airbyte-integrations/connectors/source-hubplanner/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hubplanner import SourceHubplanner +from source_hubplanner.run import run if __name__ == "__main__": - source = SourceHubplanner() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml index ab23f4f28def..b14c0ba4e0b1 100644 --- a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hubplanner registries: cloud: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/hubplanner tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py index 5a7474d7c180..dd6d82f7fb64 100644 --- a/airbyte-integrations/connectors/source-hubplanner/setup.py +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-hubplanner=source_hubplanner.run:run", + ], + }, name="source_hubplanner", description="Source implementation for Hubplanner.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py new file mode 100644 index 000000000000..325f3b045b43 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubplanner import SourceHubplanner + + +def run(): + source = SourceHubplanner() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index aae879872bf3..b2d544eab1da 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -1,118 +1,91 @@ -# HubSpot Source +# Hubspot source connector -This is the repository for the HubSpot source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubspot). -## Primary keys - -The primary key for the following streams is `id`: - -- campaigns -- companies -- contacts -- deals -- email_events -- engaments -- engagements_calls -- engagements_emails -- engagements_meetings -- engagements_notes -- engagements_tasks -- forms -- goals -- line_items -- marketing_emails -- owners -- products -- tickets -- ticket_pipelines -- workflows -- quotes - -The primary key for the following streams is `canonical-vid`: - -- contacts_list_memberships - -The primary key for the following streams is `pipelineId`: - -- deal_pipelines - -The primary key for the following streams is `vid-to-merge`: - -- contacts_merged_audit - -The following streams do not have a primary key: - -- contact_lists (The primary key could potentially be a composite key (portalId, listId) - https://legacydocs.hubspot.com/docs/methods/lists/get_lists) -- form_submissions (The entities returned by this endpoint do not have an identifier field - https://legacydocs.hubspot.com/docs/methods/forms/get-submissions-for-a-form) -- subscription_changes (The entities returned by this endpoint do not have an identified field - https://legacydocs.hubspot.com/docs/methods/email/get_subscriptions_timeline) -- property_history (The entities returned by this endpoint do not have an identifier field - https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) +This is the repository for the Hubspot source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/hubspot). ## Local development ### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` + -#### Build & Activate Virtual Environment and install dependencies +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/hubspot) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -From this connector directory, create a virtual environment: +### Locally running the connector ``` -python -m venv .venv +poetry run source-hubspot spec +poetry run source-hubspot check --config secrets/config.json +poetry run source-hubspot discover --config secrets/config.json +poetry run source-hubspot read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: - +### Running unit tests +To run unit tests locally, from the connector directory run: ``` -source .venv/bin/activate -pip install -r requirements.txt +poetry run pytest unit_tests ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. - -#### Create credentials - -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-hubspot build +``` -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubspot test creds` -and place them into `secrets/config.json`. +An image will be available on your host with the tag `airbyte/source-hubspot:dev`. -### Locally running the connector +### Running as a docker container +Then run any of the connector commands as follows: ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config_oauth.json --catalog sample_files/basic_read_catalog.json +docker run --rm airbyte/source-hubspot:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hubspot:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-hubspot test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-hubspot test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/hubspot.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/hubspot.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index dacff1cfa126..ec610513aee2 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -29,20 +29,8 @@ acceptance_tests: extra_records: yes timeout_seconds: 3600 empty_streams: - - name: form_submissions - bypass_reason: Unable to populate - - name: ticket_pipelines - bypass_reason: Unable to populate - - name: engagements_meetings - bypass_reason: Unable to populate - - name: engagements_emails - bypass_reason: Unable to populate - name: engagements_calls - bypass_reason: Unable to populate - - name: quotes - bypass_reason: Unable to populate - - name: deals_archived - bypass_reason: Unable to populate + bypass_reason: Unable to populate cost $20/month - name: owners_archived bypass_reason: unable to populate - name: tickets_web_analytics diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index d90bd6c450ef..091e59551fdb 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -1,72 +1,82 @@ {"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_dropped": 1}, "emitted_at": 1697714185530} {"stream": "campaigns", "data": {"id": 115429485, "lastUpdatedTime": 1615506409286, "appId": 113, "appName": "Batch", "contentId": 42931043849, "subject": "Test subj", "name": "Test subj", "counters": {"processed": 1, "deferred": 1, "mta_dropped": 1, "dropped": 3, "sent": 0}, "lastProcessingFinishedAt": 1615504712000, "lastProcessingStartedAt": 1615504687000, "lastProcessingStateChangeAt": 1615504712000, "numIncluded": 3, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_processed": 1, "counters_deferred": 1, "counters_mta_dropped": 1, "counters_dropped": 3, "counters_sent": 0}, "emitted_at": 1697714185763} -{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 76121938222, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false, "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-10T07:58:09.554000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "airbyte.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": null, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": null, "properties_hs_num_decision_makers": null, "properties_hs_num_open_deals": 1, "properties_hs_object_id": 4992593519, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.5476861596107483, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 76121938222, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "opportunity", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Airbyte test1", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": 1, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 200, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "airbyte.io", "properties_zip": "94114"}, "emitted_at": 1697714187356} -{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 17093729103, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-09-07T03:58:14.126Z", "archived": false, "contacts": ["151", "151"], "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": "2023-04-04T15:00:58.081000+00:00", "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:27:40.002000+00:00", "properties_custom_company_property": null, "properties_days_to_close": 844, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "dataline.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": "5183403213", "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 2, "properties_hs_object_id": 5000526215, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "companies-lifecycle-pipeline", "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.46257445216178894, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": 17093729103, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 66508792054, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": 60010, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "customer", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Dataline", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 1, "properties_num_associated_deals": 3, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 25, "properties_phone": "", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": 60000, "properties_recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": 60000, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "properties_website": "dataline.io", "properties_zip": ""}, "emitted_at": 1697714187359} -{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false, "properties_about_us": null, "properties_address": "2261 Market Street", "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:28:27.673000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "Daxtarity.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": null, "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "", "properties_hs_analytics_latest_source_data_1": "", "properties_hs_analytics_latest_source_data_2": "", "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "", "properties_hs_analytics_source_data_1": "", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 0, "properties_hs_object_id": 5000787595, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4076234698295593, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": null, "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Daxtarity", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 50, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "Daxtarity.com", "properties_zip": "94114"}, "emitted_at": 1697714187363} +{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": "CONTACTS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 86420513185, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false, "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-10T07:58:09.554000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "airbyte.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": null, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": null, "properties_hs_num_decision_makers": null, "properties_hs_num_open_deals": 1, "properties_hs_object_id": 4992593519, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.5476861596107483, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 86420513185, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "opportunity", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Airbyte test1", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": 1, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 200, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "airbyte.io", "properties_zip": "94114"}, "emitted_at": 1708012762427} +{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": 10000000, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": "2024-02-06T10:15:00+00:00", "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2024-02-06T10:15:05.376000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": "2024-02-06T10:15:00+00:00", "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": "CONTACTS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4857041537761688, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 27392304072, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": "COMPUTER_SOFTWARE", "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": "2024-02-06T10:15:00+00:00", "notes_last_updated": "2024-02-06T10:15:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": 2, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": 2, "numberofemployees": 50, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2024-02-06T10:15:05.376Z", "archived": false, "contacts": ["151", "151"], "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": 10000000, "properties_city": "San Francisco", "properties_closedate": "2023-04-04T15:00:58.081000+00:00", "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:27:40.002000+00:00", "properties_custom_company_property": null, "properties_days_to_close": 844, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "dataline.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": "2024-02-06T10:15:00+00:00", "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2024-02-06T10:15:05.376000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": "2024-02-06T10:15:00+00:00", "properties_hs_lead_status": null, "properties_hs_merged_object_ids": "5183403213", "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 2, "properties_hs_object_id": 5000526215, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "companies-lifecycle-pipeline", "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4857041537761688, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": 27392304072, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 66508792054, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": 60010, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": "COMPUTER_SOFTWARE", "properties_is_public": false, "properties_lifecyclestage": "customer", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Dataline", "properties_notes_last_contacted": "2024-02-06T10:15:00+00:00", "properties_notes_last_updated": "2024-02-06T10:15:00+00:00", "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 1, "properties_num_associated_deals": 3, "properties_num_contacted_notes": 2, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": 2, "properties_numberofemployees": 50, "properties_phone": "", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": 60000, "properties_recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "properties_state": "CA", "properties_timezone": "", "properties_total_money_raised": null, "properties_total_revenue": 60000, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "properties_website": "dataline.io", "properties_zip": ""}, "emitted_at": 1708012762428} +{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": 10000000, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2024-01-31T23:50:34.138000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": "CONTACTS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "", "total_money_raised": "31200000", "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2024-01-31T23:50:34.138Z", "archived": false, "properties_about_us": null, "properties_address": "2261 Market Street", "properties_address2": null, "properties_annualrevenue": 10000000, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:28:27.673000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "Daxtarity.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": null, "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "", "properties_hs_analytics_latest_source_data_1": "", "properties_hs_analytics_latest_source_data_2": "", "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "", "properties_hs_analytics_source_data_1": "", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2024-01-31T23:50:34.138000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 0, "properties_hs_object_id": 5000787595, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4076234698295593, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": null, "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Daxtarity", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 50, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "", "properties_total_money_raised": "31200000", "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "Daxtarity.com", "properties_zip": "94114"}, "emitted_at": 1708012762430} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 1, "createdAt": 1610634707370, "updatedAt": 1610634721116, "name": "tweeters", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634721950, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634721950, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189110} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 2, "createdAt": 1610634770432, "updatedAt": 1610634780637, "name": "tweeters 1", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634781147, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634781147, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189112} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 3, "createdAt": 1610634774356, "updatedAt": 1610634787734, "name": "tweeters 2", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634788528, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634788528, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189113} -{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "she", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 94172907549, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "lastname": "nad", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-11-22T21:10:04.346Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_address": null, "properties_annualrevenue": null, "properties_associatedcompanyid": 5000526215, "properties_associatedcompanylastupdated": null, "properties_city": null, "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "shef@dne.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "she", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "151", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "dne.io", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "CONTACTS", "properties_hs_latest_source_data_2": "CRM_UI", "properties_hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 151, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_3", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": null, "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 94172907549, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "properties_lastname": "nad", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": null, "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": null, "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": null}, "emitted_at": 1701823098427} -{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 87820387720, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"], "properties_address": "25000000 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": 5170561229, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": "HubSpot", "properties_company_size": null, "properties_country": "USA", "properties_createdate": "2021-02-22T14:05:09.944000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "testingdsapis@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Test User 5001", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "251", "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "API", "properties_hs_analytics_source_data_2": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": null, "properties_hs_count_is_worked": null, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "API", "properties_hs_latest_source_data_2": null, "properties_hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 251, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "5551222323", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 87820387720, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "properties_lastname": "Test Lastname 5001", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "555-122-2323", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": "http://hubspot.com", "properties_work_email": null, "properties_zip": "02139"}, "emitted_at": 1701823098430} -{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 87712061483, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": null, "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false, "properties_address": "25 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": null, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2021-02-23T20:10:36.191000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "macmitch@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Mac", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "401", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "IMPORT", "properties_hs_analytics_source_data_2": "13256565", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": "+18884827768", "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": "US", "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": "OTHER", "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "IMPORT", "properties_hs_latest_source_data_2": "13256565", "properties_hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": null, "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 401, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "8884827768", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": 87712061483, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": null, "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": true, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "properties_lastname": "Mitchell", "properties_lifecyclestage": "lead", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "1(888) 482-7768", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": "21430"}, "emitted_at": 1701823098432} +{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "she", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": "CONTACTS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 100362752053, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "lastname": "nad", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-11-22T21:10:04.346Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_address": null, "properties_annualrevenue": null, "properties_associatedcompanyid": 5000526215, "properties_associatedcompanylastupdated": null, "properties_city": null, "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "shef@dne.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "she", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "151", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "dne.io", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "CONTACTS", "properties_hs_latest_source_data_2": "CRM_UI", "properties_hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 151, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_3", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": null, "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 100362752053, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "properties_lastname": "nad", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": null, "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": null, "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": null}, "emitted_at": 1708012942318} +{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 94010232227, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"], "properties_address": "25000000 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": 5170561229, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": "HubSpot", "properties_company_size": null, "properties_country": "USA", "properties_createdate": "2021-02-22T14:05:09.944000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "testingdsapis@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Test User 5001", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "251", "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "API", "properties_hs_analytics_source_data_2": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": null, "properties_hs_count_is_worked": null, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "API", "properties_hs_latest_source_data_2": null, "properties_hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 251, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "5551222323", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 94010232227, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "properties_lastname": "Test Lastname 5001", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "555-122-2323", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": "http://hubspot.com", "properties_work_email": null, "properties_zip": "02139"}, "emitted_at": 1708012942320} +{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "13256565", "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 93901905989, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": null, "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false, "properties_address": "25 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": null, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2021-02-23T20:10:36.191000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "macmitch@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Mac", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "401", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "IMPORT", "properties_hs_analytics_source_data_2": "13256565", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": "+18884827768", "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": "US", "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": "OTHER", "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "IMPORT", "properties_hs_latest_source_data_2": "13256565", "properties_hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": null, "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 401, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "13256565", "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "8884827768", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": 93901905989, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": null, "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": true, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "properties_lastname": "Mitchell", "properties_lifecyclestage": "lead", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "1(888) 482-7768", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": "21430"}, "emitted_at": 1708012942321} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1614111042672, "vid": 401, "is-member": true}, "emitted_at": 1697714191502} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1615502112726, "vid": 401, "is-member": true}, "emitted_at": 1697714191513} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1697714191513} {"stream": "contacts_merged_audit", "data": {"canonical-vid": 651, "vid-to-merge": 201, "timestamp": 1688758327178, "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", "user-id": 12282590, "num-properties-moved": 45, "merged_from_email": {"value": "testingapis@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1610634377014, "selected": false}, "merged_to_email": {"value": "testingapicontact_1@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1634044981830, "selected": false}, "first-name": "test", "last-name": "testerson", "merged_from_email_value": "testingapis@hubspot.com", "merged_from_email_source-type": "API", "merged_from_email_source-id": null, "merged_from_email_source-label": null, "merged_from_email_updated-by-user-id": null, "merged_from_email_timestamp": 1610634377014, "merged_from_email_selected": false, "merged_to_email_value": "testingapicontact_1@hubspot.com", "merged_to_email_source-type": "API", "merged_to_email_source-id": null, "merged_to_email_source-label": null, "merged_to_email_updated-by-user-id": null, "merged_to_email_timestamp": 1634044981830, "merged_to_email_selected": false}, "emitted_at": 1697714194351} {"stream": "deal_pipelines", "data": {"label": "New Business Pipeline", "displayOrder": 3, "active": true, "stages": [{"label": "Initial Qualification", "displayOrder": 0, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "9567448", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Success! Closed Won", "displayOrder": 2, "metadata": {"isClosed": "true", "probability": "1.0"}, "stageId": "customclosedwonstage", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Negotiation", "displayOrder": 1, "metadata": {"isClosed": "false", "probability": "0.5"}, "stageId": "9567449", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Closed Lost", "displayOrder": 3, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "66894120", "createdAt": 1680620354263, "updatedAt": 1680620354263, "active": true}], "objectType": "DEAL", "objectTypeId": "0-3", "pipelineId": "b9152945-a594-4835-9676-a6f405fecd71", "createdAt": 1610635973956, "updatedAt": 1680620354263, "default": false}, "emitted_at": 1697714195524} -{"stream": "deals", "data": {"id": "3980651569", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:30:42.221000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3980651569, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 87180354479, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:30:42.221Z", "updatedAt": "2021-09-07T02:36:16.363Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_company": "OFFLINE", "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_1_company": "CONTACTS", "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_data_2_company": "CRM_UI", "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3980651569, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 87180354479, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1697714196730} -{"stream": "deals", "data": {"id": "3980673856", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:31:51.154000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": null, "hs_object_id": 3980673856, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 87180285546, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:31:51.154Z", "updatedAt": "2021-09-07T18:11:59.757Z", "archived": false, "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": null, "properties_hs_object_id": 3980673856, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 87180285546, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1697714196732} -{"stream": "deals", "data": {"id": "3986867076", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-14T14:38:00.797000+00:00", "days_to_close": 0, "dealname": "Test Deal 2", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3986867076, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 87079115903, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-14T14:38:00.797Z", "updatedAt": "2021-09-07T00:24:18.932Z", "archived": false, "companies": ["5183409178", "5183409178"], "properties_amount": 6, "properties_amount_in_home_currency": 6, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test Deal 2", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 6, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3986867076, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 1.2000000000000002, "properties_hs_projected_amount_in_home_currency": 1.2000000000000002, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 87079115903, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1697714196734} +{"stream": "deals", "data": {"id": "3980651569", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:30:42.221000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-21T22:30:34.782000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3980651569, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 97479374284, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:30:42.221Z", "updatedAt": "2024-01-21T22:30:34.782Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_company": "OFFLINE", "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_1_company": "CONTACTS", "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_data_2_company": "CRM_UI", "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-21T22:30:34.782000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3980651569, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 97479374284, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1708013216666} +{"stream": "deals", "data": {"id": "3980673856", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:31:51.154000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-21T02:48:34.022000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": null, "hs_object_id": 3980673856, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 97479305351, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:31:51.154Z", "updatedAt": "2024-01-21T02:48:34.022Z", "archived": false, "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-21T02:48:34.022000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": null, "properties_hs_object_id": 3980673856, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 97479305351, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1708013216668} +{"stream": "deals", "data": {"id": "3986867076", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-14T14:38:00.797000+00:00", "days_to_close": 0, "dealname": "Test Deal 2", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-20T00:59:40.882000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3986867076, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 97378135709, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-14T14:38:00.797Z", "updatedAt": "2024-01-20T00:59:40.882Z", "archived": false, "companies": ["5183409178", "5183409178"], "properties_amount": 6, "properties_amount_in_home_currency": 6, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test Deal 2", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 6, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-20T00:59:40.882000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3986867076, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 1.2000000000000002, "properties_hs_projected_amount_in_home_currency": 1.2000000000000002, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 97378135709, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1708013216669} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "17d3fcc4-bc34-38b4-9103-69b5896bbdde", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614191191202, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199237} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "e5cbe134-db76-32cb-9e82-9dafcbaf8b64", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614122124339, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199238} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "UNITED STATES", "state": "california", "city": "mountain view", "latitude": 37.40599, "longitude": -122.078514, "zipcode": "94043"}, "id": "35b79cd1-3527-3ae7-b316-be0bbf872839", "duration": 1229, "browser": {"name": "Microsoft Edge 12.246", "family": "Microsoft Edge", "producer": "Microsoft Corporation.", "producerUrl": "https://www.microsoft.com/about/", "type": "Browser", "url": "https://en.wikipedia.org/wiki/Microsoft_Edge", "version": ["12.246"]}, "created": 1614119026757, "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": true, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199239} {"stream": "email_subscriptions", "data": {"id": 23704464, "portalId": 8727216, "name": "Test sub", "description": "Test sub", "active": true, "internal": false, "category": "Marketing", "channel": "Email", "businessUnitId": 0}, "emitted_at": 1697714208242} {"stream": "email_subscriptions", "data": {"id": 94692364, "portalId": 8727216, "name": "One to One", "description": "One to One emails", "active": true, "internal": true, "category": "Sales", "channel": "Email", "internalName": "ONE_TO_ONE", "businessUnitId": 0}, "emitted_at": 1697714208243} {"stream": "email_subscriptions", "data": {"id": 10798197, "portalId": 8727216, "name": "DONT USE ME", "description": "Receive feedback requests and customer service information.", "active": true, "internal": true, "category": "Service", "channel": "Email", "order": 0, "internalName": "SERVICE_HUB_FEEDBACK", "businessUnitId": 0}, "emitted_at": 1697714208243} -{"stream": "engagements", "data": {"id": 10584327028, "portalId": 8727216, "active": true, "createdAt": 1610636372009, "lastUpdated": 1610636372009, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 5", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 5\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 5"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 5"}, "emitted_at": 1697714210187} -{"stream": "engagements", "data": {"id": 10584327043, "portalId": 8727216, "active": true, "createdAt": 1610636372714, "lastUpdated": 1610636372714, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 7", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 7\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 7"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 7"}, "emitted_at": 1697714210189} -{"stream": "engagements", "data": {"id": 10584344127, "portalId": 8727216, "active": true, "createdAt": 1610636320990, "lastUpdated": 1610636320990, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body"}, "emitted_at": 1697714210190} -{"stream": "engagements_notes", "data": {"id": "10584327028", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 5", "hs_body_preview_html": "\n \n \n note body 5\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.009000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 5", "hs_object_id": 10584327028, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.009Z", "updatedAt": "2021-01-14T14:59:32.009Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 5", "properties_hs_body_preview_html": "\n \n \n note body 5\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 5", "properties_hs_object_id": 10584327028, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218669} -{"stream": "engagements_notes", "data": {"id": "10584327043", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 7", "hs_body_preview_html": "\n \n \n note body 7\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.714000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 7", "hs_object_id": 10584327043, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.714Z", "updatedAt": "2021-01-14T14:59:32.714Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 7", "properties_hs_body_preview_html": "\n \n \n note body 7\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 7", "properties_hs_object_id": 10584327043, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218670} -{"stream": "engagements_notes", "data": {"id": "10584344127", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body", "hs_body_preview_html": "\n \n \n note body\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:58:40.990000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body", "hs_object_id": 10584344127, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:58:40.990Z", "updatedAt": "2021-01-14T14:58:40.990Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body", "properties_hs_body_preview_html": "\n \n \n note body\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body", "properties_hs_object_id": 10584344127, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218671} -{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2021-02-23T20:25:07.503000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 1, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 11257289597, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "properties_hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": null, "properties_hs_task_reminders": "1614319200000", "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": null, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "Follow up on Test deal 2", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2021-02-26T06:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230220} -{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:41:48.834000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 0, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652597343, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230222} -{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:51:52.099000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 1, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652613208, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230223} +{"stream": "engagements", "data": {"id": 10584327028, "portalId": 8727216, "active": true, "createdAt": 1610636372009, "lastUpdated": 1610636372009, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 5", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 5\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 5"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 5"}, "emitted_at": 1707257892750} +{"stream": "engagements", "data": {"id": 10584327043, "portalId": 8727216, "active": true, "createdAt": 1610636372714, "lastUpdated": 1610636372714, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 7", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 7\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 7"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 7"}, "emitted_at": 1707257892753} +{"stream": "engagements", "data": {"id": 10584344127, "portalId": 8727216, "active": true, "createdAt": 1610636320990, "lastUpdated": 1610636320990, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body"}, "emitted_at": 1707257892756} +{"stream": "engagements_notes", "data": {"id": "10584327028", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 5", "hs_body_preview_html": "\n \n \n note body 5\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.009000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 5", "hs_note_ms_teams_payload": null, "hs_object_id": 10584327028, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.009Z", "updatedAt": "2021-01-14T14:59:32.009Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 5", "properties_hs_body_preview_html": "\n \n \n note body 5\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 5", "properties_hs_note_ms_teams_payload": null, "properties_hs_object_id": 10584327028, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1708013474865} +{"stream": "engagements_notes", "data": {"id": "10584327043", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 7", "hs_body_preview_html": "\n \n \n note body 7\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.714000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 7", "hs_note_ms_teams_payload": null, "hs_object_id": 10584327043, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.714Z", "updatedAt": "2021-01-14T14:59:32.714Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 7", "properties_hs_body_preview_html": "\n \n \n note body 7\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 7", "properties_hs_note_ms_teams_payload": null, "properties_hs_object_id": 10584327043, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1708013474865} +{"stream": "engagements_notes", "data": {"id": "10584344127", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body", "hs_body_preview_html": "\n \n \n note body\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:58:40.990000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body", "hs_note_ms_teams_payload": null, "hs_object_id": 10584344127, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:58:40.990Z", "updatedAt": "2021-01-14T14:58:40.990Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body", "properties_hs_body_preview_html": "\n \n \n note body\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body", "properties_hs_note_ms_teams_payload": null, "properties_hs_object_id": 10584344127, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1708013474865} +{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 11257289597, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_ms_teams_payload": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2021-02-23T20:25:07.503000+00:00", "properties_hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 11257289597, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "properties_hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_ms_teams_payload": null, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": null, "properties_hs_task_reminders": "1614319200000", "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": null, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "Follow up on Test deal 2", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_timestamp": "2021-02-26T06:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708013653895} +{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 30652597343, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_ms_teams_payload": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:41:48.834000+00:00", "properties_hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 30652597343, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_ms_teams_payload": null, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708013653896} +{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 30652613208, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_ms_teams_payload": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:51:52.099000+00:00", "properties_hs_date_entered_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_entered_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_entered_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_entered_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_date_exited_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_date_exited_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_date_exited_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_date_exited_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_date_exited_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 30652613208, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_ms_teams_payload": null, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_time_in_60b5c368_04c4_4d32_9b4a_457e159f49b7_13292096": null, "properties_hs_time_in_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342": null, "properties_hs_time_in_af0e6a5c_2ea3_4c72_b69f_7c6cb3fdb591_1652950531": null, "properties_hs_time_in_dd5826e4_c976_4654_a527_b59ada542e52_2144133616": null, "properties_hs_time_in_fc8148fb_3a2d_4b59_834e_69b7859347cb_1813133675": null, "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708013653897} {"stream": "forms", "data": {"id": "01ba116c-f3a8-4957-8884-ff0c4420af76", "name": "DemoForm", "createdAt": "2021-01-14T14:44:48.278Z", "updatedAt": "2021-01-14T14:44:48.278Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "firstname", "label": "First Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "lastname", "label": "Last Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "adress_1", "label": "Adress 1", "required": false, "hidden": false, "fieldType": "single_line_text"}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": ""}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": [], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "11px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": null}, "legalConsentOptions": {"type": "none"}, "formType": "hubspot"}, "emitted_at": 1697714221520} {"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221521} {"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221522} -{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757009, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-07-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363120} -{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757010, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-09-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363124} -{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757011, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-08-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363125} -{"stream": "line_items", "data": {"id": "1188257165", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257165, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-17T23:50:32.502Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257165, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248811} -{"stream": "line_items", "data": {"id": "1188257309", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257309, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-19T03:57:09.834Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257309, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248814} -{"stream": "line_items", "data": {"id": "1510167477", "properties": {"amount": 20.0, "createdate": "2021-05-21T10:22:40.683000+00:00", "description": "Top hat, large", "discount": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 60.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 10, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "hs_line_item_currency_code": null, "hs_margin": 10.0, "hs_margin_acv": 30.0, "hs_margin_arr": 30.0, "hs_margin_mrr": 10.0, "hs_margin_tcv": 30.0, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_object_id": 1510167477, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": null, "hs_pre_discount_amount": 20, "hs_product_id": 646777910, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": "2022-05-28", "hs_recurring_billing_number_of_payments": 3, "hs_recurring_billing_period": "P3M", "hs_recurring_billing_start_date": "2022-02-28", "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 60.0, "hs_term_in_months": 3, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Red Hat", "price": 20, "quantity": 1, "recurringbillingfrequency": "monthly", "tax": null, "test": "2022-02-24", "test_product_price": "2022-02-23"}, "createdAt": "2021-05-21T10:22:40.683Z", "updatedAt": "2022-02-23T08:09:16.555Z", "archived": false, "properties_amount": 20.0, "properties_createdate": "2021-05-21T10:22:40.683000+00:00", "properties_description": "Top hat, large", "properties_discount": null, "properties_hs_acv": 60.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 60.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 10, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 10.0, "properties_hs_margin_acv": 30.0, "properties_hs_margin_arr": 30.0, "properties_hs_margin_mrr": 10.0, "properties_hs_margin_tcv": 30.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 20.0, "properties_hs_object_id": 1510167477, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": null, "properties_hs_pre_discount_amount": 20, "properties_hs_product_id": 646777910, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": "2022-05-28", "properties_hs_recurring_billing_number_of_payments": 3, "properties_hs_recurring_billing_period": "P3M", "properties_hs_recurring_billing_start_date": "2022-02-28", "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 60.0, "properties_hs_term_in_months": 3, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Red Hat", "properties_price": 20, "properties_quantity": 1, "properties_recurringbillingfrequency": "monthly", "properties_tax": null, "properties_test": "2022-02-24", "properties_test_product_price": "2022-02-23"}, "emitted_at": 1697714248816} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "aifeatures": null, "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645997, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1697714249852} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "aifeatures": null, "allEmailCampaignIds": [169919555], "analyticsPageId": "57347028995", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1634050240841, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundType": "CONTENT", "paddingBottom": "40px", "paddingTop": "40px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "0px", "paddingTop": "0px"}}]}}, "freezeDate": 1634050421336, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 57347028995, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [130, 129, 131, 128, 126, 127, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116], "maxRssEntries": 5, "metaDescription": "", "name": "First test email - 1", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "bgNuSvDn", "primaryEmailCampaignId": 169919555, "processingStatus": "PUBLISHED", "publishDate": 1634050421341, "publishImmediately": true, "publishedAt": 1634050421580, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0}, "deviceBreakdown": {}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 0}}, "styleSettings": {"background_color": "#ffffff", "background_image": null, "background_image_type": null, "body_border_color": null, "body_border_color_choice": null, "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": null, "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#000000", "opacity": 100}, "height": 1, "line_type": null}}}, "subcategory": "batch", "subject": "Subject l", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/plain_text.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634050455543, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249853} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "aifeatures": null, "allEmailCampaignIds": [], "analyticsPageId": "42930862366", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "clonedFrom": 41886608509, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1615502115346, "createdById": 100, "currentState": "AUTOMATED_DRAFT", "currentlyPublished": false, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "AUTOMATED_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-1", "widgets": ["module-0-1-1"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-1", "widgets": ["module-1-1-1"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-1", "widgets": ["module-2-1-1"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1634042970319, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 42930862366, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": false, "isInstanceLayoutPage": false, "isPublished": false, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "lastEditSessionId": 1634042969643, "lastEditUpdateId": 0, "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "Test subject (Test campaing - Clone)", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "UmZGYZsU", "processingStatus": "UNDEFINED", "publishDate": 1634042970320, "publishImmediately": true, "publishedUrl": "", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "slug": "-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "smartEmailFields": {}, "state": "AUTOMATED_DRAFT", "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "automated", "subject": "Test subject", "subscription": 11890831, "subscriptionName": "Test subscription", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634042970321, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249854} +{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_filter_groups_for_key_grouping": null, "hs_kpi_filter_groups_for_key_team_grouping": null, "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_filter_groups_for_key_grouping": null, "properties_hs_kpi_filter_groups_for_key_team_grouping": null, "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757009, "properties_hs_object_source": null, "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-07-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1708013857256} +{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_filter_groups_for_key_grouping": null, "hs_kpi_filter_groups_for_key_team_grouping": null, "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_filter_groups_for_key_grouping": null, "properties_hs_kpi_filter_groups_for_key_team_grouping": null, "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757010, "properties_hs_object_source": null, "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-09-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1708013857258} +{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_filter_groups_for_key_grouping": null, "hs_kpi_filter_groups_for_key_team_grouping": null, "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_filter_groups_for_key_grouping": null, "properties_hs_kpi_filter_groups_for_key_team_grouping": null, "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757011, "properties_hs_object_source": null, "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-08-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1708013857259} +{"stream": "line_items", "data": {"id": "1510167477", "properties": {"amount": 20.0, "createdate": "2021-05-21T10:22:40.683000+00:00", "description": "Top hat, large", "discount": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 60.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 10, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "hs_line_item_currency_code": null, "hs_margin": 10.0, "hs_margin_acv": 30.0, "hs_margin_arr": 30.0, "hs_margin_mrr": 10.0, "hs_margin_tcv": 30.0, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_object_id": 1510167477, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_position_on_quote": null, "hs_pre_discount_amount": 20, "hs_product_id": 646777910, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": "2022-05-28", "hs_recurring_billing_number_of_payments": 3, "hs_recurring_billing_period": "P3M", "hs_recurring_billing_start_date": "2022-02-28", "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 60.0, "hs_term_in_months": 3, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Red Hat", "price": 20, "quantity": 1, "recurringbillingfrequency": "monthly", "tax": null, "test": "2022-02-24", "test_product_price": "2022-02-23"}, "createdAt": "2021-05-21T10:22:40.683Z", "updatedAt": "2022-02-23T08:09:16.555Z", "archived": false, "properties_amount": 20.0, "properties_createdate": "2021-05-21T10:22:40.683000+00:00", "properties_description": "Top hat, large", "properties_discount": null, "properties_hs_acv": 60.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 60.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 10, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 10.0, "properties_hs_margin_acv": 30.0, "properties_hs_margin_arr": 30.0, "properties_hs_margin_mrr": 10.0, "properties_hs_margin_tcv": 30.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 20.0, "properties_hs_object_id": 1510167477, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": null, "properties_hs_pre_discount_amount": 20, "properties_hs_product_id": 646777910, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": "2022-05-28", "properties_hs_recurring_billing_number_of_payments": 3, "properties_hs_recurring_billing_period": "P3M", "properties_hs_recurring_billing_start_date": "2022-02-28", "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 60.0, "properties_hs_term_in_months": 3, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Red Hat", "properties_price": 20, "properties_quantity": 1, "properties_recurringbillingfrequency": "monthly", "properties_tax": null, "properties_test": "2022-02-24", "properties_test_product_price": "2022-02-23"}, "emitted_at": 1708014135793} +{"stream": "line_items", "data": {"id": "2089468681", "properties": {"amount": 10.0, "createdate": "2021-10-12T13:50:13.117000+00:00", "description": "baseball hat, large", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-10-12T13:50:13.117000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 2089468681, "hs_object_source": "API", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": 12282590, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646316535, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-10-12T13:50:13.117Z", "updatedAt": "2021-10-12T13:50:13.117Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-10-12T13:50:13.117000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-10-12T13:50:13.117000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 2089468681, "properties_hs_object_source": "API", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646316535, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014135796} +{"stream": "line_items", "data": {"id": "2089616136", "properties": {"amount": 10.0, "createdate": "2021-10-12T13:50:13.028000+00:00", "description": "baseball hat, large", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-10-12T13:50:13.028000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 2089616136, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646316535, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-10-12T13:50:13.028Z", "updatedAt": "2021-10-12T13:50:13.028Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-10-12T13:50:13.028000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-10-12T13:50:13.028000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 2089616136, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646316535, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014135799} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "aifeatures": null, "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingIlsListsExcluded": [], "mailingIlsListsIncluded": [], "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645000, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1708014401022} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "aifeatures": null, "allEmailCampaignIds": [169919555], "analyticsPageId": "57347028995", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1634050240841, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundType": "CONTENT", "paddingBottom": "40px", "paddingTop": "40px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "0px", "paddingTop": "0px"}}]}}, "freezeDate": 1634050421336, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 57347028995, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingIlsListsExcluded": [], "mailingIlsListsIncluded": [], "mailingListsExcluded": [], "mailingListsIncluded": [130, 129, 131, 128, 126, 127, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116], "maxRssEntries": 5, "metaDescription": "", "name": "First test email - 1", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "bgNuSvDn", "primaryEmailCampaignId": 169919555, "processingStatus": "PUBLISHED", "publishDate": 1634050421000, "publishImmediately": true, "publishedAt": 1634050421580, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0}, "deviceBreakdown": {}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 0}}, "styleSettings": {"background_color": "#ffffff", "background_image": null, "background_image_type": null, "body_border_color": null, "body_border_color_choice": null, "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": null, "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#000000", "opacity": 100}, "height": 1, "line_type": null}}}, "subcategory": "batch", "subject": "Subject l", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/plain_text.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634050455543, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1708014401026} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "aifeatures": null, "allEmailCampaignIds": [], "analyticsPageId": "42930862366", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "clonedFrom": 41886608509, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1615502115346, "createdById": 100, "currentState": "AUTOMATED_DRAFT", "currentlyPublished": false, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "AUTOMATED_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-1", "widgets": ["module-0-1-1"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-1", "widgets": ["module-1-1-1"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-1", "widgets": ["module-2-1-1"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1634042970319, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 42930862366, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": false, "isInstanceLayoutPage": false, "isPublished": false, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "lastEditSessionId": 1634042969643, "lastEditUpdateId": 0, "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingIlsListsExcluded": [], "mailingIlsListsIncluded": [], "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "Test subject (Test campaing - Clone)", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "UmZGYZsU", "processingStatus": "UNDEFINED", "publishDate": 1634042970000, "publishImmediately": true, "publishedUrl": "", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "slug": "-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "smartEmailFields": {}, "state": "AUTOMATED_DRAFT", "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "automated", "subject": "Test subject", "subscription": 11890831, "subscriptionName": "Test subscription", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634042970321, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1708014401029} {"stream": "owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1697714250730} {"stream": "owners", "data": {"id": "65568071", "email": "test-integration-test-user1@airbyte.io", "firstName": "", "lastName": "", "userId": 23660227, "createdAt": "2021-03-15T11:00:50.053Z", "updatedAt": "2021-03-15T11:00:50.053Z", "archived": false}, "emitted_at": 1697714250731} {"stream": "owners", "data": {"id": "65568800", "email": "test-integration-test-user2@airbyte.io", "firstName": "", "lastName": "", "userId": 23660229, "createdAt": "2021-03-15T11:01:02.183Z", "updatedAt": "2021-03-15T11:01:02.183Z", "archived": false}, "emitted_at": 1697714250732} -{"stream": "products", "data": {"id": "646176421", "properties": {"amount": null, "createdate": "2021-02-23T20:03:18.336000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176421, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product", "price": 100, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:18.336Z", "updatedAt": "2021-02-23T20:03:18.336Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:18.336000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176421, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product", "properties_price": 100, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252635} -{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252637} -{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252638} +{"stream": "products", "data": {"id": "646176421", "properties": {"amount": null, "createdate": "2021-02-23T20:03:18.336000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176421, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product", "price": 100, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:18.336Z", "updatedAt": "2021-02-23T20:03:18.336Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:18.336000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176421, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product", "properties_price": 100, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014628640} +{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014628643} +{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014628645} {"stream": "contacts_property_history", "data": {"value": "testo", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1700681340515, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} {"stream": "contacts_property_history", "data": {"value": "test", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1675120629904, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} {"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_2", "value": "CRM_UI", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_2", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242} {"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_1", "value": "CONTACTS", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_1", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242} -{"stream": "deals_property_history", "data": {"name": "dealname", "value": "Test deal 2", "timestamp": 1614111692862, "sourceId": "userId:12282590", "source": "CRM_UI", "sourceVid": [], "requestId": "1ce13074-883d-4d9c-9d07-e01e8f23f363", "updatedByUserId": 12282590, "property": "dealname", "dealId": 4315375411, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905810513} +{"stream": "deals_property_history", "data": {"name": "dealname", "value": "Test Deal 2", "timestamp": 1610635080797, "source": "API", "sourceVid": [], "requestId": "cdc0501c-7d08-40e4-a937-953492b1a6c2", "property": "dealname", "dealId": 3986867076, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1707258294359} {"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "portalId": 8727216, "causedByEvent": {"id": "d70b78b9-a411-4d3e-808b-fe931be35b43", "created": 1616173134301}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255435} {"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "subscriptionId": 10798197, "portalId": 8727216, "causedByEvent": {"id": "ff118718-786d-4a35-94f9-6bbd413654de", "created": 1616173134301}, "changeType": "SUBSCRIPTION_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255436} {"stream": "subscription_changes", "data": {"timestamp": 1616173106737, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173106737, "portalId": 8727216, "causedByEvent": {"id": "24539f1f-0b20-4296-a5bf-6ba3bb9dc1b8", "created": 1616173106737}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255437} -{"stream": "tickets", "data": {"id": "312929579", "properties": {"closed_date": "2021-02-23T20:08:49.603000+00:00", "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312929579, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "4", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312929579, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 0, "hs_time_in_3": 0, "hs_time_in_4": 87748604133, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "CHAT", "subject": "Marketing Starter", "tags": null, "time_to_close": 0, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties_closed_date": "2021-02-23T20:08:49.603000+00:00", "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312929579, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "4", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312929579, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 0, "properties_hs_time_in_3": 0, "properties_hs_time_in_4": 87748604133, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "CHAT", "properties_subject": "Marketing Starter", "properties_tags": null, "properties_time_to_close": 0, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534671} -{"stream": "tickets", "data": {"id": "312972611", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312972611, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "2", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312972611, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 87748604132, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "FORM", "subject": "Sales Starter", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.663Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312972611, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "2", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312972611, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 87748604132, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "FORM", "properties_subject": "Sales Starter", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534672} -{"stream": "tickets", "data": {"id": "312975112", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312975112, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312975112, "hs_ticket_priority": "MEDIUM", "hs_time_in_1": 87748604134, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "PHONE", "subject": "Free CRM", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.515Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": null, "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312975112, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "1", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312975112, "properties_hs_ticket_priority": "MEDIUM", "properties_hs_time_in_1": 87748604134, "properties_hs_time_in_2": null, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "PHONE", "properties_subject": "Free CRM", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534673} +{"stream": "tickets", "data": {"id": "312929579", "properties": {"closed_date": "2021-02-23T20:08:49.603000+00:00", "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_151692305": null, "hs_date_entered_151692306": null, "hs_date_entered_151692307": null, "hs_date_entered_151692308": null, "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_151692305": null, "hs_date_exited_151692306": null, "hs_date_exited_151692307": null, "hs_date_exited_151692308": null, "hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312929579, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "4", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312929579, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_151692305": null, "hs_time_in_151692306": null, "hs_time_in_151692307": null, "hs_time_in_151692308": null, "hs_time_in_2": 0, "hs_time_in_3": 0, "hs_time_in_4": 93903870829, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "CHAT", "subject": "Marketing Starter", "tags": null, "time_to_close": 0, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties_closed_date": "2021-02-23T20:08:49.603000+00:00", "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_151692305": null, "properties_hs_date_entered_151692306": null, "properties_hs_date_entered_151692307": null, "properties_hs_date_entered_151692308": null, "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_151692305": null, "properties_hs_date_exited_151692306": null, "properties_hs_date_exited_151692307": null, "properties_hs_date_exited_151692308": null, "properties_hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312929579, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "4", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312929579, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_151692305": null, "properties_hs_time_in_151692306": null, "properties_hs_time_in_151692307": null, "properties_hs_time_in_151692308": null, "properties_hs_time_in_2": 0, "properties_hs_time_in_3": 0, "properties_hs_time_in_4": 93903870829, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "CHAT", "properties_subject": "Marketing Starter", "properties_tags": null, "properties_time_to_close": 0, "properties_time_to_first_agent_reply": null}, "emitted_at": 1708014800593} +{"stream": "tickets", "data": {"id": "312972611", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_151692305": null, "hs_date_entered_151692306": null, "hs_date_entered_151692307": null, "hs_date_entered_151692308": null, "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_151692305": null, "hs_date_exited_151692306": null, "hs_date_exited_151692307": null, "hs_date_exited_151692308": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312972611, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "2", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312972611, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_151692305": null, "hs_time_in_151692306": null, "hs_time_in_151692307": null, "hs_time_in_151692308": null, "hs_time_in_2": 93903870829, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "FORM", "subject": "Sales Starter", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.663Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_151692305": null, "properties_hs_date_entered_151692306": null, "properties_hs_date_entered_151692307": null, "properties_hs_date_entered_151692308": null, "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_151692305": null, "properties_hs_date_exited_151692306": null, "properties_hs_date_exited_151692307": null, "properties_hs_date_exited_151692308": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312972611, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "2", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312972611, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_151692305": null, "properties_hs_time_in_151692306": null, "properties_hs_time_in_151692307": null, "properties_hs_time_in_151692308": null, "properties_hs_time_in_2": 93903870829, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "FORM", "properties_subject": "Sales Starter", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1708014800594} +{"stream": "tickets", "data": {"id": "312975112", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_151692305": null, "hs_date_entered_151692306": null, "hs_date_entered_151692307": null, "hs_date_entered_151692308": null, "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_151692305": null, "hs_date_exited_151692306": null, "hs_date_exited_151692307": null, "hs_date_exited_151692308": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312975112, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312975112, "hs_ticket_priority": "MEDIUM", "hs_time_in_1": 93903870829, "hs_time_in_151692305": null, "hs_time_in_151692306": null, "hs_time_in_151692307": null, "hs_time_in_151692308": null, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "PHONE", "subject": "Free CRM", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.515Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_151692305": null, "properties_hs_date_entered_151692306": null, "properties_hs_date_entered_151692307": null, "properties_hs_date_entered_151692308": null, "properties_hs_date_entered_2": null, "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": null, "properties_hs_date_exited_151692305": null, "properties_hs_date_exited_151692306": null, "properties_hs_date_exited_151692307": null, "properties_hs_date_exited_151692308": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312975112, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "1", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312975112, "properties_hs_ticket_priority": "MEDIUM", "properties_hs_time_in_1": 93903870829, "properties_hs_time_in_151692305": null, "properties_hs_time_in_151692306": null, "properties_hs_time_in_151692307": null, "properties_hs_time_in_151692308": null, "properties_hs_time_in_2": null, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "PHONE", "properties_subject": "Free CRM", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1708014800595} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058115, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50206671}, "name": "Test Workflow", "id": 21058115, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635826795}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907577}, "contactListIds": {"enrolled": 12, "active": 13, "completed": 14, "succeeded": 15}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635826921, "updatedAt": 1611847907577, "contactListIds_enrolled": 12, "contactListIds_active": 13, "contactListIds_completed": 14, "contactListIds_succeeded": 15}, "emitted_at": 1697714264418} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058121, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205684}, "name": "Test Workflow 1", "id": 21058121, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635850713}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907579}, "contactListIds": {"enrolled": 16, "active": 17, "completed": 18, "succeeded": 19}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635850758, "updatedAt": 1611847907579, "contactListIds_enrolled": 16, "contactListIds_active": 17, "contactListIds_completed": 18, "contactListIds_succeeded": 19}, "emitted_at": 1697714264419} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058122, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205036}, "name": "Test Workflow 2", "id": 21058122, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635859664}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907578}, "contactListIds": {"enrolled": 20, "active": 21, "completed": 22, "succeeded": 23}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635859748, "updatedAt": 1611847907578, "contactListIds_enrolled": 20, "contactListIds_active": 21, "contactListIds_completed": 22, "contactListIds_succeeded": 23}, "emitted_at": 1697714264420} -{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 1, "properties_car_name": 3232324, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:15.836000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880072, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1697714265295} -{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 2, "properties_car_name": 23232, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:20.583000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880073, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1697714265296} -{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5936415312, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Marcos Pet", "properties_pet_type": "Dog"}, "emitted_at": 1697714266285} -{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880054, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Integration Test Pet", "properties_pet_type": "Unknown"}, "emitted_at": 1697714266285} -{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.435Z", "id": "d287cdb7-3e8a-4f4d-92db-486e32f99ad4", "properties_hs_region": "officiis exercitationem modi adipisicing odit Hic", "properties_hs_campaign_id": "libero", "properties_hs_page_url": "Lorem", "properties_hs_element_id": "dolor sit", "properties_hs_browser": "architecto molestias, officiis exercitationem sit", "properties_hs_screen_width": "1531.0", "properties_hs_device_type": "sit adipisicing nobis officiis modi dolor sit", "properties_hs_link_href": "dolor magnam,", "properties_hs_element_class": "exercitationem modi nobis amet odit molestias,", "properties_hs_operating_system": "culpa! ipsum adipisicing consectetur nobis culpa!", "properties_hs_touchpoint_source": "libero modi odit ipsum Lorem accusantium culpa!", "properties_hs_utm_medium": "elit. ipsum officiis molestias, ipsum dolor quas"}, "emitted_at": 1701822848687} -{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.723Z", "id": "2f756b9a-a68d-4566-8e63-bc66b9149b41", "properties_hs_page_id": "modi sit", "properties_hs_city": "possimus modi culpa! veniam Lorem odit Lorem quas", "properties_hs_parent_module_id": "reprehenderit exercitationem dolor adipisicing", "properties_hs_user_agent": "possimus reprehenderit architecto odit ipsum, sit", "properties_hs_operating_version": "adipisicing", "properties_hs_element_id": "architecto exercitationem consectetur modi Lorem", "properties_hs_page_content_type": "amet", "properties_hs_screen_height": "4588.0", "properties_hs_operating_system": "reiciendis placeat possimus ipsum, adipisicing", "properties_hs_language": "adipisicing reprehenderit sit ipsum, amet nobis", "properties_hs_region": "placeat accusantium adipisicing culpa! modi quas", "properties_hs_utm_source": "molestias, reprehenderit reprehenderit", "properties_hs_referrer": "possimus consectetur odit sit Lorem nobis culpa!"}, "emitted_at": 1701822848688} \ No newline at end of file +{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 1, "properties_car_name": 3232324, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:15.836000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880072, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1708014961477} +{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 2, "properties_car_name": 23232, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:20.583000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880073, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1708014961480} +{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5936415312, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Marcos Pet", "properties_pet_type": "Dog"}, "emitted_at": 1708015148856} +{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880054, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Integration Test Pet", "properties_pet_type": "Unknown"}, "emitted_at": 1708015148859} +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "151", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-11-24T22:35:09.286Z", "id": "54338ee5-9db1-4ba3-afa8-1b052508fc8f"}, "emitted_at": 1707257902940} +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "151", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T21:50:11.797Z", "id": "b850d903-254c-4df6-b159-9263b2b7eed0", "properties_hs_campaign_id": "illum quas dolor modi exercitationem", "properties_hs_operating_version": "Lorem ipsum culpa! illum elit. esse esse officiis", "properties_hs_tracking_name": "quas ipsum amet illum molestias,", "properties_hs_page_content_type": "elit. libero Lorem", "properties_hs_region": "consectetur ipsum, architecto ipsum Lorem nobis", "properties_hs_device_type": "accusantium dolor sit elit. veniam reprehenderit", "properties_hs_element_id": "culpa! sit ipsum sit Lorem consectetur quas odit", "properties_hs_device_name": "elit. molestias, elit. amet", "properties_hs_touchpoint_source": "dolor", "properties_hs_page_id": "magnam, magnam,", "properties_hs_link_href": "officiis exercitationem adipisicing odit dolor", "properties_hs_city": "reiciendis placeat dolor placeat architecto dolor", "properties_hs_language": "officiis adipisicing", "properties_hs_asset_description": "dolor architecto", "properties_hs_page_url": "accusantium quas architecto ipsum ipsum possimus", "properties_hs_utm_campaign": "magnam, Lorem modi culpa!", "properties_hs_utm_medium": "placeat dolor dolor consectetur elit.", "properties_hs_element_text": "Hic molestias, Lorem ipsum, possimus adipisicing", "properties_hs_operating_system": "magnam, molestias,"}, "emitted_at": 1707257902946} +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "151", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:06:45.502Z", "id": "4265b60b-7873-45a6-8983-882134c51dc2", "properties_hs_campaign_id": "officiis quas nobis adipisicing sit", "properties_hs_operating_version": "repellendus esse", "properties_hs_page_content_type": "adipisicing consectetur repellendus consectetur", "properties_hs_region": "repellendus officiis odit esse", "properties_hs_device_name": "veniam", "properties_hs_touchpoint_source": "modi modi repellendus Lorem Lorem reiciendis Hic", "properties_hs_page_id": "officiis", "properties_hs_referrer": "consectetur placeat architecto libero elit. modi", "properties_hs_asset_type": "odit reprehenderit placeat", "properties_hs_screen_height": "2623.0", "properties_hs_screen_width": "4270.0", "properties_hs_language": "reprehenderit molestias, reiciendis magnam, Lorem", "properties_hs_page_url": "placeat amet officiis possimus veniam", "properties_hs_element_class": "repellendus magnam, ipsum, amet magnam, Hic ipsum", "properties_hs_country": "exercitationem", "properties_hs_utm_source": "architecto reiciendis sit illum odit ipsum nobis", "properties_hs_utm_medium": "ipsum, officiis magnam, odit modi reiciendis odit", "properties_hs_element_text": "veniam ipsum, consectetur reiciendis adipisicing", "properties_hs_operating_system": "dolor reprehenderit amet officiis exercitationem"}, "emitted_at": 1707257902951} +{"stream": "form_submissions", "data": {"submittedAt": 1707094502866, "values": [{"name": "email", "value": "integration-test+hubspot_form_100@airbyte.io", "objectTypeId": "0-1"}], "pageUrl": "https://share.hsforms.com/17X1n1tQkRLOOmod8jZV67A571yo", "updatedAt": 1707094502866, "formId": "ed7d67d6-d424-44b3-8e9a-877c8d957aec"}, "emitted_at": 1707094528032} +{"stream": "contacts_form_submissions", "data": {"canonical-vid": 3001, "conversion-id": "2ec044dd-5ba6-4bbf-b64d-2b3a561d8434", "timestamp": 1707094108543, "form-id": "49773438-eebc-4622-a70b-f2102839d416", "portal-id": 8727216, "page-url": "https://meetings.hubspot.com/team-1-airbyte", "title": "Meetings Link: team-1-airbyte", "form-type": "MEETING", "contact-associated-by": ["EMAIL"], "meta-data": []}, "emitted_at": 1707094509475} +{"stream": "contacts_form_submissions", "data": {"canonical-vid": 3101, "conversion-id": "aed975ea-68dd-456a-aef1-c80ef08001e8", "timestamp": 1707094502866, "form-id": "ed7d67d6-d424-44b3-8e9a-877c8d957aec", "portal-id": 8727216, "page-url": "https://share.hsforms.com/17X1n1tQkRLOOmod8jZV67A571yo", "canonical-url": "https://share.hsforms.com/17X1n1tQkRLOOmod8jZV67A571yo", "page-title": "Form", "title": "New form 100", "form-type": "HUBSPOT", "meta-data": []}, "emitted_at": 1707094509476} +{"stream": "deals_archived", "data": {"id": "15165693770", "properties": {"amount": 0, "amount_in_home_currency": 0, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2023-09-15T09:08:03.642000+00:00", "createdate": "2023-09-15T09:08:20.208000+00:00", "days_to_close": 0, "dealname": "Test 1715 Deal Acrhived Line Items", "dealstage": "closedwon", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": 1, "hs_closed_won_date": "2023-09-15T09:08:03.642000+00:00", "hs_created_by_user_id": 12282590, "hs_createdate": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_contractsent": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_presentationscheduled": "2023-09-15T09:08:20.208000+00:00", "hs_date_entered_qualifiedtobuy": "2023-09-15T09:08:20.208000+00:00", "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": "2023-09-15T09:08:20.208000+00:00", "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": "2023-09-15T09:08:20.208000+00:00", "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": "2023-09-15T09:08:20.208000+00:00", "hs_date_exited_presentationscheduled": "2023-09-15T09:08:20.208000+00:00", "hs_date_exited_qualifiedtobuy": "2023-09-15T09:08:20.208000+00:00", "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 1, "hs_deal_stage_probability_shadow": 1, "hs_exchange_rate": null, "hs_forecast_amount": 0, "hs_forecast_probability": null, "hs_is_closed": true, "hs_is_closed_won": true, "hs_is_deal_split": false, "hs_is_open_count": 0, "hs_lastmodifieddate": "2023-09-18T09:09:00.660000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": 0, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 15165693770, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": "low", "hs_projected_amount": 0, "hs_projected_amount_in_home_currency": 0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 0, "hs_time_in_closedlost": null, "hs_time_in_closedwon": 13246483990, "hs_time_in_contractsent": 0, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": 0, "hs_time_in_presentationscheduled": 0, "hs_time_in_qualifiedtobuy": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-09-15T09:08:20.208000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2023-09-18T09:08:59.252000+00:00", "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": 0, "num_notes": 2, "pipeline": "default"}, "createdAt": "2023-09-15T09:08:20.208Z", "updatedAt": "2023-09-18T09:09:00.660Z", "archived": true, "archivedAt": "2024-02-05T00:58:23.662Z", "properties_amount": 0, "properties_amount_in_home_currency": 0, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2023-09-15T09:08:03.642000+00:00", "properties_createdate": "2023-09-15T09:08:20.208000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test 1715 Deal Acrhived Line Items", "properties_dealstage": "closedwon", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": 1, "properties_hs_closed_won_date": "2023-09-15T09:08:03.642000+00:00", "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_contractsent": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_presentationscheduled": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_entered_qualifiedtobuy": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_exited_presentationscheduled": "2023-09-15T09:08:20.208000+00:00", "properties_hs_date_exited_qualifiedtobuy": "2023-09-15T09:08:20.208000+00:00", "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 1, "properties_hs_deal_stage_probability_shadow": 1, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 0, "properties_hs_forecast_probability": null, "properties_hs_is_closed": true, "properties_hs_is_closed_won": true, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 0, "properties_hs_lastmodifieddate": "2023-09-18T09:09:00.660000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": 0, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 15165693770, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": "low", "properties_hs_projected_amount": 0, "properties_hs_projected_amount_in_home_currency": 0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 0, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": 13246483990, "properties_hs_time_in_contractsent": 0, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": 0, "properties_hs_time_in_presentationscheduled": 0, "properties_hs_time_in_qualifiedtobuy": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-09-15T09:08:20.208000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": "2023-09-18T09:08:59.252000+00:00", "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": 0, "properties_num_notes": 2, "properties_pipeline": "default"}, "emitted_at": 1708015384348} +{"stream": "ticket_pipelines", "data": {"label": "Test_ticket_pipeline", "displayOrder": 1, "id": "80068448", "stages": [{"label": "New", "displayOrder": 0, "metadata": {"ticketState": "OPEN", "isClosed": "false"}, "id": "151692305", "createdAt": "2024-02-05T01:01:42.937Z", "updatedAt": "2024-02-05T01:01:42.937Z", "archived": false, "writePermissions": "CRM_PERMISSIONS_ENFORCEMENT"}, {"label": "Waiting on contact", "displayOrder": 1, "metadata": {"ticketState": "OPEN", "isClosed": "false"}, "id": "151692306", "createdAt": "2024-02-05T01:01:42.937Z", "updatedAt": "2024-02-05T01:01:42.937Z", "archived": false, "writePermissions": "CRM_PERMISSIONS_ENFORCEMENT"}, {"label": "Waiting on us", "displayOrder": 2, "metadata": {"ticketState": "OPEN", "isClosed": "false"}, "id": "151692307", "createdAt": "2024-02-05T01:01:42.937Z", "updatedAt": "2024-02-05T01:01:42.937Z", "archived": false, "writePermissions": "CRM_PERMISSIONS_ENFORCEMENT"}, {"label": "Closed", "displayOrder": 3, "metadata": {"ticketState": "CLOSED", "isClosed": "true"}, "id": "151692308", "createdAt": "2024-02-05T01:01:42.937Z", "updatedAt": "2024-02-05T01:01:42.937Z", "archived": false, "writePermissions": "CRM_PERMISSIONS_ENFORCEMENT"}], "createdAt": "2024-02-05T01:01:42.937Z", "updatedAt": "2024-02-05T01:01:42.937Z", "archived": false}, "emitted_at": 1707258209328} +{"stream": "engagements_emails", "data": {"id": "46838275228", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test body -- Prefer fewer emails from me? Click here", "hs_body_preview_html": "\n \n \n
      \n test body \n
      \n
      \n -- \n
      \n Prefer fewer emails from me? Click here \n
      \n
      \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": "12282590", "hs_created_by_user_id": 12282590, "hs_createdate": "2024-02-05T01:13:21.505000+00:00", "hs_direction_and_unique_id": "EMAIL:432a7d905bf8fc42ba938819a9e6e291", "hs_email_attached_video_id": null, "hs_email_attached_video_name": null, "hs_email_attached_video_opened": false, "hs_email_attached_video_watched": false, "hs_email_bcc_email": null, "hs_email_bcc_firstname": null, "hs_email_bcc_lastname": null, "hs_email_bcc_raw": null, "hs_email_bounce_error_detail_message": null, "hs_email_bounce_error_detail_status_code": null, "hs_email_cc_email": null, "hs_email_cc_firstname": null, "hs_email_cc_lastname": null, "hs_email_cc_raw": null, "hs_email_click_count": null, "hs_email_direction": "EMAIL", "hs_email_encoded_email_associations_request": null, "hs_email_error_message": null, "hs_email_facsimile_send_id": "6b0d1024453e0b541501565ae69498c7", "hs_email_from_email": "integration-test-user@airbyte.io", "hs_email_from_firstname": "Team-1", "hs_email_from_lastname": "Airbyte", "hs_email_from_raw": null, "hs_email_has_inline_images_stripped": null, "hs_email_headers": "{\"from\":{\"email\":\"integration-test-user@airbyte.io\",\"firstName\":\"Team-1\",\"lastName\":\"Airbyte\"},\"to\":[{\"raw\":\"gl_serhii.lazebnyi@airbyte.io\",\"email\":\"gl_serhii.lazebnyi@airbyte.io\"}],\"cc\":[],\"bcc\":[],\"sender\":{\"email\":\"integration-test-user@airbyte.io\"}}", "hs_email_html": "
      test body
      --
      Prefer fewer emails from me? Click here

      ", "hs_email_logged_from": "CRM", "hs_email_media_processing_status": "SKIPPED", "hs_email_member_of_forwarded_subthread": null, "hs_email_message_id": "CAK4c3Gyf4xNPCtrON3BFLN9WOWUpfe+sfb+7wh5qYuCD-K71AA@mail.gmail.com", "hs_email_migrated_via_portal_data_migration": null, "hs_email_ms_teams_payload": null, "hs_email_open_count": null, "hs_email_pending_inline_image_ids": null, "hs_email_post_send_status": "SENT", "hs_email_recipient_drop_reasons": null, "hs_email_reply_count": null, "hs_email_send_event_id": null, "hs_email_send_event_id_created": null, "hs_email_sender_email": "integration-test-user@airbyte.io", "hs_email_sender_firstname": null, "hs_email_sender_lastname": null, "hs_email_sender_raw": null, "hs_email_sent_count": 1.0, "hs_email_sent_via": "GMAIL", "hs_email_status": "SENT", "hs_email_stripped_attachment_count": null, "hs_email_subject": "test deal ", "hs_email_text": "test body\n-- \nPrefer fewer emails from me? Click here: https://d11qV604.na1.hs-salescrm-sub.com/preferences/en/manage?data=W2nXS-N30h-MkW3DX4xr38lXTKW2KXbZn3H3ZTKW4kt7Y_3XR2G0W30sn1g2zt_2NW47kvvy23ncKnW47Vmcy4pxy7cW41tzTm1X87X1W364bL-36tRLFW30J_Vy36F403W45FGpL3XHz-RW4ftDwZ4msYq_W24-jyc2HCSCvW3VGBr52TLG1vW2nFrmM3P2tStW43Skr81VxgJXW3z26wT4pc1KRW1Vpb_f3d3w7qW36dtk_4rCSHJW3F507n1_6v4MW2CWCvk49rVZpW23jtn51St_bDW2RKdYG2RNzKSW47znqq1_dHnNW4mGNp33Y1JRBW25m60s1Nk9WFW2MMKcf2F-zTNW4kddlH1NFHhxW25nrXX2KQX5rW3GJy1x2Yh7XsW2Pnx-93f_bXGW47SgSp1XqcMJW2FTQ1Z2KPBb6W32kvXr2KnzH9W3HcvHw3LRJmmW2MLX-W3LBLBJW3Q-74Q2KYV0CW1_9nCQ2r36_S0", "hs_email_thread_id": "3b2bf39b9ed8cfc53310ee557627d073", "hs_email_thread_summary": null, "hs_email_to_email": "gl_serhii.lazebnyi@airbyte.io", "hs_email_to_firstname": null, "hs_email_to_lastname": null, "hs_email_to_raw": "gl_serhii.lazebnyi@airbyte.io", "hs_email_tracker_key": "87989bf6-7771-4486-b3d7-73a31af32b2c", "hs_email_validation_skipped": null, "hs_engagement_source": "EMAIL_INTEGRATION", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2024-02-05T01:13:26.539000+00:00", "hs_merged_object_ids": null, "hs_modified_by": "12282590", "hs_object_id": 46838275228, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_owner_ids_bcc": null, "hs_owner_ids_cc": null, "hs_owner_ids_from": "52550153", "hs_owner_ids_to": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_scs_association_status": null, "hs_scs_audit_id": null, "hs_timestamp": "2024-02-05T01:13:21.109000+00:00", "hs_unique_creation_key": null, "hs_unique_id": "432a7d905bf8fc42ba938819a9e6e291", "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2024-02-05T01:13:21.505000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2024-02-05T01:13:21.505Z", "updatedAt": "2024-02-05T01:13:26.539Z", "archived": false, "companies": ["5000526215"], "deals": ["5388306989"], "contacts": ["3251"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "test body -- Prefer fewer emails from me? Click here", "properties_hs_body_preview_html": "\n \n \n
      \n test body \n
      \n
      \n -- \n
      \n Prefer fewer emails from me? Click here \n
      \n
      \n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": "12282590", "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2024-02-05T01:13:21.505000+00:00", "properties_hs_direction_and_unique_id": "EMAIL:432a7d905bf8fc42ba938819a9e6e291", "properties_hs_email_attached_video_id": null, "properties_hs_email_attached_video_name": null, "properties_hs_email_attached_video_opened": false, "properties_hs_email_attached_video_watched": false, "properties_hs_email_bcc_email": null, "properties_hs_email_bcc_firstname": null, "properties_hs_email_bcc_lastname": null, "properties_hs_email_bcc_raw": null, "properties_hs_email_bounce_error_detail_message": null, "properties_hs_email_bounce_error_detail_status_code": null, "properties_hs_email_cc_email": null, "properties_hs_email_cc_firstname": null, "properties_hs_email_cc_lastname": null, "properties_hs_email_cc_raw": null, "properties_hs_email_click_count": null, "properties_hs_email_direction": "EMAIL", "properties_hs_email_encoded_email_associations_request": null, "properties_hs_email_error_message": null, "properties_hs_email_facsimile_send_id": "6b0d1024453e0b541501565ae69498c7", "properties_hs_email_from_email": "integration-test-user@airbyte.io", "properties_hs_email_from_firstname": "Team-1", "properties_hs_email_from_lastname": "Airbyte", "properties_hs_email_from_raw": null, "properties_hs_email_has_inline_images_stripped": null, "properties_hs_email_headers": "{\"from\":{\"email\":\"integration-test-user@airbyte.io\",\"firstName\":\"Team-1\",\"lastName\":\"Airbyte\"},\"to\":[{\"raw\":\"gl_serhii.lazebnyi@airbyte.io\",\"email\":\"gl_serhii.lazebnyi@airbyte.io\"}],\"cc\":[],\"bcc\":[],\"sender\":{\"email\":\"integration-test-user@airbyte.io\"}}", "properties_hs_email_html": "
      test body
      --
      Prefer fewer emails from me? Click here

      ", "properties_hs_email_logged_from": "CRM", "properties_hs_email_media_processing_status": "SKIPPED", "properties_hs_email_member_of_forwarded_subthread": null, "properties_hs_email_message_id": "CAK4c3Gyf4xNPCtrON3BFLN9WOWUpfe+sfb+7wh5qYuCD-K71AA@mail.gmail.com", "properties_hs_email_migrated_via_portal_data_migration": null, "properties_hs_email_ms_teams_payload": null, "properties_hs_email_open_count": null, "properties_hs_email_pending_inline_image_ids": null, "properties_hs_email_post_send_status": "SENT", "properties_hs_email_recipient_drop_reasons": null, "properties_hs_email_reply_count": null, "properties_hs_email_send_event_id": null, "properties_hs_email_send_event_id_created": null, "properties_hs_email_sender_email": "integration-test-user@airbyte.io", "properties_hs_email_sender_firstname": null, "properties_hs_email_sender_lastname": null, "properties_hs_email_sender_raw": null, "properties_hs_email_sent_count": 1.0, "properties_hs_email_sent_via": "GMAIL", "properties_hs_email_status": "SENT", "properties_hs_email_stripped_attachment_count": null, "properties_hs_email_subject": "test deal ", "properties_hs_email_text": "test body\n-- \nPrefer fewer emails from me? Click here: https://d11qV604.na1.hs-salescrm-sub.com/preferences/en/manage?data=W2nXS-N30h-MkW3DX4xr38lXTKW2KXbZn3H3ZTKW4kt7Y_3XR2G0W30sn1g2zt_2NW47kvvy23ncKnW47Vmcy4pxy7cW41tzTm1X87X1W364bL-36tRLFW30J_Vy36F403W45FGpL3XHz-RW4ftDwZ4msYq_W24-jyc2HCSCvW3VGBr52TLG1vW2nFrmM3P2tStW43Skr81VxgJXW3z26wT4pc1KRW1Vpb_f3d3w7qW36dtk_4rCSHJW3F507n1_6v4MW2CWCvk49rVZpW23jtn51St_bDW2RKdYG2RNzKSW47znqq1_dHnNW4mGNp33Y1JRBW25m60s1Nk9WFW2MMKcf2F-zTNW4kddlH1NFHhxW25nrXX2KQX5rW3GJy1x2Yh7XsW2Pnx-93f_bXGW47SgSp1XqcMJW2FTQ1Z2KPBb6W32kvXr2KnzH9W3HcvHw3LRJmmW2MLX-W3LBLBJW3Q-74Q2KYV0CW1_9nCQ2r36_S0", "properties_hs_email_thread_id": "3b2bf39b9ed8cfc53310ee557627d073", "properties_hs_email_thread_summary": null, "properties_hs_email_to_email": "gl_serhii.lazebnyi@airbyte.io", "properties_hs_email_to_firstname": null, "properties_hs_email_to_lastname": null, "properties_hs_email_to_raw": "gl_serhii.lazebnyi@airbyte.io", "properties_hs_email_tracker_key": "87989bf6-7771-4486-b3d7-73a31af32b2c", "properties_hs_email_validation_skipped": null, "properties_hs_engagement_source": "EMAIL_INTEGRATION", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2024-02-05T01:13:26.539000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": "12282590", "properties_hs_object_id": 46838275228, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_owner_ids_bcc": null, "properties_hs_owner_ids_cc": null, "properties_hs_owner_ids_from": "52550153", "properties_hs_owner_ids_to": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_scs_association_status": null, "properties_hs_scs_audit_id": null, "properties_hs_timestamp": "2024-02-05T01:13:21.109000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": "432a7d905bf8fc42ba938819a9e6e291", "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2024-02-05T01:13:21.505000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708015555151} +{"stream": "engagements_meetings", "data": {"id": "46837884323", "properties": {"hs_activity_type": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_attendee_owner_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_contact_first_outreach_date": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2024-02-05T01:08:01.995000+00:00", "hs_engagement_source": "MEETINGS", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_guest_emails": null, "hs_i_cal_uid": "imqqv2eda8h5rh74gabnagl60o@google.com", "hs_include_description_in_reminder": null, "hs_internal_meeting_notes": null, "hs_lastmodifieddate": "2024-02-05T01:40:30.343000+00:00", "hs_meeting_body": null, "hs_meeting_calendar_event_hash": "7e8970ad5f400444979d9c979d5369b4", "hs_meeting_change_id": "7231dcd51227b02a05d158f5e7a602f3", "hs_meeting_created_from_link_id": "6678679", "hs_meeting_end_time": "2024-02-05T14:15:00+00:00", "hs_meeting_external_url": "https://www.google.com/calendar/event?eid=aW1xcXYyZWRhOGg1cmg3NGdhYm5hZ2w2MG8gaW50ZWdyYXRpb24tdGVzdC11c2VyQGFpcmJ5dGUuaW8", "hs_meeting_location": null, "hs_meeting_location_type": null, "hs_meeting_ms_teams_payload": null, "hs_meeting_outcome": "SCHEDULED", "hs_meeting_payments_session_id": null, "hs_meeting_pre_meeting_prospect_reminders": null, "hs_meeting_source": "MEETINGS_PUBLIC", "hs_meeting_source_id": "imqqv2eda8h5rh74gabnagl60o", "hs_meeting_start_time": "2024-02-05T14:00:00+00:00", "hs_meeting_title": "Test User and Team-1 Airbyte", "hs_meeting_web_conference_meeting_id": null, "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_object_id": 46837884323, "hs_object_source": "MEETINGS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "MEETINGS", "hs_object_source_user_id": 12282590, "hs_outcome_canceled_count": 0, "hs_outcome_completed_count": 0, "hs_outcome_no_show_count": 0, "hs_outcome_rescheduled_count": 0, "hs_outcome_scheduled_count": 1, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_roster_object_coordinates": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46837884323,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707139800000,\"uuid\":\"MEETING:7a71d47b-0a87-40c4-8e1a-a140184a29d0\"}]}", "hs_time_to_book_meeting_from_first_contact": 0, "hs_timestamp": "2024-02-05T14:00:00+00:00", "hs_timezone": "Europe/Warsaw", "hs_unique_creation_key": null, "hs_unique_id": "imqqv2eda8h5rh74gabnagl60o", "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2024-02-05T01:08:10.888000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2024-02-05T01:08:01.995Z", "updatedAt": "2024-02-05T01:40:30.343Z", "archived": false, "properties_hs_activity_type": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_attendee_owner_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_contact_first_outreach_date": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2024-02-05T01:08:01.995000+00:00", "properties_hs_engagement_source": "MEETINGS", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_guest_emails": null, "properties_hs_i_cal_uid": "imqqv2eda8h5rh74gabnagl60o@google.com", "properties_hs_include_description_in_reminder": null, "properties_hs_internal_meeting_notes": null, "properties_hs_lastmodifieddate": "2024-02-05T01:40:30.343000+00:00", "properties_hs_meeting_body": null, "properties_hs_meeting_calendar_event_hash": "7e8970ad5f400444979d9c979d5369b4", "properties_hs_meeting_change_id": "7231dcd51227b02a05d158f5e7a602f3", "properties_hs_meeting_created_from_link_id": "6678679", "properties_hs_meeting_end_time": "2024-02-05T14:15:00+00:00", "properties_hs_meeting_external_url": "https://www.google.com/calendar/event?eid=aW1xcXYyZWRhOGg1cmg3NGdhYm5hZ2w2MG8gaW50ZWdyYXRpb24tdGVzdC11c2VyQGFpcmJ5dGUuaW8", "properties_hs_meeting_location": null, "properties_hs_meeting_location_type": null, "properties_hs_meeting_ms_teams_payload": null, "properties_hs_meeting_outcome": "SCHEDULED", "properties_hs_meeting_payments_session_id": null, "properties_hs_meeting_pre_meeting_prospect_reminders": null, "properties_hs_meeting_source": "MEETINGS_PUBLIC", "properties_hs_meeting_source_id": "imqqv2eda8h5rh74gabnagl60o", "properties_hs_meeting_start_time": "2024-02-05T14:00:00+00:00", "properties_hs_meeting_title": "Test User and Team-1 Airbyte", "properties_hs_meeting_web_conference_meeting_id": null, "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_object_id": 46837884323, "properties_hs_object_source": "MEETINGS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "MEETINGS", "properties_hs_object_source_user_id": 12282590, "properties_hs_outcome_canceled_count": 0, "properties_hs_outcome_completed_count": 0, "properties_hs_outcome_no_show_count": 0, "properties_hs_outcome_rescheduled_count": 0, "properties_hs_outcome_scheduled_count": 1, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_roster_object_coordinates": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46837884323,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707139800000,\"uuid\":\"MEETING:7a71d47b-0a87-40c4-8e1a-a140184a29d0\"}]}", "properties_hs_time_to_book_meeting_from_first_contact": 0, "properties_hs_timestamp": "2024-02-05T14:00:00+00:00", "properties_hs_timezone": "Europe/Warsaw", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": "imqqv2eda8h5rh74gabnagl60o", "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2024-02-05T01:08:10.888000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708015722269} +{"stream": "engagements_meetings", "data": {"id": "46838182245", "properties": {"hs_activity_type": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_attendee_owner_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_contact_first_outreach_date": "2024-02-05T15:15:00+00:00", "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2024-02-05T01:08:32.416000+00:00", "hs_engagement_source": "MEETINGS", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_guest_emails": null, "hs_i_cal_uid": null, "hs_include_description_in_reminder": null, "hs_internal_meeting_notes": null, "hs_lastmodifieddate": "2024-02-05T01:08:37.402000+00:00", "hs_meeting_body": null, "hs_meeting_calendar_event_hash": "0b24520e196b77a0db079ab0357565e8", "hs_meeting_change_id": "9cc62faac2139a8ae373f992facb9504", "hs_meeting_created_from_link_id": "6678679", "hs_meeting_end_time": "2024-02-05T15:30:00+00:00", "hs_meeting_external_url": "https://www.google.com/calendar/event?eid=amg5N3RhcWppbjEzaGg4NDI0aXZoc3I0M2MgaW50ZWdyYXRpb24tdGVzdC11c2VyQGFpcmJ5dGUuaW8", "hs_meeting_location": null, "hs_meeting_location_type": null, "hs_meeting_ms_teams_payload": null, "hs_meeting_outcome": "SCHEDULED", "hs_meeting_payments_session_id": null, "hs_meeting_pre_meeting_prospect_reminders": null, "hs_meeting_source": "MEETINGS_PUBLIC", "hs_meeting_source_id": "jh97taqjin13hh8424ivhsr43c", "hs_meeting_start_time": "2024-02-05T15:15:00+00:00", "hs_meeting_title": "Test User and Team-1 Airbyte", "hs_meeting_web_conference_meeting_id": null, "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_object_id": 46838182245, "hs_object_source": "MEETINGS", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "MEETINGS", "hs_object_source_user_id": 12282590, "hs_outcome_canceled_count": 0, "hs_outcome_completed_count": 0, "hs_outcome_no_show_count": 0, "hs_outcome_rescheduled_count": 0, "hs_outcome_scheduled_count": 1, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_roster_object_coordinates": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46838182245,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707144300000,\"uuid\":\"MEETING:dc82686f-39ac-416f-ad15-4f2d706047c9\"}]}", "hs_time_to_book_meeting_from_first_contact": 0, "hs_timestamp": "2024-02-05T15:15:00+00:00", "hs_timezone": "Europe/Warsaw", "hs_unique_creation_key": null, "hs_unique_id": "jh97taqjin13hh8424ivhsr43c", "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2024-02-05T01:08:33.582000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2024-02-05T01:08:32.416Z", "updatedAt": "2024-02-05T01:08:37.402Z", "archived": false, "contacts": ["3201"], "properties_hs_activity_type": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_attendee_owner_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_contact_first_outreach_date": "2024-02-05T15:15:00+00:00", "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2024-02-05T01:08:32.416000+00:00", "properties_hs_engagement_source": "MEETINGS", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_guest_emails": null, "properties_hs_i_cal_uid": null, "properties_hs_include_description_in_reminder": null, "properties_hs_internal_meeting_notes": null, "properties_hs_lastmodifieddate": "2024-02-05T01:08:37.402000+00:00", "properties_hs_meeting_body": null, "properties_hs_meeting_calendar_event_hash": "0b24520e196b77a0db079ab0357565e8", "properties_hs_meeting_change_id": "9cc62faac2139a8ae373f992facb9504", "properties_hs_meeting_created_from_link_id": "6678679", "properties_hs_meeting_end_time": "2024-02-05T15:30:00+00:00", "properties_hs_meeting_external_url": "https://www.google.com/calendar/event?eid=amg5N3RhcWppbjEzaGg4NDI0aXZoc3I0M2MgaW50ZWdyYXRpb24tdGVzdC11c2VyQGFpcmJ5dGUuaW8", "properties_hs_meeting_location": null, "properties_hs_meeting_location_type": null, "properties_hs_meeting_ms_teams_payload": null, "properties_hs_meeting_outcome": "SCHEDULED", "properties_hs_meeting_payments_session_id": null, "properties_hs_meeting_pre_meeting_prospect_reminders": null, "properties_hs_meeting_source": "MEETINGS_PUBLIC", "properties_hs_meeting_source_id": "jh97taqjin13hh8424ivhsr43c", "properties_hs_meeting_start_time": "2024-02-05T15:15:00+00:00", "properties_hs_meeting_title": "Test User and Team-1 Airbyte", "properties_hs_meeting_web_conference_meeting_id": null, "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_object_id": 46838182245, "properties_hs_object_source": "MEETINGS", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "MEETINGS", "properties_hs_object_source_user_id": 12282590, "properties_hs_outcome_canceled_count": 0, "properties_hs_outcome_completed_count": 0, "properties_hs_outcome_no_show_count": 0, "properties_hs_outcome_rescheduled_count": 0, "properties_hs_outcome_scheduled_count": 1, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_roster_object_coordinates": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46838182245,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707144300000,\"uuid\":\"MEETING:dc82686f-39ac-416f-ad15-4f2d706047c9\"}]}", "properties_hs_time_to_book_meeting_from_first_contact": 0, "properties_hs_timestamp": "2024-02-05T15:15:00+00:00", "properties_hs_timezone": "Europe/Warsaw", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": "jh97taqjin13hh8424ivhsr43c", "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2024-02-05T01:08:33.582000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708015722270} +{"stream": "engagements_meetings", "data": {"id": "46838579861", "properties": {"hs_activity_type": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_attendee_owner_ids": null, "hs_body_preview": "attendee description", "hs_body_preview_html": "\n \n \n

      attendee description

      \n \n", "hs_body_preview_is_truncated": false, "hs_contact_first_outreach_date": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2024-02-05T01:15:53.269000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": "12282590", "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_guest_emails": null, "hs_i_cal_uid": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g@google.com", "hs_include_description_in_reminder": true, "hs_internal_meeting_notes": "

      test note

      ", "hs_lastmodifieddate": "2024-02-05T01:15:53.749000+00:00", "hs_meeting_body": "

      attendee description

      ", "hs_meeting_calendar_event_hash": null, "hs_meeting_change_id": null, "hs_meeting_created_from_link_id": null, "hs_meeting_end_time": "2024-02-06T10:45:00+00:00", "hs_meeting_external_url": "https://www.google.com/calendar/event?eid=YzRyamFkcG82OG8zY2JiM2NwajM0YjlrY2dzNjJiYjJja3JtOGI5bjY4cTM4ZDMzNnRobTZjMWg2ZyBpbnRlZ3JhdGlvbi10ZXN0LXVzZXJAYWlyYnl0ZS5pbw", "hs_meeting_location": "test address location", "hs_meeting_location_type": "ADDRESS", "hs_meeting_ms_teams_payload": null, "hs_meeting_outcome": "SCHEDULED", "hs_meeting_payments_session_id": null, "hs_meeting_pre_meeting_prospect_reminders": null, "hs_meeting_source": "BIDIRECTIONAL_API", "hs_meeting_source_id": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g", "hs_meeting_start_time": "2024-02-06T10:15:00+00:00", "hs_meeting_title": "test hubspot deal meeting ", "hs_meeting_web_conference_meeting_id": null, "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_object_id": 46838579861, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_outcome_canceled_count": 0, "hs_outcome_completed_count": 0, "hs_outcome_no_show_count": 0, "hs_outcome_rescheduled_count": 0, "hs_outcome_scheduled_count": 1, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_roster_object_coordinates": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46838579861,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707212700000,\"uuid\":\"MEETING:cfda6fc6-d8ae-4e46-971a-4c483a6aec5c\"}]}", "hs_time_to_book_meeting_from_first_contact": 0, "hs_timestamp": "2024-02-06T10:15:00+00:00", "hs_timezone": "Europe/Warsaw", "hs_unique_creation_key": null, "hs_unique_id": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g", "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2024-02-05T01:15:53.269000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2024-02-05T01:15:53.269Z", "updatedAt": "2024-02-05T01:15:53.749Z", "archived": false, "companies": ["5000526215"], "deals": ["5388306989"], "properties_hs_activity_type": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_attendee_owner_ids": null, "properties_hs_body_preview": "attendee description", "properties_hs_body_preview_html": "\n \n \n

      attendee description

      \n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_contact_first_outreach_date": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2024-02-05T01:15:53.269000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": "12282590", "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_guest_emails": null, "properties_hs_i_cal_uid": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g@google.com", "properties_hs_include_description_in_reminder": true, "properties_hs_internal_meeting_notes": "

      test note

      ", "properties_hs_lastmodifieddate": "2024-02-05T01:15:53.749000+00:00", "properties_hs_meeting_body": "

      attendee description

      ", "properties_hs_meeting_calendar_event_hash": null, "properties_hs_meeting_change_id": null, "properties_hs_meeting_created_from_link_id": null, "properties_hs_meeting_end_time": "2024-02-06T10:45:00+00:00", "properties_hs_meeting_external_url": "https://www.google.com/calendar/event?eid=YzRyamFkcG82OG8zY2JiM2NwajM0YjlrY2dzNjJiYjJja3JtOGI5bjY4cTM4ZDMzNnRobTZjMWg2ZyBpbnRlZ3JhdGlvbi10ZXN0LXVzZXJAYWlyYnl0ZS5pbw", "properties_hs_meeting_location": "test address location", "properties_hs_meeting_location_type": "ADDRESS", "properties_hs_meeting_ms_teams_payload": null, "properties_hs_meeting_outcome": "SCHEDULED", "properties_hs_meeting_payments_session_id": null, "properties_hs_meeting_pre_meeting_prospect_reminders": null, "properties_hs_meeting_source": "BIDIRECTIONAL_API", "properties_hs_meeting_source_id": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g", "properties_hs_meeting_start_time": "2024-02-06T10:15:00+00:00", "properties_hs_meeting_title": "test hubspot deal meeting ", "properties_hs_meeting_web_conference_meeting_id": null, "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_object_id": 46838579861, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_outcome_canceled_count": 0, "properties_hs_outcome_completed_count": 0, "properties_hs_outcome_no_show_count": 0, "properties_hs_outcome_rescheduled_count": 0, "properties_hs_outcome_scheduled_count": 1, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_roster_object_coordinates": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":46838579861,\"portalId\":8727216,\"engagementType\":\"MEETING\",\"taskType\":\"PRE_MEETING_NOTIFICATION\",\"timestamp\":1707212700000,\"uuid\":\"MEETING:cfda6fc6-d8ae-4e46-971a-4c483a6aec5c\"}]}", "properties_hs_time_to_book_meeting_from_first_contact": 0, "properties_hs_timestamp": "2024-02-06T10:15:00+00:00", "properties_hs_timezone": "Europe/Warsaw", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": "c4rjadpo68o3cbb3cpj34b9kcgs62bb2ckrm8b9n68q38d336thm6c1h6g", "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2024-02-05T01:15:53.269000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1708015722271} diff --git a/airbyte-integrations/connectors/source-hubspot/main.py b/airbyte-integrations/connectors/source-hubspot/main.py index 0f1cf13e4d10..dc073ca21ed6 100644 --- a/airbyte-integrations/connectors/source-hubspot/main.py +++ b/airbyte-integrations/connectors/source-hubspot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hubspot import SourceHubspot +from source_hubspot.run import run if __name__ == "__main__": - source = SourceHubspot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index cfb78174833b..38f03af7a246 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 2.0.0 + dockerImageTag: 3.2.0 dockerRepository: airbyte/source-hubspot documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot githubIssueLabel: source-hubspot icon: hubspot.svg license: ELv2 name: HubSpot + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hubspot registries: cloud: enabled: true @@ -29,7 +33,15 @@ data: message: >- This version eliminates the Property History stream in favor of creating 3 different streams, Contacts, Companies, and Deals, which can now all fetch their property history. It will affect only users who use Property History stream, who will need to fix schema conflicts and sync Contacts Property History stream instead of Property History. - upgradeDeadline: 2023-12-21 + upgradeDeadline: 2024-01-15 + 3.0.0: + message: >- + This update brings extended schema with data type changes for the Marketing Emails stream. + Users will need to refresh it and reset this stream after upgrading. + upgradeDeadline: 2024-02-12 + scopedImpact: + - scopeType: stream + impactedScopes: ["marketing_emails"] suggestedStreams: streams: - contacts diff --git a/airbyte-integrations/connectors/source-hubspot/poetry.lock b/airbyte-integrations/connectors/source-hubspot/poetry.lock new file mode 100644 index 000000000000..10e4cc8d33a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/poetry.lock @@ -0,0 +1,1050 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.61.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.61.2.tar.gz", hash = "sha256:494192d4e52bc30b88ae45d0161ce9ad351e7c4090fd5ec44649adb30323ad74"}, + {file = "airbyte_cdk-0.61.2-py3-none-any.whl", hash = "sha256:345382749991d628fd45d05c6045bde57f7ce493ace672ad6a93c0d1296fb735"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mock" +version = "5.1.0" +description = "Rolling backport of unittest.mock for all Pythons" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mock-5.1.0-py3-none-any.whl", hash = "sha256:18c694e5ae8a208cdb3d2c20a993ca1a7b0efa258c247a1e565150f477f83744"}, + {file = "mock-5.1.0.tar.gz", hash = "sha256:5e96aad5ccda4718e0a229ed94b2024df75cc2d55575ba5762d31f5767b8767d"}, +] + +[package.extras] +build = ["blurb", "twine", "wheel"] +docs = ["sphinx"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "c2ede1134d353ed454678bde83d4114935f614a63f4e086bb3df790798d9fb4e" diff --git a/airbyte-integrations/connectors/source-hubspot/pyproject.toml b/airbyte-integrations/connectors/source-hubspot/pyproject.toml new file mode 100644 index 000000000000..56e1120e9a28 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.1.1" +name = "source-hubspot" +description = "Source implementation for HubSpot." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/hubspot" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_hubspot" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.61.2" + +[tool.poetry.scripts] +source-hubspot = "source_hubspot.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +mock = "^5.1.0" +pytest-mock = "^3.6" +pytest = "^6.2" diff --git a/airbyte-integrations/connectors/source-hubspot/requirements.txt b/airbyte-integrations/connectors/source-hubspot/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json index 94d4a262b18f..fea5e895d162 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json @@ -353,6 +353,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts_form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py deleted file mode 100644 index 0f0721b230ff..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "mock", - "pytest~=6.2", - "pytest-mock~=3.6", - "requests-mock~=1.9.3", -] - -setup( - name="source_hubspot", - description="Source implementation for HubSpot.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py index e6cbfa152204..06abe8cf4bff 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py @@ -70,11 +70,6 @@ def __bool__(self): def as_url_param(self): """""" - @property - @abc.abstractmethod - def as_url_param_with_history(self) -> str: - """""" - @property @abc.abstractmethod def _term_representation(self): @@ -110,9 +105,6 @@ class APIv1Property(IURLPropertyRepresentation): def as_url_param(self): return {"property": self.properties} - def as_url_param_with_history(self) -> str: - return "&".join(map(lambda prop: f"propertiesWithHistory={prop}", self.properties)) - class APIv2Property(IURLPropertyRepresentation): _term_representation = "property={property}&" @@ -120,9 +112,6 @@ class APIv2Property(IURLPropertyRepresentation): def as_url_param(self): return {"property": self.properties} - def as_url_param_with_history(self) -> str: - return "&".join(map(lambda prop: f"propertiesWithHistory={prop}", self.properties)) - class APIv3Property(IURLPropertyRepresentation): _term_representation = "{property}," @@ -130,5 +119,13 @@ class APIv3Property(IURLPropertyRepresentation): def as_url_param(self): return {"properties": ",".join(self.properties)} - def as_url_param_with_history(self) -> str: - raise NotImplementedError("Not implemented") + +class APIPropertiesWithHistory(IURLPropertyRepresentation): + """ + It works for both v1 and v2 versions of API + """ + + _term_representation = "propertiesWithHistory={property}&" + + def as_url_param(self): + return "&".join(map(lambda prop: f"propertiesWithHistory={prop}", self.properties)) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py new file mode 100644 index 000000000000..26f4d0abef15 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubspot import SourceHubspot + + +def run(): + source = SourceHubspot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json index 17bab830c91a..01697a1022fa 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json @@ -39,6 +39,9 @@ "sourceMetadata": { "type": ["null", "string"] }, + "dataSensitivity": { + "type": ["null", "string"] + }, "companyId": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json index 4da808123cc1..a81f43a1b423 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json @@ -22,6 +22,12 @@ }, "lastSizeChangeAt": { "type": ["null", "integer"] + }, + "listReferencesCount": { + "type": ["null", "integer"] + }, + "parentFolderId": { + "type": ["null", "integer"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_form_submissions.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_form_submissions.json new file mode 100644 index 000000000000..21d1881eefc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_form_submissions.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "canonical-vid": { + "type": ["null", "integer"] + }, + "canonical-url": { + "type": ["null", "string"] + }, + "conversion-id": { + "type": ["null", "string"] + }, + "page-title": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "integer"] + }, + "form-id": { + "type": ["null", "string"] + }, + "portal-id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "page-url": { + "type": ["null", "string"] + }, + "form-type": { + "type": ["null", "string"] + }, + "contact-associated-by": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "meta-data": { + "type": ["null", "array"], + "items": {} + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json index f5f4dec16ae0..3c877a2ce929 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json @@ -3,6 +3,9 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { + "dataSensitivity": { + "type": ["null", "string"] + }, "updatedByUserId": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json index 43ac501b429f..afbd6540f1b2 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json @@ -109,6 +109,24 @@ "items": { "type": ["null", "integer"] } + }, + "contentIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "quoteIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "marketingEventIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } } } }, @@ -183,6 +201,69 @@ "body": { "type": ["null", "string"] }, + "meetingOutcome": { + "type": ["null", "string"] + }, + "calendarEventHash": { + "type": ["null", "string"] + }, + "createdFromLinkId": { + "type": ["null", "integer"] + }, + "meetingChangeId": { + "type": ["null", "string"] + }, + "locationType": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "internalMeetingNotes": { + "type": ["null", "string"] + }, + "includeDescriptionInReminder": { + "type": ["null", "boolean"] + }, + "iCalUid": { + "type": ["null", "string"] + }, + "preMeetingProspectReminders": { + "type": ["null", "array"], + "items": {} + }, + "attendeeOwnerIds": { + "type": ["null", "array"], + "items": {} + }, + "guestEmails": { + "type": ["null", "array"], + "items": {} + }, + "ownerIdsBcc": { + "type": ["null", "array"], + "items": {} + }, + "ownerIdsCc": { + "type": ["null", "array"], + "items": {} + }, + "ownerIdsFrom": { + "type": ["null", "array"], + "items": {} + }, + "ownerIdsTo": { + "type": ["null", "array"], + "items": {} + }, + "pendingInlineImageIds": { + "type": ["null", "array"], + "items": {} + }, + "validationSkipped": { + "type": ["null", "array"], + "items": {} + }, "from": { "type": ["null", "object"], "properties": { @@ -283,6 +364,9 @@ "title": { "type": ["null", "string"] }, + "timezone": { + "type": ["null", "string"] + }, "toNumber": { "type": ["null", "string"] }, @@ -328,6 +412,12 @@ "attachedVideoOpened": { "type": ["null", "boolean"] }, + "bounceErrorDetail": { + "type": ["null", "object"] + }, + "emailSendEventId": { + "type": ["null", "object"] + }, "attachedVideoWatched": { "type": ["null", "boolean"] }, @@ -384,6 +474,82 @@ } } }, + "metadata_attendeeOwnerIds": { + "type": ["null", "array"], + "items": {} + }, + "metadata_calendarEventHash": { + "type": ["null", "string"] + }, + "metadata_createdFromLinkId": { + "type": ["null", "integer"] + }, + "metadata_meetingChangeId": { + "type": ["null", "string"] + }, + "ownerIdsBcc": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "ownerIdsFrom": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "ownerIdsTo": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "pendingInlineImageIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "validationSkipped": { + "type": ["null", "array"], + "items": {} + }, + "ownerIdsCc": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "metadata_guestEmails": { + "type": ["null", "array"], + "items": {} + }, + "metadata_iCalUid": { + "type": ["null", "string"] + }, + "metadata_includeDescriptionInReminder": { + "type": ["null", "boolean"] + }, + "metadata_internalMeetingNotes": { + "type": ["null", "string"] + }, + "metadata_location": { + "type": ["null", "string"] + }, + "metadata_locationType": { + "type": ["null", "string"] + }, + "metadata_meetingOutcome": { + "type": ["null", "string"] + }, + "metadata_timezone": { + "type": ["null", "string"] + }, + "metadata_preMeetingProspectReminders": { + "type": ["null", "array"], + "items": {} + }, "metadata_body": { "type": ["null", "string"] }, @@ -452,6 +618,36 @@ } } }, + "metadata_bounceErrorDetail": { + "type": ["null", "object"] + }, + "metadata_emailSendEventId": { + "type": ["null", "object"] + }, + "metadata_ownerIdsBcc": { + "type": ["null", "array"], + "items": {} + }, + "metadata_ownerIdsCc": { + "type": ["null", "array"], + "items": {} + }, + "metadata_ownerIdsFrom": { + "type": ["null", "array"], + "items": {} + }, + "metadata_ownerIdsTo": { + "type": ["null", "array"], + "items": {} + }, + "metadata_pendingInlineImageIds": { + "type": ["null", "array"], + "items": {} + }, + "metadata_validationSkipped": { + "type": ["null", "array"], + "items": {} + }, "metadata_bcc": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index c1f20f05459f..48e77f1ef8ed 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -52,6 +52,20 @@ }, "fieldType": { "type": ["null", "string"] + }, + "validation": { + "type": ["null", "object"], + "properties": { + "blockedEmailDomains": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "useDefaultBlockList": { + "type": ["null", "boolean"] + } + } } } } @@ -169,6 +183,9 @@ "type": ["null", "string"] } } + }, + "cssClass": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json index aa15243a3daf..d4f888288aa8 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json @@ -33,7 +33,7 @@ "type": ["null", "string"] }, "aifeatures": { - "type": ["null", "string"] + "type": ["null", "object"] }, "allEmailCampaignIds": { "type": ["null", "array"], @@ -215,6 +215,9 @@ "resolvedDomain": { "type": ["null", "string"] }, + "rootMicId": { + "type": ["null", "string"] + }, "selected": { "type": ["null", "integer"] }, @@ -233,6 +236,9 @@ "counters": { "type": ["null", "object"], "properties": { + "sent": { + "type": ["null", "integer"] + }, "open": { "type": ["null", "integer"] }, @@ -248,6 +254,9 @@ "click": { "type": ["null", "integer"] }, + "reply": { + "type": ["null", "integer"] + }, "dropped": { "type": ["null", "integer"] }, @@ -313,6 +322,9 @@ "failedToLoad": { "type": ["null", "boolean"] }, + "qualifierStats": { + "type": ["null", "object"] + }, "ratios": { "type": ["null", "object"], "properties": { @@ -328,6 +340,9 @@ "openratio": { "type": ["null", "number"] }, + "replyratio": { + "type": ["null", "number"] + }, "unsubscribedratio": { "type": ["null", "number"] }, @@ -455,164 +470,61 @@ "flexAreas": { "type": ["null", "object"], "properties": { - "properties": { - "main": { - "type": ["null", "object"], - "properties": { - "boxed": { - "type": ["null", "boolean"] - }, - "isSingleColumnFullWidth": { - "type": ["null", "boolean"] - }, - "sections": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] - }, - "id": { - "type": ["null", "string"] - }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundColor": { - "type": ["null", "string"] - }, - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { + "main": { + "type": ["null", "object"], + "properties": { + "boxed": { + "type": ["null", "boolean"] + }, + "isSingleColumnFullWidth": { + "type": ["null", "boolean"] + }, + "sections": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "columns": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "widgets": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "width": { + "type": ["null", "integer"] } } } }, - { + "id": { + "type": ["null", "string"] + }, + "style": { "type": ["null", "object"], "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] - }, - "id": { + "backgroundColor": { "type": ["null", "string"] }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { - "type": ["null", "string"] - } - } - } - } - }, - { - "type": ["null", "object"], - "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] + "backgroundType": { + "type": ["null", "string"] }, - "id": { + "paddingBottom": { "type": ["null", "string"] }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundColor": { - "type": ["null", "string"] - }, - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { - "type": ["null", "string"] - } - } + "paddingTop": { + "type": ["null", "string"] } } } - ] + } } } } @@ -659,14 +571,96 @@ "email_body_padding": { "type": ["null", "string"] }, "email_body_width": { "type": ["null", "string"] }, "heading_one_font": { + "type": ["null", "object"], + "properties": { + "bold": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "font": { "type": ["null", "string"] }, + "font_style": { "type": ["null", "object"] }, + "italic": { "type": ["null", "boolean"] }, + "size": { "type": ["null", "string"] }, + "underline": { "type": ["null", "boolean"] } + } + }, + "heading_two_font": { + "type": ["null", "object"], "properties": { - "bold": { "type": ["null", "string"] }, + "bold": { "type": ["null", "boolean"] }, "color": { "type": ["null", "string"] }, "font": { "type": ["null", "string"] }, "font_style": { "type": ["null", "object"] }, - "italic": { "type": ["null", "string"] }, + "italic": { "type": ["null", "boolean"] }, "size": { "type": ["null", "string"] }, - "underline": { "type": ["null", "string"] } + "underline": { "type": ["null", "boolean"] } + } + }, + "links_font": { + "type": ["null", "object"], + "properties": { + "bold": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "font": { "type": ["null", "string"] }, + "font_style": { "type": ["null", "object"] }, + "italic": { "type": ["null", "boolean"] }, + "size": { "type": ["null", "string"] }, + "underline": { "type": ["null", "boolean"] } + } + }, + "primary_accent_color": { + "type": ["null", "string"] + }, + "primary_font": { + "type": ["null", "string"] + }, + "primary_font_color": { + "type": ["null", "string"] + }, + "primary_font_line_height": { + "type": ["null", "string"] + }, + "primary_font_size": { + "type": ["null", "string"] + }, + "secondary_accent_color": { + "type": ["null", "string"] + }, + "secondary_font": { + "type": ["null", "string"] + }, + "secondary_font_color": { + "type": ["null", "string"] + }, + "secondary_font_line_height": { + "type": ["null", "string"] + }, + "secondary_font_size": { + "type": ["null", "string"] + }, + "use_email_client_default_settings": { + "type": ["null", "boolean"] + }, + "user_module_defaults": { + "type": ["null", "object"], + "properties": { + "button_email": { + "type": ["null", "object"], + "properties": { + "background_color": { "type": ["null", "string"] }, + "corner_radius": { "type": ["null", "integer"] }, + "font": { "type": ["null", "string"] }, + "font_color": { "type": ["null", "string"] }, + "font_size": { "type": ["null", "integer"] }, + "font_style": { "type": ["null", "object"] } + } + }, + "email_divider": { + "type": ["null", "object"], + "properties": { + "color": { "type": ["null", "object"] }, + "height": { "type": ["null", "integer"] }, + "line_type": { "type": ["null", "string"] } + } + } } } } @@ -706,6 +700,18 @@ }, "rssEmailEntryTemplateEnabled": { "type": ["null", "boolean"] + }, + "mailingIlsListsExcluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailingIlsListsIncluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json index 569707354cbc..75ab2aa9321e 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json @@ -27,6 +27,12 @@ "properties_hs_device_type": { "type": ["null", "string"] }, + "properties_hs_title": { + "type": ["null", "string"] + }, + "properties_hs_form_correlation_id": { + "type": ["null", "string"] + }, "properties_hs_element_class": { "type": ["null", "string"] }, @@ -39,6 +45,15 @@ "properties_hs_language": { "type": ["null", "string"] }, + "properties_hs_document_id": { + "type": ["null", "string"] + }, + "properties_hs_presentation_id": { + "type": ["null", "string"] + }, + "properties_hs_user_id": { + "type": ["null", "string"] + }, "properties_hs_link_href": { "type": ["null", "string"] }, @@ -69,6 +84,9 @@ "properties_hs_region": { "type": ["null", "string"] }, + "properties_hs_url": { + "type": ["null", "string"] + }, "properties_hs_screen_height": { "type": ["null", "string"] }, @@ -102,6 +120,24 @@ "id": { "type": ["null", "string"] }, + "properties_hs_base_url": { + "type": ["null", "string"] + }, + "properties_hs_form_id": { + "type": ["null", "string"] + }, + "properties_hs_form_type": { + "type": ["null", "string"] + }, + "properties_hs_url_domain": { + "type": ["null", "string"] + }, + "properties_hs_url_path": { + "type": ["null", "string"] + }, + "properties_hs_visitor_type": { + "type": ["null", "string"] + }, "objectId": { "type": ["null", "string"] }, @@ -112,7 +148,8 @@ "type": ["null", "string"] }, "occurredAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/ticket_pipelines.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/ticket_pipelines.json index 5d75223f3153..342d303f23d7 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/ticket_pipelines.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/ticket_pipelines.json @@ -5,6 +5,9 @@ "label": { "type": ["null", "string"] }, + "displayOrder": { + "type": ["null", "integer"] + }, "id": { "type": ["null", "string"] }, @@ -46,6 +49,12 @@ }, "active": { "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "writePermissions": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json index 228121336cd7..2c40a25a4630 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json @@ -40,6 +40,12 @@ "items": { "type": ["null", "string"] } + }, + "completed": { + "type": ["null", "integer"] + }, + "succeeded": { + "type": ["null", "integer"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 97d42860a344..dd643ff170b2 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -21,6 +21,7 @@ CompaniesWebAnalytics, ContactLists, Contacts, + ContactsFormSubmissions, ContactsListMemberships, ContactsMergedAudit, ContactsPropertyHistory, @@ -117,6 +118,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Companies(**common_params), ContactLists(**common_params), Contacts(**common_params), + ContactsFormSubmissions(**common_params), ContactsListMemberships(**common_params), ContactsMergedAudit(**common_params), DealPipelines(**common_params), diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 4f9695034a4e..3720bb68daae 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -32,6 +32,7 @@ from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout, InvalidStartDateConfigError from source_hubspot.helpers import ( + APIPropertiesWithHistory, APIv1Property, APIv2Property, APIv3Property, @@ -1347,16 +1348,7 @@ class ContactLists(IncrementalStream): unnest_fields = ["metaData"] -class ContactsListMemberships(Stream): - """Contacts list Memberships, API v1 - The Stream was created due to issue #8477, where supporting List Memberships in Contacts stream was requested. - According to the issue this feature is supported in API v1 by setting parameter showListMemberships=true - in get all contacts endpoint. API will return list memberships for each contact record. - But for syncing Contacts API v3 is used, where list memberships for contacts isn't supported. - Therefore, new stream was created based on get all contacts endpoint of API V1. - Docs: https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts - """ - +class ContactsAllBase(Stream): url = "/contacts/v1/lists/all/contacts/all" updated_at_field = "timestamp" more_key = "has-more" @@ -1366,16 +1358,14 @@ class ContactsListMemberships(Stream): primary_key = "canonical-vid" scopes = {"crm.objects.contacts.read"} properties_scopes = {"crm.schemas.contacts.read"} + records_field = None + filter_field = None + filter_value = None def _transform(self, records: Iterable) -> Iterable: - """Extracting list membership records from contacts - According to documentation Contacts may have multiple vids, - but the canonical-vid will be the primary ID for a record. - Docs: https://legacydocs.hubspot.com/docs/methods/contacts/contacts-overview - """ for record in super()._transform(records): canonical_vid = record.get("canonical-vid") - for item in record.get("list-memberships", []): + for item in record.get(self.records_field, []): yield {"canonical-vid": canonical_vid, **item} def request_params( @@ -1385,10 +1375,33 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params.update({"showListMemberships": True}) + if self.filter_field and self.filter_value: + params.update({self.filter_field: self.filter_value}) return params +class ContactsListMemberships(ContactsAllBase, ABC): + """Contacts list Memberships, API v1 + The Stream was created due to issue #8477, where supporting List Memberships in Contacts stream was requested. + According to the issue this feature is supported in API v1 by setting parameter showListMemberships=true + in get all contacts endpoint. API will return list memberships for each contact record. + But for syncing Contacts API v3 is used, where list memberships for contacts isn't supported. + Therefore, new stream was created based on get all contacts endpoint of API V1. + Docs: https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts + """ + + records_field = "list-memberships" + filter_field = "showListMemberships" + filter_value = True + + +class ContactsFormSubmissions(ContactsAllBase, ABC): + + records_field = "form-submissions" + filter_field = "formSubmissionMode" + filter_value = "all" + + class Deals(CRMSearchStream): """Deals, API v3""" @@ -1833,6 +1846,11 @@ def entity(self) -> str: def primary_key(self) -> str: """Indicates a field name which is considered to be a primary key of the stream""" + @property + @abstractmethod + def entity_primary_key(self) -> str: + """Indicates a field name which is considered to be a primary key of the parent entity""" + @property @abstractmethod def additional_keys(self) -> list: @@ -1872,7 +1890,7 @@ def request_params( def _transform(self, records: Iterable) -> Iterable: for record in records: properties = record.get("properties") - primary_key = record.get(self.primary_key) + primary_key = record.get(self.entity_primary_key) additional_keys = {additional_key: record.get(additional_key) for additional_key in self.additional_keys} value_dict: Dict for property_name, value_dict in properties.items(): @@ -1887,7 +1905,7 @@ def _transform(self, records: Iterable) -> Iterable: if versions: for version in versions: version["property"] = property_name - version[self.primary_key] = primary_key + version[self.entity_primary_key] = primary_key yield version | additional_keys @@ -1921,9 +1939,13 @@ def entity(self): return "contacts" @property - def primary_key(self) -> list: + def entity_primary_key(self) -> list: return "vid" + @property + def primary_key(self) -> list: + return ["vid", "property", "timestamp"] + @property def additional_keys(self) -> list: return ["portal-id", "is-contact", "canonical-vid"] @@ -1942,6 +1964,11 @@ def url(self): class CompaniesPropertyHistory(PropertyHistory): + @cached_property + def _property_wrapper(self) -> IURLPropertyRepresentation: + properties = list(self.properties.keys()) + return APIPropertiesWithHistory(properties=properties) + @property def scopes(self) -> set: return {"crm.objects.companies.read"} @@ -1964,16 +1991,20 @@ def page_filter(self) -> str: @property def more_key(self) -> str: - return "hasMore" + return "has-more" @property def entity(self) -> str: return "companies" @property - def primary_key(self) -> list: + def entity_primary_key(self) -> list: return "companyId" + @property + def primary_key(self) -> list: + return ["companyId", "property", "timestamp"] + @property def additional_keys(self) -> list: return ["portalId", "isDeleted"] @@ -2001,10 +2032,15 @@ def path( next_page_token: Mapping[str, Any] = None, properties: IURLPropertyRepresentation = None, ) -> str: - return f"{self.url}?{properties.as_url_param_with_history()}" + return f"{self.url}?{properties.as_url_param()}" class DealsPropertyHistory(PropertyHistory): + @cached_property + def _property_wrapper(self) -> IURLPropertyRepresentation: + properties = list(self.properties.keys()) + return APIPropertiesWithHistory(properties=properties) + @property def scopes(self) -> set: return {"crm.objects.deals.read"} @@ -2034,9 +2070,13 @@ def entity(self) -> set: return "deals" @property - def primary_key(self) -> list: + def entity_primary_key(self) -> list: return "dealId" + @property + def primary_key(self) -> list: + return ["dealId", "property", "timestamp"] + @property def additional_keys(self) -> list: return ["portalId", "isDeleted"] @@ -2064,7 +2104,7 @@ def path( next_page_token: Mapping[str, Any] = None, properties: IURLPropertyRepresentation = None, ) -> str: - return f"{self.url}?{properties.as_url_param_with_history()}" + return f"{self.url}?{properties.as_url_param()}" class SubscriptionChanges(IncrementalStream): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index cd6e6fa60386..de5e3c40a44a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -93,7 +93,7 @@ def test_streams(requests_mock, config_experimental): streams = SourceHubspot().streams(config_experimental) - assert len(streams) == 44 + assert len(streams) == 45 def test_custom_streams(config_experimental): @@ -135,18 +135,18 @@ def test_convert_datetime_to_string(): def test_cast_datetime(common_params, caplog): field_value = pendulum.now() - field_name = "curent_time" + field_name = "current_time" Companies(**common_params)._cast_datetime(field_name, field_value) - expected_warining_message = { + expected_warning_message = { "type": "LOG", "log": { "level": "WARN", "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: argument of type 'DateTime' is not iterable", }, } - assert expected_warining_message["log"]["message"] in caplog.text + assert expected_warning_message["log"]["message"] in caplog.text def test_check_connection_backoff_on_limit_reached(requests_mock, config): diff --git a/airbyte-integrations/connectors/source-insightly/main.py b/airbyte-integrations/connectors/source-insightly/main.py index 5797ba54c977..a486e61080cb 100644 --- a/airbyte-integrations/connectors/source-insightly/main.py +++ b/airbyte-integrations/connectors/source-insightly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_insightly import SourceInsightly +from source_insightly.run import run if __name__ == "__main__": - source = SourceInsightly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 321d751ffa7a..7fb79460cecd 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-insightly registries: oss: enabled: true @@ -20,7 +24,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/insightly tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index a3c070098791..40a7bc03b71c 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-insightly=source_insightly.run:run", + ], + }, name="source_insightly", description="Source implementation for Insightly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/run.py b/airbyte-integrations/connectors/source-insightly/source_insightly/run.py new file mode 100644 index 000000000000..d3b9124ae551 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_insightly import SourceInsightly + + +def run(): + source = SourceInsightly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-instagram/README.md b/airbyte-integrations/connectors/source-instagram/README.md index 8cfb12455f66..6d7485e922a0 100644 --- a/airbyte-integrations/connectors/source-instagram/README.md +++ b/airbyte-integrations/connectors/source-instagram/README.md @@ -1,68 +1,55 @@ -# Instagram Source +# Instagram source connector + This is the repository for the Instagram source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/instagram). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/instagram). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/instagram) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_instagram/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/instagram) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_instagram/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source instagram test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-instagram spec +poetry run source-instagram check --config secrets/config.json +poetry run source-instagram discover --config secrets/config.json +poetry run source-instagram read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -airbyte-ci connectors --name source-instagram build +airbyte-ci connectors --name=source-instagram build ``` -An image will be built with the tag `airbyte/source-instagram:dev`. +An image will be available on your host with the tag `airbyte/source-instagram:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-instagram:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-instagram:dev spec @@ -71,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instagram:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-instagram:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-instagram test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-instagram test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/instagram.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/instagram.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml index f70680a35b17..cbb6e96e83a4 100644 --- a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml @@ -50,6 +50,8 @@ acceptance_tests: - name: online_followers bypass_reason: Depend on each online user media: + - name: like_count + bypass_reason: Auto updated field - name: media_url bypass_reason: Contains auto generated hash - name: thumbnail_url diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index d90034355c7a..4d4baa753a0e 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -2,6 +2,6 @@ {"stream": "media", "data": {"id": "17884386203808767", "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "ig_id": "3123724930722523505", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfBPpWnNa8TFbux-TpRO48bJGSkaIKPFOnmXhcv39jLd_A&oe=6559369A&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CtZs0Y3v2lx", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=1ZTHPkRhzl8AX-hZcw_&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBdTKQTru0U2JNSqNnuPN0cWYv1u6o6t6u3EHIFteUV7w&oe=655C7D4E", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "timestamp": "2023-06-12T19:20:02+00:00", "like_count": 9, "comments_count": 2, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757119} {"stream": "media", "data": {"id": "17864256500936159", "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "ig_id": "3106359072491902976", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfC6GeTJWR8KJZ3-eb1-faBZ8P8G8AFyswEDdD4gFzmPMg&oe=65594B26&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CscAR5EsRgA", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=jLyY4sWj0v0AX-iadbF&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfA-x6QyIXxT7o_lEwDH0k7tDb_bgCGeP61AseCpluCtPA&oe=655D3C59", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "timestamp": "2023-05-19T20:08:33+00:00", "like_count": 7, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} {"stream": "media", "data": {"id": "17964324206288599", "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "ig_id": "3104241732634871967", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfACHaQfoSJ_vMXbm4Xw3gmWnG_vnJgUsIYUePDdtIUS-w&oe=6558DBB2&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CsUe2iqpQif", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=Y6VzeGH_9lkAX_wkzpd&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDil0e2W7Iqq0-d7rf9JkdOluS7U2C3nhK17EfQ3c07fw&oe=655D28FC", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "timestamp": "2023-05-16T22:01:45+00:00", "like_count": 13, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-17T08:00:00+00:00", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Atlanta, Georgia": 4, "Algiers, Algiers Province": 4, "Caracas, Capital District": 4, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 8, "Kolkata, West Bengal": 5, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 16, "Dili, Timor-Leste": 3, "Ahmedabad, Gujarat": 4, "Skopje, Municipality of Centar (Skopje)": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Gurugram, Haryana": 6, "Delhi, Delhi": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 4, "Istanbul, Istanbul Province": 9, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 7, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 7, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 14, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1700230802791} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-17T08:00:00+00:00", "value": {"DE": 31, "HK": 4, "TW": 5, "FI": 5, "RU": 9, "TZ": 8, "FR": 10, "SA": 8, "BR": 64, "SE": 6, "SG": 6, "MA": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 24, "MY": 9, "CO": 11, "ES": 13, "VE": 9, "AR": 23, "AT": 4, "TH": 7, "AU": 35, "PE": 4, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1700230802792} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-17T08:00:00+00:00", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 50, "M.25-34": 365, "M.35-44": 228, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 42, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1700230802792} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"city","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481116} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"country","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481343} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"age,gender","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481574} diff --git a/airbyte-integrations/connectors/source-instagram/main.py b/airbyte-integrations/connectors/source-instagram/main.py index 7dfe30785519..0a871930a015 100644 --- a/airbyte-integrations/connectors/source-instagram/main.py +++ b/airbyte-integrations/connectors/source-instagram/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_instagram import SourceInstagram +from source_instagram.run import run if __name__ == "__main__": - source = SourceInstagram() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 5e690acafaaa..a946127344a9 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,12 +7,16 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 2.0.0 + dockerImageTag: 3.0.3 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg license: MIT name: Instagram + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-instagram registries: cloud: enabled: true @@ -21,6 +25,13 @@ data: releaseStage: generally_available releases: breakingChanges: + 3.0.0: + message: "The existing Instagram API (v11) has been deprecated. Customers who use streams `Media Insights`, `Story Insights` or `User Lifetime Insights` must take action with their connections. Please follow the to update to the latest Instagram API (v18). For more details, see our migration guide." + upgradeDeadline: "2024-01-05" + scopedImpact: + - scopeType: stream + impactedScopes: + ["media_insights", "story_insights", "user_lifetime_insights"] 2.0.0: message: This release introduces a default primary key for the streams UserLifetimeInsights and UserInsights. diff --git a/airbyte-integrations/connectors/source-instagram/poetry.lock b/airbyte-integrations/connectors/source-instagram/poetry.lock new file mode 100644 index 000000000000..82e6349aba76 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/poetry.lock @@ -0,0 +1,1496 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cached-property" +version = "1.5.2" +description = "A decorator for caching properties in classes." +optional = false +python-versions = "*" +files = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "curlify" +version = "2.2.1" +description = "Library to convert python requests object to curl command." +optional = false +python-versions = "*" +files = [ + {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, +] + +[package.dependencies] +requests = "*" + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "facebook-business" +version = "18.0.5" +description = "Facebook Business SDK" +optional = false +python-versions = "*" +files = [ + {file = "facebook_business-18.0.5-py3-none-any.whl", hash = "sha256:5b84043a9b67ea31ef2ccb95254c883d65f6f4a585bf077bfc2decb4896ed615"}, + {file = "facebook_business-18.0.5.tar.gz", hash = "sha256:7435860e1787b6d0c6fb29b43d8c2d07f503a53c8ef77202232bea4b68a33992"}, +] + +[package.dependencies] +aiohttp = {version = "*", markers = "python_version >= \"3.5.3\""} +curlify = ">=2.1.0" +pycountry = ">=19.8.18" +requests = ">=2.3.0" +six = ">=1.7.3" + +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pycountry" +version = "23.12.11" +description = "ISO country, subdivision, language, currency and script definitions and their translations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycountry-23.12.11-py3-none-any.whl", hash = "sha256:2ff91cff4f40ff61086e773d61e72005fe95de4a57bfc765509db05695dc50ab"}, + {file = "pycountry-23.12.11.tar.gz", hash = "sha256:00569d82eaefbc6a490a311bfa84a9c571cff9ddbf8b0a4f4e7b4f868b4ad925"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "0ca461e187737a477daf3b3609b25d12853d6e7bed738f2104d0ed027bc49ca8" diff --git a/airbyte-integrations/connectors/source-instagram/pyproject.toml b/airbyte-integrations/connectors/source-instagram/pyproject.toml new file mode 100644 index 000000000000..481ce4c340e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.0.3" +name = "source-instagram" +description = "Source implementation for Instagram." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/instagram" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_instagram" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +facebook-business = "==18.0.5" +airbyte-cdk = "==0.58.8" +cached-property = "==1.5.2" + +[tool.poetry.scripts] +source-instagram = "source_instagram.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-instagram/requirements.txt b/airbyte-integrations/connectors/source-instagram/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-instagram/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-instagram/setup.py b/airbyte-integrations/connectors/source-instagram/setup.py deleted file mode 100644 index a15d5f7ab25e..000000000000 --- a/airbyte-integrations/connectors/source-instagram/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "cached_property~=1.5", - "facebook_business~=11.0", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", -] - -setup( - name="source_instagram", - description="Source implementation for Instagram.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index 293876dd705e..16426efc1f9a 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -65,7 +65,6 @@ def call( class InstagramAPI: def __init__(self, access_token: str): - self._api = FacebookAdsApi.init(access_token=access_token) # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) FacebookAdsApi.set_default_api(self.api) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/run.py b/airbyte-integrations/connectors/source-instagram/source_instagram/run.py new file mode 100644 index 000000000000..c012b2e2292a --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_instagram import SourceInstagram + + +def run(): + source = SourceInstagram() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json index 8a1759549e83..63aa03b6efcc 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json @@ -10,8 +10,11 @@ "id": { "type": ["null", "string"] }, - "engagement": { - "type": ["null", "integer"] + "ig_reels_avg_watch_time": { + "type": ["null", "number"] + }, + "ig_reels_video_view_total_time": { + "type": ["null", "number"] }, "impressions": { "type": ["null", "integer"] @@ -25,18 +28,6 @@ "video_views": { "type": ["null", "integer"] }, - "carousel_album_engagement": { - "type": ["null", "integer"] - }, - "carousel_album_impressions": { - "type": ["null", "integer"] - }, - "carousel_album_reach": { - "type": ["null", "integer"] - }, - "carousel_album_saved": { - "type": ["null", "integer"] - }, "comments": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json index 81513dcd8246..cf81cd498060 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json @@ -10,9 +10,6 @@ "id": { "type": ["null", "string"] }, - "exits": { - "type": ["null", "integer"] - }, "impressions": { "type": ["null", "integer"] }, @@ -21,12 +18,6 @@ }, "replies": { "type": ["null", "integer"] - }, - "taps_forward": { - "type": ["null", "integer"] - }, - "taps_back": { - "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json index 4cb5092f5ace..40265de413f6 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json @@ -7,16 +7,14 @@ "business_account_id": { "type": ["null", "string"] }, - "date": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" + "breakdown": { + "type": ["null", "string"] }, "metric": { "type": ["null", "string"] }, "value": { - "type": ["integer", "object"] + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index bf5d39de1e1c..4e6d27c4fb2c 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -143,10 +143,18 @@ def read_records( class UserLifetimeInsights(DatetimeTransformerMixin, InstagramStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" - primary_key = ["business_account_id", "metric", "date"] - LIFETIME_METRICS = ["audience_city", "audience_country", "audience_gender_age", "audience_locale"] + primary_key = ["business_account_id", "breakdown"] + BREAKDOWNS = ["city", "country", "age,gender"] + BASE_METRIC = ["follower_demographics"] period = "lifetime" + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for slice in super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): + for breakdown in self.BREAKDOWNS: + yield slice | {"breakdown": breakdown} + def read_records( self, sync_mode: SyncMode, @@ -156,13 +164,14 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: account = stream_slice["account"] ig_account = account["instagram_business_account"] - for insight in ig_account.get_insights(params=self.request_params()): + for insight in ig_account.get_insights(params=self.request_params(stream_slice=stream_slice)): + insight_data = insight.export_all_data() yield { "page_id": account["page_id"], + "breakdown": stream_slice["breakdown"], "business_account_id": ig_account.get("id"), "metric": insight["name"], - "date": insight["values"][0].get("end_time"), - "value": insight["values"][0].get("value"), + "value": self._transform_breakdown_results(insight_data["total_value"]["breakdowns"][0]["results"]), } def request_params( @@ -171,9 +180,15 @@ def request_params( stream_state: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_slice=stream_slice, stream_state=stream_state) - params.update({"metric": self.LIFETIME_METRICS, "period": self.period}) + params.update( + {"metric": self.BASE_METRIC, "metric_type": "total_value", "period": self.period, "breakdown": stream_slice["breakdown"]} + ) return params + @staticmethod + def _transform_breakdown_results(breakdown_results: Iterable[Mapping[str, Any]]) -> Mapping[str, Any]: + return {res.get("dimension_values")[0]: res.get("value") for res in breakdown_results} + class UserInsights(DatetimeTransformerMixin, InstagramIncrementalStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" @@ -316,7 +331,7 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: class Media(DatetimeTransformerMixin, InstagramStream): """Children objects can only be of the media_type == "CAROUSEL_ALBUM". - And children object does not support INVALID_CHILDREN_FIELDS fields, + And children objects do not support INVALID_CHILDREN_FIELDS fields, so they are excluded when trying to get child objects to avoid the error """ @@ -358,9 +373,20 @@ def _get_children(self, ids: List): class MediaInsights(Media): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - MEDIA_METRICS = ["engagement", "impressions", "reach", "saved"] - CAROUSEL_ALBUM_METRICS = ["carousel_album_engagement", "carousel_album_impressions", "carousel_album_reach", "carousel_album_saved"] - REELS_METRICS = ["comments", "likes", "reach", "saved", "shares", "total_interactions", "plays"] + MEDIA_METRICS = ["total_interactions", "impressions", "reach", "saved", "video_views", "likes", "comments", "shares"] + CAROUSEL_ALBUM_METRICS = ["total_interactions", "impressions", "reach", "saved", "video_views"] + + REELS_METRICS = [ + "comments", + "ig_reels_avg_watch_time", + "ig_reels_video_view_total_time", + "likes", + "plays", + "reach", + "saved", + "shares", + "total_interactions", + ] def read_records( self, @@ -387,6 +413,8 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: """Get insights for specific media""" if item.get("media_product_type") == "REELS": metrics = self.REELS_METRICS + elif item.get("media_type") == "VIDEO" and item.get("media_product_type") == "FEED": + metrics = ["impressions", "reach", "saved", "video_views", "video_views"] elif item.get("media_type") == "VIDEO": metrics = self.MEDIA_METRICS + ["video_views"] elif item.get("media_type") == "CAROUSEL_ALBUM": @@ -445,7 +473,7 @@ def read_records( class StoryInsights(Stories): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - metrics = ["exits", "impressions", "reach", "replies", "taps_forward", "taps_back"] + metrics = ["impressions", "reach", "replies"] def read_records( self, diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index a065d01b77cf..44be2de8ca2c 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -54,13 +54,16 @@ def fb_account_response_fixture(account_id, some_config, requests_mock): "json": { "data": [ { - "account_id": account_id, + "access_token": "access_token", + "category": "Software company", "id": f"act_{account_id}", - } - ], - "paging": {"cursors": {"before": "MjM4NDYzMDYyMTcyNTAwNzEZD", "after": "MjM4NDYzMDYyMTcyNTAwNzEZD"}}, - }, - "status_code": 200, + "paging": {"cursors": { + "before": "cursor", + "after": "cursor"}}, + "summary": {"total_count": 1}, + "status_code": 200 + }] + } } @@ -99,6 +102,22 @@ def user_insight_data_fixture(): } +@fixture(name="user_lifetime_insight_data") +def user_lifetime_insight_data_fixture(): + return { + "name": "impressions", + "period": "day", + "total_value": {"breakdowns": [ + {"dimension_keys": ["city"], "results": [{"dimension_values": ["London, England"], "value": 22}, + {"dimension_values": ["Sydney, New South Wales"], "value": 33} + ]} + ]}, + "title": "Impressions", + "description": "Total number of times this profile has been seen", + "id": "17841400008460056/insights/impressions/day", + } + + @fixture(name="user_lifetime_insights") def user_lifetime_insights(): class UserLiftimeInsightEntityMock: diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 19470cb9c22b..0d6d1779272b 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -183,50 +183,22 @@ def test_user_insights_read(api, config, user_insight_data, requests_mock): assert records -def test_user_lifetime_insights_read(api, config, user_insight_data, requests_mock): +def test_user_lifetime_insights_read(api, config, user_lifetime_insight_data, requests_mock): test_id = "test_id" stream = UserLifetimeInsights(api=api) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_insight_data}]) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_lifetime_insight_data}]) records = read_full_refresh(stream) - assert records == [ - { - "page_id": "act_unknown_account", - "business_account_id": "test_id", - "metric": "impressions", - "date": "2020-05-04T07:00:00+0000", - "value": 4, - } - ] - - -@pytest.mark.parametrize( - "values,expected", - [ - ({"end_time": "2020-05-04T07:00:00+0000", "value": "test_value"}, {"date": "2020-05-04T07:00:00+0000", "value": "test_value"}), - ({"value": "test_value"}, {"date": None, "value": "test_value"}), - ({"end_time": "2020-05-04T07:00:00+0000"}, {"date": "2020-05-04T07:00:00+0000", "value": None}), - ({}, {"date": None, "value": None}), - ], - ids=[ - "`end_time` and `value` are present", - "no `end_time`, but `value` is present", - "`end_time` is present, but no `value`", - "no `end_time` and no `value`", - ], -) -def test_user_lifetime_insights_read_with_missing_keys(api, user_lifetime_insights, values, expected): - """ - This tests shows the behaviour of the `read_records` when either `end_time` or `value` key is not present in the data. - """ - stream = UserLifetimeInsights(api=api) - user_lifetime_insights(values) - test_slice = {"account": {"page_id": 1, "instagram_business_account": user_lifetime_insights}} - for insight in stream.read_records(sync_mode=None, stream_slice=test_slice): - assert insight["date"] == expected.get("date") - assert insight["value"] == expected.get("value") + expected_record = { + "breakdown": "city", + "business_account_id": "test_id", + "metric": "impressions", + "page_id": "act_unknown_account", + "value": {"London, England": 22, "Sydney, New South Wales": 33} + } + assert expected_record in records @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-instatus/main.py b/airbyte-integrations/connectors/source-instatus/main.py index dc0a480e8f7e..0e0c5af556b5 100644 --- a/airbyte-integrations/connectors/source-instatus/main.py +++ b/airbyte-integrations/connectors/source-instatus/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_instatus import SourceInstatus +from source_instatus.run import run if __name__ == "__main__": - source = SourceInstatus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-instatus/metadata.yaml b/airbyte-integrations/connectors/source-instatus/metadata.yaml index 67980e6c3fb3..225e47a7158b 100644 --- a/airbyte-integrations/connectors/source-instatus/metadata.yaml +++ b/airbyte-integrations/connectors/source-instatus/metadata.yaml @@ -8,6 +8,10 @@ data: icon: instatus.svg license: MIT name: Instatus + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-instatus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-instatus/setup.py b/airbyte-integrations/connectors/source-instatus/setup.py index 0a0586841809..8a914c480bb3 100644 --- a/airbyte-integrations/connectors/source-instatus/setup.py +++ b/airbyte-integrations/connectors/source-instatus/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-instatus=source_instatus.run:run", + ], + }, name="source_instatus", description="Source implementation for Instatus.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py new file mode 100644 index 000000000000..ade50f0a9cdd --- /dev/null +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_instatus import SourceInstatus + + +def run(): + source = SourceInstatus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-intercom/README.md b/airbyte-integrations/connectors/source-intercom/README.md index d5b904c935df..931a8c75ddea 100644 --- a/airbyte-integrations/connectors/source-intercom/README.md +++ b/airbyte-integrations/connectors/source-intercom/README.md @@ -1,118 +1,55 @@ -# Intercom Yaml Source +# Intercom source connector -This is the repository for the Intercom Yaml configuration based source connector. + +This is the repository for the Intercom source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/intercom). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python3 -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials +### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/intercom) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_intercom/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source intercom test creds` -and place them into `secrets/config.json`. - -### Locally running the connector docker image ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-intercom spec +poetry run source-intercom check --config secrets/config.json +poetry run source-intercom discover --config secrets/config.json +poetry run source-intercom read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: - +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-intercom build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-intercom:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-intercom:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-intercom:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-intercom:dev . -# Running the spec command against your patched connector -docker run airbyte/source-intercom:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-intercom:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intercom:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-intercom:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-intercom test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-intercom test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/intercom.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/intercom.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml index b2c5d59e44cc..37dd822467c7 100644 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -6,9 +6,6 @@ acceptance_tests: spec: tests: - spec_path: "source_intercom/spec.json" - # Spec fix: advanced auth configuration contain `client_id` and `client_secret` fields but they were missing in spec. - backward_compatibility_tests_config: - disable_for_version: "0.2.1" connection: tests: - config_path: "secrets/config.json" @@ -18,9 +15,6 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" - # Schema fix: update schemas with undeclared fields which is not breaking change - backward_compatibility_tests_config: - disable_for_version: "0.2.1" basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index 2bd1cb003b2c..e874bc451c67 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -73,5 +73,16 @@ "updated_at": 7626086649 } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "activity_logs" + }, + "stream_state": { + "created_at": 7626086649 + } + } } ] diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json index 2e0e4e62a618..66ccdc871d86 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -11,6 +11,20 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "companies", diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl index ed784188abc5..d06ce90f22b7 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl @@ -1,127 +1,36 @@ -{"stream":"admins","data":{"type":"admin","email":"integration-test@airbyte.io","id":"4423433","name":"Airbyte Team","job_title":"Admin","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366492} -{"stream":"admins","data":{"type":"admin","email":"operator+wjw5eps7@intercom.io","id":"4423434","name":"Operator","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366495} -{"stream":"admins","data":{"type":"admin","email":"jared@daxtarity.com","id":"4425337","name":"Jared Rhizor","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366498} -{"stream":"admins","data":{"type":"admin","email":"user2.sample.airbyte@gmail.com","id":"6405371","name":"user2","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366500} -{"stream":"admins","data":{"type":"admin","email":"user1.sample@zohomail.eu","id":"6405388","name":"User1","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366502} -{"stream":"admins","data":{"type":"admin","email":"user3.sample.airbyte@outlook.com","id":"6407134","name":"User3 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366504} -{"stream":"admins","data":{"type":"admin","email":"user4.sample.airbyte@outlook.com","id":"6407142","name":"User4 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366506} -{"stream":"admins","data":{"type":"admin","email":"user5.sample.airbyte@outlook.com","id":"6407146","name":"User5 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366508} -{"stream":"admins","data":{"type":"admin","email":"user6.sample.airbyte@outlook.com","id":"6407148","name":"User6 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366509} -{"stream":"admins","data":{"type":"admin","email":"user7.sample.airbyte@outlook.com","id":"6407153","name":"User7 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366511} -{"stream":"admins","data":{"type":"admin","email":"user8.sample.airbyte@outlook.com","id":"6407155","name":"User8 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366513} -{"stream":"admins","data":{"type":"admin","email":"user9.sample.airbyte@outlook.com","id":"6407156","name":"User9 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366514} -{"stream":"admins","data":{"type":"admin","email":"user10.sample.airbyte@outlook.com","id":"6407160","name":"User10 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366516} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5731d460cdc137c906d-qualification-company", "id": "63ecc5731d460cdc137c906c", "app_id": "wjw5eps7", "name": "Test Company 8", "created_at": 1676461427, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 49, "website": "www.company8.com", "industry": "Manufacturing", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867526} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc52f00fc87e58e8fb1f2-qualification-company", "id": "63ecc52f00fc87e58e8fb1f1", "app_id": "wjw5eps7", "name": "Test Company 7", "created_at": 1676461359, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 23, "website": "www.company7.com", "industry": "Production", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867529} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc46a811f1737ded479ef-qualification-company", "id": "63ecc46a811f1737ded479ee", "app_id": "wjw5eps7", "name": "Test Company 4", "created_at": 1676461162, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 150, "website": "www.company4.com", "industry": "Software", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867531} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5d32059cdacf4ac6171-qualification-company", "id": "63ecc5d32059cdacf4ac6170", "app_id": "wjw5eps7", "name": "Test Company 9", "created_at": 1676461523, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 75, "website": "www.company9.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867536} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc61266325d8ebd24ed11-qualification-company", "id": "63ecc61266325d8ebd24ed10", "app_id": "wjw5eps7", "name": "Test Company 10", "created_at": 1676461586, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 38, "website": "www.company10.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867538} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfccb064f24a4941d219-qualification-company", "id": "63ecbfccb064f24a4941d218", "app_id": "wjw5eps7", "name": "Test Company", "created_at": 1676459980, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799571", "name": "Tag1"}, {"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867533} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfef66325dc8a0ac006f-qualification-company", "id": "63ecbfef66325dc8a0ac006e", "app_id": "wjw5eps7", "name": "Test Company 2", "created_at": 1676460015, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT 123", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867540} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc41866325d2e90b0d3c6-qualification-company", "id": "63ecc41866325d2e90b0d3c5", "app_id": "wjw5eps7", "name": "Test Company 3", "created_at": 1676461080, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 50, "website": "www.company3.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867542} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc3d60e3c81baaad9f9ef-qualification-company", "id": "63ecc3d60e3c81baaad9f9ee", "app_id": "wjw5eps7", "name": "Company 1", "created_at": 1676461015, "updated_at": 1689068298, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 25, "website": "www.company1.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867548} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc4e99a2c64721f435a23-qualification-company", "id": "63ecc4e99a2c64721f435a22", "app_id": "wjw5eps7", "name": "Test Company 6", "created_at": 1676461289, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 55, "website": "www.company6.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867544} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc7afb3789118eb91306b-qualification-company", "id": "63ecc7afb3789118eb91306a", "app_id": "wjw5eps7", "name": "Test Company 11", "created_at": 1676461999, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 9, "website": "www.company11.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867546} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "name", "full_name": "name", "label": "Company name", "description": "The name of a company", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977060} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "company_id", "full_name": "company_id", "label": "Company ID", "description": "A number identifying a company", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977062} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "last_request_at", "full_name": "last_request_at", "label": "Company last seen", "description": "The last day anyone from a company visited your site or app", "data_type": "date", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977064} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "remote_created_at", "full_name": "remote_created_at", "label": "Company created at", "description": "The day a company was added to Intercom", "data_type": "date", "api_writable": true, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977065} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "user_count", "full_name": "user_count", "label": "People", "description": "The number of people in a company", "data_type": "integer", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977067} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "session_count", "full_name": "session_count", "label": "Company web sessions", "description": "All visits from anyone in a company to your product's site or app", "data_type": "integer", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977069} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "name", "full_name": "plan.name", "label": "Plan", "description": "A specific plan or level within your product that companies have signed up to", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977070} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "monthly_spend", "full_name": "monthly_spend", "label": "Monthly Spend", "description": "The monthly revenue you receive from a company", "data_type": "float", "api_writable": true, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977072} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "size", "full_name": "size", "label": "Company size", "description": "The number of people employed in this company, expressed as a single number", "data_type": "integer", "api_writable": true, "ui_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977074} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "industry", "full_name": "industry", "label": "Company industry", "description": "The category or domain this company belongs to e.g. 'ecommerce' or 'SaaS'", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977075} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "website", "full_name": "website", "label": "Company website", "description": "The web address for the company's primary marketing site", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977077} -{"stream": "company_attributes", "data": {"id": 9184602, "type": "data_attribute", "name": "creation_source", "full_name": "custom_attributes.creation_source", "label": "creation_source", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": true, "archived": false, "created_at": 1676459581, "updated_at": 1676459581, "model": "company"}, "emitted_at": 1680518977079} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "id", "full_name": "id", "label": "ID", "description": "The Intercom defined id representing the company", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977080} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "created_at", "full_name": "created_at", "label": "Created at", "description": "The time the company was added to Intercom", "data_type": "date", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977082} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "updated_at", "full_name": "updated_at", "label": "Updated at", "description": "The last time the company was updated", "data_type": "date", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977083} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "id", "full_name": "plan.id", "label": "Plan ID", "description": "The Intercom defined id representing the plan", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977085} -{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "app_id", "full_name": "app_id", "label": "App ID", "description": "The Intercom defined id representing the app", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977087} -{"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a43d9c86cceefd8796e", "name": "Revenue", "created_at": 1676286531, "updated_at": 1676462321, "person_type": "user"}, "emitted_at": 1680518981971} -{"stream": "company_segments", "data": {"type": "segment", "id": "6241a4513be7e1fb6e627591", "name": "Test", "created_at": 1648469073, "updated_at": 1676462331, "person_type": "user"}, "emitted_at": 1680518981975} -{"stream": "company_segments", "data": {"type": "segment", "id": "6241a4b8c8b709894fa54df1", "name": "Test_1", "created_at": 1648469176, "updated_at": 1676462341, "person_type": "user"}, "emitted_at": 1680518981979} -{"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a19d248071b8d297b39", "name": "Companies less then 100 people", "created_at": 1676286489, "updated_at": 1676461957, "person_type": "user"}, "emitted_at": 1680518982240} -{"stream": "company_segments", "data": {"type": "segment", "id": "63eb62f228758099dbc7fabe", "name": "Companies not IT", "created_at": 1676370674, "updated_at": 1676461960, "person_type": "user"}, "emitted_at": 1680518982244} -{"stream": "company_segments", "data": {"type": "segment", "id": "63eb63c3046264426ef4bfd6", "name": "Companies tag not 3", "created_at": 1676370883, "updated_at": 1676461915, "person_type": "user"}, "emitted_at": 1680518982248} -{"stream": "company_segments", "data": {"type": "segment", "id": "63eb62d7265dcc25ab8f7dcb", "name": "Company Tag is not 2", "created_at": 1676370647, "updated_at": 1676461937, "person_type": "user"}, "emitted_at": 1680518982251} -{"stream": "company_segments", "data": {"type": "segment", "id": "63eb629e1c757c1d4ea70724", "name": "Compenies less then 20", "created_at": 1676370591, "updated_at": 1676461941, "person_type": "user"}, "emitted_at": 1680518982255} -{"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a43d9c86cceefd8796e", "name": "Revenue", "created_at": 1676286531, "updated_at": 1676462321, "person_type": "user"}, "emitted_at": 1680518982259} -{"stream": "company_segments", "data": {"type": "segment", "id": "63ecc7f36d40e8184b5d47a6", "name": "Sales", "created_at": 1676462067, "updated_at": 1676462069, "person_type": "user"}, "emitted_at": 1680518982262} -{"stream": "company_segments", "data": {"type": "segment", "id": "6241a4b8c8b709894fa54df1", "name": "Test_1", "created_at": 1648469176, "updated_at": 1676462341, "person_type": "user"}, "emitted_at": 1680518982266} -{"stream": "conversations", "data": {"type": "conversation", "id": "1", "created_at": 1607553243, "updated_at": 1626346673, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "701718739", "delivered_as": "customer_initiated", "subject": "", "body": "

      hey there

      ", "author": {"type": "lead", "id": "5fd150d50697b6d0bbc4a2c2", "name": null, "email": ""}, "attachments": [], "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d", "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": {"created_at": 1607553243, "type": "conversation", "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "admin_assignee_id": null, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": 4317957, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": 4317954, "first_contact_reply_at": 1607553243, "first_assignment_at": null, "first_admin_reply_at": 1625654131, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": 1607553246, "last_admin_reply_at": 1625656000, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 7}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": null, "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694977} -{"stream": "conversations", "data": {"type": "conversation", "id": "59", "created_at": 1676460979, "updated_at": 1689068230, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952658", "delivered_as": "automated", "subject": "", "body": "

      Test 1

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea418c0931f79d99a197ff"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": false, "state": "closed", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 3}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695018} -{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

      Test 3

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test3", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694982} -{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

      Test 4

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 4", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694985} -{"stream": "conversations", "data": {"type": "conversation", "id": "63", "created_at": 1676461327, "updated_at": 1676461328, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953153", "delivered_as": "automated", "subject": "", "body": "

      Test 6

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2b2d44e63848146e7"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 6", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694989} -{"stream": "conversations", "data": {"type": "conversation", "id": "64", "created_at": 1676461395, "updated_at": 1676461396, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953262", "delivered_as": "automated", "subject": "", "body": "

      Test 7

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2c340f850172f2905"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 7", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694994} -{"stream": "conversations", "data": {"type": "conversation", "id": "65", "created_at": 1676461499, "updated_at": 1676461499, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953436", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 1

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694998} -{"stream": "conversations", "data": {"type": "conversation", "id": "66", "created_at": 1676461563, "updated_at": 1676461564, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953541", "delivered_as": "automated", "subject": "", "body": "

      Test 9

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a3b0e17c505e52044d"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 9", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695003} -{"stream": "conversations", "data": {"type": "conversation", "id": "67", "created_at": 1676461636, "updated_at": 1676461637, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953649", "delivered_as": "automated", "subject": "", "body": "

      Test 10

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a7b0e17c5039fbb824"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 10", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695007} -{"stream": "conversations", "data": {"type": "conversation", "id": "68", "created_at": 1676461800, "updated_at": 1676461800, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953852", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 5001

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ecc6c2811f17873ed2d007"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 5001", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695011} -{"stream": "conversations", "data": {"type": "conversation", "id": "69", "created_at": 1676462031, "updated_at": 1676462031, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51954139", "delivered_as": "automated", "subject": "", "body": "

      Test 11

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a80931f79b6998e89f"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 11", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695015} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288120839", "part_type": "comment", "body": "

      is this showing up

      ", "created_at": 1607553246, "updated_at": 1607553246, "notified_at": 1607553246, "assigned_to": null, "author": {"id": "5fd150d50697b6d0bbc4a2c2", "type": "user", "name": null, "email": ""}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241806} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121348", "part_type": "comment", "body": "

      Airbyte [DEV] will reply as soon as they can.

      ", "created_at": 1607553249, "updated_at": 1607553249, "notified_at": 1607553249, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241811} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121392", "part_type": "comment", "body": "

      Give the team a way to reach you:

      ", "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241815} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121429", "part_type": "comment", "body": null, "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241819} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9852986065", "part_type": "comment", "body": "

      This message was deleted

      ", "created_at": 1625654131, "updated_at": 1626346672, "notified_at": 1625654131, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1688632241822} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9853397844", "part_type": "comment", "body": "

      This message was deleted

      ", "created_at": 1625656000, "updated_at": 1626346669, "notified_at": 1625656000, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1688632241825} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759948453", "part_type": "assignment", "body": null, "created_at": 1676460980, "updated_at": 1676460980, "notified_at": 1676460980, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "59"}, "emitted_at": 1688632242153} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759993209", "part_type": "assignment", "body": null, "created_at": 1676461134, "updated_at": 1676461134, "notified_at": 1676461134, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "60"}, "emitted_at": 1688632242481} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760015015", "part_type": "assignment", "body": null, "created_at": 1676461197, "updated_at": 1676461197, "notified_at": 1676461197, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "61"}, "emitted_at": 1688632242922} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760059085", "part_type": "assignment", "body": null, "created_at": 1676461328, "updated_at": 1676461328, "notified_at": 1676461328, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "63"}, "emitted_at": 1688632243464} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760081807", "part_type": "assignment", "body": null, "created_at": 1676461396, "updated_at": 1676461396, "notified_at": 1676461396, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "64"}, "emitted_at": 1688632243779} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760115881", "part_type": "assignment", "body": null, "created_at": 1676461499, "updated_at": 1676461499, "notified_at": 1676461499, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "65"}, "emitted_at": 1688632244129} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760137018", "part_type": "assignment", "body": null, "created_at": 1676461564, "updated_at": 1676461564, "notified_at": 1676461564, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "66"}, "emitted_at": 1688632244457} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760160934", "part_type": "assignment", "body": null, "created_at": 1676461637, "updated_at": 1676461637, "notified_at": 1676461637, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "67"}, "emitted_at": 1688632244747} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760216689", "part_type": "assignment", "body": null, "created_at": 1676461800, "updated_at": 1676461800, "notified_at": 1676461800, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "68"}, "emitted_at": 1688632245084} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760294809", "part_type": "assignment", "body": null, "created_at": 1676462031, "updated_at": 1676462031, "notified_at": 1676462031, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "69"}, "emitted_at": 1688632245506} -{"stream": "contact_attributes", "data": {"id": 9182660, "type": "data_attribute", "name": "Company", "full_name": "custom_attributes.Company", "label": "Company", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": true, "archived": false, "created_at": 1676295702, "updated_at": 1676295702, "model": "contact"}, "emitted_at": 1680518989376} -{"stream": "contact_attributes", "data": {"id": 9182661, "type": "data_attribute", "name": "Tag", "full_name": "custom_attributes.Tag", "label": "Tag", "data_type": "string", "api_writable": true, "ui_writable": false, "custom": true, "archived": false, "created_at": 1676295702, "updated_at": 1676295702, "model": "contact"}, "emitted_at": 1680518989378} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "id", "full_name": "id", "label": "ID", "description": "The Intercom defined id representing the user", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989379} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "avatar", "full_name": "avatar", "label": "Avatar image url", "description": "An avatar image URL", "data_type": "string", "api_writable": true, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989381} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "updated_at", "full_name": "updated_at", "label": "Updated at", "description": "The last time the user was updated", "data_type": "date", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989382} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "workspace_id", "full_name": "workspace_id", "label": "Workspace ID", "description": "The Intercom defined id representing the workspace", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989384} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "android_app_name", "full_name": "android_app_name", "label": "Android App name", "description": "The name of Android app a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989385} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "android_sdk_version", "full_name": "android_sdk_version", "label": "Android SDK version", "description": "The version of the Android SDK a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989386} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "ios_app_name", "full_name": "ios_app_name", "label": "iOS App name", "description": "The name of iOS app a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989387} -{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "ios_sdk_version", "full_name": "ios_sdk_version", "label": "iOS SDK version", "description": "The version of the iOS SDK a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989389} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41aaeddb9b627ce9b882", "workspace_id": "wjw5eps7", "external_id": "20033080", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33080", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296619, "updated_at": 1676296619, "signed_up_at": 2328134400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41aaeddb9b627ce9b882/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147400} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1b0e17c51fa4eb704", "workspace_id": "wjw5eps7", "external_id": "20037835", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User37835", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2738966400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1b0e17c51fa4eb704/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147403} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1c340f84dffe8cddc", "workspace_id": "wjw5eps7", "external_id": "20033579", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33579", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2371248000, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1c340f84dffe8cddc/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147406} -{"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f238", "name": "Active", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1680518991074} -{"stream": "segments", "data": {"type": "segment", "id": "63ea19b36ea882cf2f785a45", "name": "Country", "created_at": 1676286387, "updated_at": 1676286387, "person_type": "user"}, "emitted_at": 1680518991078} -{"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f237", "name": "New", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1680518991082} -{"stream": "segments", "data": {"type": "segment", "id": "63ea45c316a606e302e8c804", "name": "New Users", "created_at": 1676297667, "updated_at": 1676297667, "person_type": "user"}, "emitted_at": 1680518991086} -{"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f239", "name": "Slipping Away", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1680518991091} -{"stream": "segments", "data": {"type": "segment", "id": "63ea1953fe6b33647e06b01f", "name": "Test Segment 2", "created_at": 1676286291, "updated_at": 1676286291, "person_type": "user"}, "emitted_at": 1680518991095} -{"stream": "segments", "data": {"type": "segment", "id": "63ea1966651fc0fc6468eaa9", "name": "Test Segment 3", "created_at": 1676286310, "updated_at": 1676286310, "person_type": "user"}, "emitted_at": 1680518991099} -{"stream": "segments", "data": {"type": "segment", "id": "63ea19843ca5145fafc58601", "name": "Test Segment 4", "created_at": 1676286340, "updated_at": 1676286340, "person_type": "user"}, "emitted_at": 1680518991103} -{"stream": "segments", "data": {"type": "segment", "id": "5fcf65de5f0be4c8398c4f8f", "name": "Test segment", "created_at": 1607427550, "updated_at": 1607427550, "person_type": "user"}, "emitted_at": 1680518991108} -{"stream": "tags", "data": {"type": "tag", "id": "7800216", "name": "CSV Import - 2023-02-13 13:36:22 UTC"}, "emitted_at": 1680518991624} -{"stream": "tags", "data": {"type": "tag", "id": "7800227", "name": "CSV Import - 2023-02-13 13:40:07 UTC"}, "emitted_at": 1680518991626} -{"stream": "tags", "data": {"type": "tag", "id": "7800265", "name": "CSV Import - 2023-02-13 13:50:25 UTC"}, "emitted_at": 1680518991627} -{"stream": "tags", "data": {"type": "tag", "id": "7800280", "name": "CSV Import - 2023-02-13 13:53:25 UTC"}, "emitted_at": 1680518991629} -{"stream": "tags", "data": {"type": "tag", "id": "7800292", "name": "CSV Import - 2023-02-13 13:55:20 UTC"}, "emitted_at": 1680518991630} -{"stream": "tags", "data": {"type": "tag", "id": "4849642", "name": "onboarded to feature A"}, "emitted_at": 1680518991631} -{"stream": "tags", "data": {"type": "tag", "id": "7799571", "name": "Tag1"}, "emitted_at": 1680518991632} -{"stream": "tags", "data": {"type": "tag", "id": "7799640", "name": "Tag10"}, "emitted_at": 1680518991634} -{"stream": "tags", "data": {"type": "tag", "id": "7799570", "name": "Tag2"}, "emitted_at": 1680518991635} -{"stream": "tags", "data": {"type": "tag", "id": "7799608", "name": "Tag3"}, "emitted_at": 1680518991636} -{"stream": "tags", "data": {"type": "tag", "id": "7799607", "name": "Tag4"}, "emitted_at": 1680518991638} -{"stream": "tags", "data": {"type": "tag", "id": "7799633", "name": "Tag5"}, "emitted_at": 1680518991639} -{"stream": "tags", "data": {"type": "tag", "id": "7799634", "name": "Tag6"}, "emitted_at": 1680518991641} -{"stream": "tags", "data": {"type": "tag", "id": "7799637", "name": "Tag7"}, "emitted_at": 1680518991642} -{"stream": "tags", "data": {"type": "tag", "id": "7799636", "name": "Tag8"}, "emitted_at": 1680518991643} -{"stream": "tags", "data": {"type": "tag", "id": "7799641", "name": "Tag9"}, "emitted_at": 1680518991644} -{"stream":"teams","data":{"type":"team","id":"5077733","name":"test","admin_ids":[]},"emitted_at":1695812351359} -{"stream":"teams","data":{"type":"team","id":"6407164","name":"Test team 2","admin_ids":[]},"emitted_at":1695812351360} -{"stream":"teams","data":{"type":"team","id":"6407165","name":"Test team 3","admin_ids":[]},"emitted_at":1695812351361} -{"stream":"teams","data":{"type":"team","id":"6407166","name":"Test team 4","admin_ids":[]},"emitted_at":1695812351362} -{"stream":"teams","data":{"type":"team","id":"6407167","name":"Test team 5","admin_ids":[]},"emitted_at":1695812351363} -{"stream":"teams","data":{"type":"team","id":"6407168","name":"Test team 6","admin_ids":[]},"emitted_at":1695812351364} -{"stream":"teams","data":{"type":"team","id":"6407170","name":"Test team 7","admin_ids":[]},"emitted_at":1695812351365} -{"stream":"teams","data":{"type":"team","id":"6407173","name":"Test team 8","admin_ids":[]},"emitted_at":1695812351366} -{"stream":"teams","data":{"type":"team","id":"6407174","name":"Test team 9","admin_ids":[]},"emitted_at":1695812351367} -{"stream":"teams","data":{"type":"team","id":"6407175","name":"Test team 10","admin_ids":[]},"emitted_at":1695812351368} +{"stream": "activity_logs", "data": {"id": "f7cf4eba-3a37-44b0-aecf-f347fe116712", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"admin": {"id": 4423433, "first_name": "John", "last_name": "Lafleur"}, "before": {"permissions": {"access_billing_settings": true, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": true, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": true, "manage_inbox_rules": true, "manage_inbox_views": true, "manage_messages_settings": true, "manage_messenger_settings": true, "manage_saved_replies": true, "manage_tags": true, "manage_teammates": true, "reassign_conversations": true, "redact_conversation_parts": true, "send_messages": true}, "conversation_access": {}}, "after": {"permissions": {"access_billing_settings": true, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": true, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": true, "manage_inbox_rules": true, "manage_inbox_views": true, "manage_messages_settings": true, "manage_messenger_settings": true, "manage_saved_replies": true, "manage_tags": true, "manage_teammates": true, "reassign_conversations": true, "redact_conversation_parts": true, "send_messages": true}, "conversation_access": {"access_type": "all", "assignee_blocked_list": null, "include_unassigned": false}}}, "created_at": 1625657753, "activity_type": "admin_permission_change", "activity_description": "Airbyte Team changed John Lafleur's permissions."}, "emitted_at": 1707747693643} +{"stream": "activity_logs", "data": {"id": "1fb8c7f2-bb57-49c9-bffc-7c49e0e54b40", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"team": {"id": 5077733, "name": "test", "member_count": 1}}, "created_at": 1625657582, "activity_type": "app_team_creation", "activity_description": "Airbyte Team created a new team, test, with 1 member."}, "emitted_at": 1707747693645} +{"stream": "activity_logs", "data": {"id": "5f569e46-45c3-4f76-93b7-9096bca00431", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"admin": {"id": 4425337, "first_name": "Jared", "last_name": "Rhizor"}, "before": {"permissions": {"access_billing_settings": false, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": false, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": false, "manage_inbox_rules": true, "manage_inbox_views": false, "manage_messages_settings": false, "manage_messenger_settings": false, "manage_saved_replies": false, "manage_tags": true, "manage_teammates": true, "reassign_conversations": false, "redact_conversation_parts": false, "send_messages": false}, "conversation_access": {}}, "after": {"permissions": {"access_billing_settings": false, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": false, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": false, "manage_inbox_rules": true, "manage_inbox_views": false, "manage_messages_settings": false, "manage_messenger_settings": false, "manage_saved_replies": false, "manage_tags": true, "manage_teammates": true, "reassign_conversations": false, "redact_conversation_parts": false, "send_messages": false}, "conversation_access": {"access_type": "all", "assignee_blocked_list": null, "include_unassigned": false}}}, "created_at": 1625657461, "activity_type": "admin_permission_change", "activity_description": "Airbyte Team changed Jared Rhizor's permissions."}, "emitted_at": 1707747693647} +{"stream": "admins", "data": {"type": "admin", "email": "integration-test@airbyte.io", "id": "4423433", "name": "Airbyte Team", "job_title": "Admin", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": false, "team_ids": [], "team_priority_level": {}}, "emitted_at": 1707747707709} +{"stream": "admins", "data": {"type": "admin", "email": "operator+wjw5eps7@intercom.io", "id": "4423434", "name": "Operator", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": false, "team_ids": [], "team_priority_level": {}}, "emitted_at": 1707747707714} +{"stream": "admins", "data": {"type": "admin", "email": "jared@daxtarity.com", "id": "4425337", "name": "Jared Rhizor", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": false, "team_ids": [], "team_priority_level": {}}, "emitted_at": 1707747707716} +{"stream": "tags", "data": {"type": "tag", "id": "7800216", "name": "CSV Import - 2023-02-13 13:36:22 UTC"}, "emitted_at": 1707747708301} +{"stream": "tags", "data": {"type": "tag", "id": "7800227", "name": "CSV Import - 2023-02-13 13:40:07 UTC"}, "emitted_at": 1707747708330} +{"stream": "tags", "data": {"type": "tag", "id": "7800265", "name": "CSV Import - 2023-02-13 13:50:25 UTC"}, "emitted_at": 1707747708332} +{"stream": "teams", "data": {"type": "team", "id": "5077733", "name": "test", "admin_ids": []}, "emitted_at": 1707747708900} +{"stream": "teams", "data": {"type": "team", "id": "6407164", "name": "Test team 2", "admin_ids": []}, "emitted_at": 1707747708902} +{"stream": "teams", "data": {"type": "team", "id": "6407165", "name": "Test team 3", "admin_ids": []}, "emitted_at": 1707747708903} +{"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f238", "name": "Active", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1707747709505} +{"stream": "segments", "data": {"type": "segment", "id": "63ea19b36ea882cf2f785a45", "name": "Country", "created_at": 1676286387, "updated_at": 1676286387, "person_type": "user"}, "emitted_at": 1707747709507} +{"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f237", "name": "New", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1707747709509} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5731d460cdc137c906d-qualification-company", "id": "63ecc5731d460cdc137c906c", "app_id": "wjw5eps7", "name": "Test Company 8", "created_at": 1676461427, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 49, "website": "www.company8.com", "industry": "Manufacturing", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1707747710155} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc52f00fc87e58e8fb1f2-qualification-company", "id": "63ecc52f00fc87e58e8fb1f1", "app_id": "wjw5eps7", "name": "Test Company 7", "created_at": 1676461359, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 23, "website": "www.company7.com", "industry": "Production", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1707747710157} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc46a811f1737ded479ef-qualification-company", "id": "63ecc46a811f1737ded479ee", "app_id": "wjw5eps7", "name": "Test Company 4", "created_at": 1676461162, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 150, "website": "www.company4.com", "industry": "Software", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1707747710159} +{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "name", "full_name": "name", "label": "Company name", "description": "The name of a company", "data_type": "string", "api_writable": true, "ui_writable": true, "messenger_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1707747711124} +{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "company_id", "full_name": "company_id", "label": "Company ID", "description": "A number identifying a company", "data_type": "string", "api_writable": false, "ui_writable": false, "messenger_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1707747711128} +{"stream": "company_attributes", "data": {"type": "data_attribute", "name": "last_request_at", "full_name": "last_request_at", "label": "Company last seen", "description": "The last day anyone from a company visited your site or app", "data_type": "date", "api_writable": false, "ui_writable": false, "messenger_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1707747711130} +{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "role", "full_name": "role", "label": "Contact role", "description": "A person's role in their company, such as the name of their department", "data_type": "string", "api_writable": true, "ui_writable": false, "messenger_writable": true, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1707747711744} +{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "name", "full_name": "name", "label": "Name", "description": "A person's full name", "data_type": "string", "api_writable": true, "ui_writable": true, "messenger_writable": true, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1707747711747} +{"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "owner_id", "full_name": "owner_id", "label": "Owner", "description": "The teammate that owns a lead or user in Intercom", "data_type": "integer", "api_writable": true, "ui_writable": false, "messenger_writable": true, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1707747711750} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41aaeddb9b627ce9b882", "workspace_id": "wjw5eps7", "external_id": "20033080", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33080", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296619, "updated_at": 1676296619, "signed_up_at": 2328134400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41aaeddb9b627ce9b882/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/companies", "total_count": 0, "has_more": false}, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1707747712535} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1b0e17c51fa4eb704", "workspace_id": "wjw5eps7", "external_id": "20037835", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User37835", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2738966400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1b0e17c51fa4eb704/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/companies", "total_count": 0, "has_more": false}, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1707747712539} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1c340f84dffe8cddc", "workspace_id": "wjw5eps7", "external_id": "20033579", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33579", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2371248000, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1c340f84dffe8cddc/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/companies", "total_count": 0, "has_more": false}, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1707747712542} +{"stream": "conversations", "data": {"type": "conversation", "id": "1", "created_at": 1607553243, "updated_at": 1626346673, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "701718739", "delivered_as": "customer_initiated", "subject": "", "body": "

      hey there

      ", "author": {"type": "lead", "id": "5fd150d50697b6d0bbc4a2c2", "name": null, "email": ""}, "attachments": [], "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": {"created_at": 1607553243, "type": "conversation", "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": 4317957, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": 4317954, "first_contact_reply_at": 1607553243, "first_assignment_at": null, "first_admin_reply_at": 1625654131, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": 1607553246, "last_admin_reply_at": 1625656000, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 7}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "assignee": null}, "emitted_at": 1707747714058} +{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

      Test 3

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "assignee": {"type": "admin", "id": "4423433"}}, "emitted_at": 1707747714064} +{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

      Test 4

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "assignee": {"type": "admin", "id": "4423433"}}, "emitted_at": 1707747714069} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288120839", "part_type": "comment", "body": "

      is this showing up

      ", "created_at": 1607553246, "updated_at": 1607553246, "notified_at": 1607553246, "assigned_to": null, "author": {"id": "5fd150d50697b6d0bbc4a2c2", "type": "user", "name": null, "email": ""}, "attachments": [], "external_id": null, "conversation_id": "1"}, "emitted_at": 1707747716219} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121348", "part_type": "comment", "body": "

      Airbyte [DEV] will reply as soon as they can.

      ", "created_at": 1607553249, "updated_at": 1607553249, "notified_at": 1607553249, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "conversation_id": "1"}, "emitted_at": 1707747716222} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121392", "part_type": "comment", "body": "

      Give the team a way to reach you:

      ", "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "conversation_id": "1"}, "emitted_at": 1707747716225} +{"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a19d248071b8d297b39", "name": "Companies less then 100 people", "created_at": 1676286489, "updated_at": 1676461957, "person_type": "user"}, "emitted_at": 1707747722461} +{"stream": "company_segments", "data": {"type": "segment", "id": "63eb62f228758099dbc7fabe", "name": "Companies not IT", "created_at": 1676370674, "updated_at": 1676461960, "person_type": "user"}, "emitted_at": 1707747722463} +{"stream": "company_segments", "data": {"type": "segment", "id": "63eb63c3046264426ef4bfd6", "name": "Companies tag not 3", "created_at": 1676370883, "updated_at": 1676461915, "person_type": "user"}, "emitted_at": 1707747722465} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json index 2c4a3735e86d..04647c9bf1a7 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json @@ -1,5 +1,19 @@ { "streams": [ + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "companies", diff --git a/airbyte-integrations/connectors/source-intercom/main.py b/airbyte-integrations/connectors/source-intercom/main.py index a1f0ae6911bf..410860c90fd8 100644 --- a/airbyte-integrations/connectors/source-intercom/main.py +++ b/airbyte-integrations/connectors/source-intercom/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_intercom import SourceIntercom +from source_intercom.run import run if __name__ == "__main__": - source = SourceIntercom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index 05c8251ba966..c2af6045c305 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a - dockerImageTag: 0.3.2 + dockerImageTag: 0.6.0 dockerRepository: airbyte/source-intercom documentationUrl: https://docs.airbyte.com/integrations/sources/intercom githubIssueLabel: source-intercom icon: intercom.svg license: MIT name: Intercom + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-intercom registries: cloud: enabled: true @@ -33,5 +37,4 @@ data: supportLevel: certified tags: - language:low-code - - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-intercom/poetry.lock b/airbyte-integrations/connectors/source-intercom/poetry.lock new file mode 100644 index 000000000000..3afe43237c13 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/poetry.lock @@ -0,0 +1,1011 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.0.tar.gz", hash = "sha256:622f56bd7101493a74f11c33a45a31c251032333989996f137cac8370873c614"}, + {file = "airbyte_cdk-0.62.0-py3-none-any.whl", hash = "sha256:b21330a566b33dbdddde33243eb9855f086ad4272e3585ca626be1225451a3b8"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "8.0.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.1-py3-none-any.whl", hash = "sha256:3e4f16fe1c0a9dc9d9389161c127c3edc5d810c38d6793042fb81d9f48a59fca"}, + {file = "pytest-8.0.1.tar.gz", hash = "sha256:267f6563751877d772019b13aacbe4e860d73fe8f651f28112e9ac37de7513ae"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, + {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "892fad6a9e1ef67e204e4009a08694c4f73ee2891e8895165e0f646d72eecfec" diff --git a/airbyte-integrations/connectors/source-intercom/pyproject.toml b/airbyte-integrations/connectors/source-intercom/pyproject.toml new file mode 100644 index 000000000000..24942ecbbb9c --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.6.0" +name = "source-intercom" +description = "Source implementation for Intercom Yaml." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/intercom" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_intercom" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.0" + +[tool.poetry.scripts] +source-intercom = "source_intercom.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.12.0" +pytest = "^8.0.0" diff --git a/airbyte-integrations/connectors/source-intercom/requirements.txt b/airbyte-integrations/connectors/source-intercom/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-intercom/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py deleted file mode 100644 index f5fce35eb718..000000000000 --- a/airbyte-integrations/connectors/source-intercom/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest", - "pytest-mock", -] - -setup( - name="source_intercom", - description="Source implementation for Intercom Yaml.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py index 6e87c9b9e8a1..600ba64945b1 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py @@ -167,7 +167,6 @@ def read_parent_stream( ) for parent_slice in parent_stream_slices_gen: - parent_records_gen = self.parent_stream.read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=parent_slice, stream_state=stream_state ) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml index e5c648bb4692..efb838f7ce00 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml @@ -20,7 +20,8 @@ definitions: type: BearerAuthenticator api_token: "{{ config['access_token'] }}" request_headers: - Intercom-Version: "2.5" # ATTENTION: API version change is possible here + # API version header + Intercom-Version: "2.10" Accept: "application/json" error_handler: type: "DefaultErrorHandler" @@ -297,7 +298,46 @@ definitions: data_field: "conversations" page_size: 150 + # activity logs stream is incremental based on created_at field + activity_logs: + $ref: "#/definitions/stream_full_refresh" + primary_key: id + $parameters: + name: "activity_logs" + path: "admins/activity_logs" + data_field: "activity_logs" + retriever: + $ref: "#/definitions/retriever" + description: "The Retriever without passing page size option" + paginator: + type: "DefaultPaginator" + url_base: "#/definitions/requester/url_base" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.get('pages', {}).get('next') }}" + stop_condition: "{{ 'next' not in response.get('pages', {}) }}" + page_token_option: + type: RequestPath + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + cursor_datetime_formats: + - "%s" + datetime_format: "%s" + cursor_granularity: "PT1S" + step: "P30D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_time_option: + field_name: "created_at_before" + inject_into: "request_parameter" + start_time_option: + field_name: "created_at_after" + inject_into: "request_parameter" + streams: + - "#/definitions/activity_logs" - "#/definitions/admins" - "#/definitions/tags" - "#/definitions/teams" diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/run.py b/airbyte-integrations/connectors/source-intercom/source_intercom/run.py new file mode 100644 index 000000000000..434766998b6e --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_intercom import SourceIntercom + + +def run(): + source = SourceIntercom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json new file mode 100644 index 000000000000..3136288524e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json @@ -0,0 +1,37 @@ +{ + "type": "object", + "properties": { + "performed_by": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "ip": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "activity_type": { + "type": ["null", "string"] + }, + "activity_description": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json index 21a989d14e26..af7b5c5b0de8 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -58,6 +58,9 @@ }, "updated_at": { "type": ["null", "integer"] + }, + "messenger_writable": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json index 97187685769b..528fd0ae9ba7 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json @@ -51,6 +51,9 @@ }, "updated_at": { "type": ["null", "integer"] + }, + "messenger_writable": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json index e669fa2f0fee..5b9db058f841 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -124,6 +124,12 @@ }, "city": { "type": ["null", "string"] + }, + "continent_code": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-intruder/main.py b/airbyte-integrations/connectors/source-intruder/main.py index 3bb638ef0b9b..ec9ff51965a6 100644 --- a/airbyte-integrations/connectors/source-intruder/main.py +++ b/airbyte-integrations/connectors/source-intruder/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_intruder import SourceIntruder +from source_intruder.run import run if __name__ == "__main__": - source = SourceIntruder() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-intruder/metadata.yaml b/airbyte-integrations/connectors/source-intruder/metadata.yaml index a70265c693ff..49d692755d42 100644 --- a/airbyte-integrations/connectors/source-intruder/metadata.yaml +++ b/airbyte-integrations/connectors/source-intruder/metadata.yaml @@ -8,6 +8,10 @@ data: icon: intruder.svg license: MIT name: Intruder + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-intruder registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-intruder/setup.py b/airbyte-integrations/connectors/source-intruder/setup.py index 67cb265463db..1860d710d501 100644 --- a/airbyte-integrations/connectors/source-intruder/setup.py +++ b/airbyte-integrations/connectors/source-intruder/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-intruder=source_intruder.run:run", + ], + }, name="source_intruder", description="Source implementation for Intruder.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-intruder/source_intruder/run.py b/airbyte-integrations/connectors/source-intruder/source_intruder/run.py new file mode 100644 index 000000000000..fcdedf420bc8 --- /dev/null +++ b/airbyte-integrations/connectors/source-intruder/source_intruder/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_intruder import SourceIntruder + + +def run(): + source = SourceIntruder() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ip2whois/main.py b/airbyte-integrations/connectors/source-ip2whois/main.py index 9ca61a68d1e1..746592c48705 100644 --- a/airbyte-integrations/connectors/source-ip2whois/main.py +++ b/airbyte-integrations/connectors/source-ip2whois/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ip2whois import SourceIp2whois +from source_ip2whois.run import run if __name__ == "__main__": - source = SourceIp2whois() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml index a14ed278b363..03866ecbd8ca 100644 --- a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml +++ b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ip2whois.svg license: MIT name: IP2Whois + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ip2whois registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-ip2whois/setup.py b/airbyte-integrations/connectors/source-ip2whois/setup.py index 92303a3f0ea4..10c6ca83410b 100644 --- a/airbyte-integrations/connectors/source-ip2whois/setup.py +++ b/airbyte-integrations/connectors/source-ip2whois/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ip2whois=source_ip2whois.run:run", + ], + }, name="source_ip2whois", description="Source implementation for Ip2whois.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py b/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py new file mode 100644 index 000000000000..7c3e0cda5573 --- /dev/null +++ b/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ip2whois import SourceIp2whois + + +def run(): + source = SourceIp2whois() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-iterable/.coveragerc b/airbyte-integrations/connectors/source-iterable/.coveragerc new file mode 100644 index 000000000000..abcf74bbab2f --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_iterable/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile deleted file mode 100644 index d05453734efe..000000000000 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_iterable ./source_iterable -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.30 -LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/README.md b/airbyte-integrations/connectors/source-iterable/README.md index 49d73415a8b1..db00a8823860 100644 --- a/airbyte-integrations/connectors/source-iterable/README.md +++ b/airbyte-integrations/connectors/source-iterable/README.md @@ -1,67 +1,91 @@ -# Coinmarketcap Source +# Iterable source connector -This is the repository for the Coinmarketcap configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coinmarketcap). + +This is the repository for the Iterable source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/iterable). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coinmarketcap) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coinmarketcap/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coinmarketcap test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name source-coinmarketcap build +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/iterable) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_iterable/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-iterable spec +poetry run source-iterable check --config secrets/config.json +poetry run source-iterable discover --config secrets/config.json +poetry run source-iterable read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -An image will be built with the tag `airbyte/source-coinmarketcap:dev`. +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -**Via `docker build`:** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-coinmarketcap:dev . +airbyte-ci connectors --name=source-iterable build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-iterable:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-coinmarketcap:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coinmarketcap:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-iterable:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-iterable:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-iterable:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-iterable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-iterable test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-iterable test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/iterable.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/iterable.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml index 60a8bbe63074..e6e5ed3decab 100644 --- a/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml @@ -102,6 +102,8 @@ acceptance_tests: incremental: tests: - config_path: "secrets/config.json" + # Temporarily skipping icnremental tests as email_complaint is failing despite being included in missing_streams + skip_comprehensive_incremental_tests: true configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-iterable/main.py b/airbyte-integrations/connectors/source-iterable/main.py index 3a4a2f7982ff..eef7d894cbc4 100644 --- a/airbyte-integrations/connectors/source-iterable/main.py +++ b/airbyte-integrations/connectors/source-iterable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_iterable import SourceIterable +from source_iterable.run import run if __name__ == "__main__": - source = SourceIterable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-iterable/metadata.yaml b/airbyte-integrations/connectors/source-iterable/metadata.yaml index 3df40f4c8d7b..ce432b88e86e 100644 --- a/airbyte-integrations/connectors/source-iterable/metadata.yaml +++ b/airbyte-integrations/connectors/source-iterable/metadata.yaml @@ -1,27 +1,33 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.iterable.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 - dockerImageTag: 0.1.30 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-iterable + documentationUrl: https://docs.airbyte.com/integrations/sources/iterable githubIssueLabel: source-iterable icon: iterable.svg license: MIT name: Iterable + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-iterable registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/iterable + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-iterable/poetry.lock b/airbyte-integrations/connectors/source-iterable/poetry.lock new file mode 100644 index 000000000000..03b1a7d4555c --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/poetry.lock @@ -0,0 +1,1079 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.1.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.5" +files = [ + {file = "freezegun-1.1.0-py2.py3-none-any.whl", hash = "sha256:2ae695f7eb96c62529f03a038461afe3c692db3465e215355e1bb4b0ab408712"}, + {file = "freezegun-1.1.0.tar.gz", hash = "sha256:177f9dd59861d871e27a484c3332f35a6e3f5d14626f2bf91be37891f18927f3"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.23.1" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, + {file = "responses-0.23.1.tar.gz", hash = "sha256:c4d9aa9fc888188f0c673eff79a8dadbe2e75b7fe879dc80a221a06e0a68138f"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.22.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "21991266470e2efceb259f4d42e1cdc62654126c7ed975ac9e0cbaf43f588347" diff --git a/airbyte-integrations/connectors/source-iterable/pyproject.toml b/airbyte-integrations/connectors/source-iterable/pyproject.toml new file mode 100644 index 000000000000..9c92df926217 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.0" +name = "source-iterable" +description = "Source implementation for Iterable." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/iterable" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_iterable" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +pendulum = "==2.1.2" +airbyte-cdk = "==0.63.2" +requests = "==2.31.0" +python-dateutil = "==2.8.2" + +[tool.poetry.scripts] +source-iterable = "source_iterable.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.6.1" +freezegun = "==1.1.0" +requests-mock = "^1.9.3" +responses = "==0.23.1" diff --git a/airbyte-integrations/connectors/source-iterable/requirements.txt b/airbyte-integrations/connectors/source-iterable/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-iterable/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-iterable/setup.py b/airbyte-integrations/connectors/source-iterable/setup.py deleted file mode 100644 index 5d2e499d31c7..000000000000 --- a/airbyte-integrations/connectors/source-iterable/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "pendulum~=2.1.2", - "python-dateutil~=2.8.2", - "requests~=2.25", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "responses==0.23.1", "freezegun==1.1.0"] - - -setup( - name="source_iterable", - description="Source implementation for Iterable.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, - package_data={"": ["*.json", "schemas/*.json"]}, -) diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/run.py b/airbyte-integrations/connectors/source-iterable/source_iterable/run.py new file mode 100644 index 000000000000..c2e01ead95e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_iterable import SourceIterable + + +def run(): + source = SourceIterable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json index fd74f6a40f9f..14cc02a90c99 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json @@ -29,6 +29,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "recipientState": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json index 5a0fecaf3447..f8439312858c 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json @@ -55,6 +55,9 @@ }, "email": { "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json index fd74f6a40f9f..14cc02a90c99 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json @@ -29,6 +29,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "recipientState": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json index 2e085dceeff0..36064e7ab3c9 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json @@ -46,6 +46,9 @@ }, "email": { "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json index e2614d971b18..1f328b78436b 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json @@ -122,6 +122,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json index 374a9671f998..a96ce2d53e7a 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json @@ -122,6 +122,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json index 3ac82b5cecba..8839d6d76ea2 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json @@ -30,6 +30,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "profileUpdatedAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json index 03b00577f7ba..c69cfa5bcb31 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json @@ -46,6 +46,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json index 028d32c78854..3c88b02b1ab9 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json @@ -23,6 +23,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "data": { "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py index 3294e110df46..2592b27d5fe0 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py @@ -23,17 +23,6 @@ def session_mock(): response_mock.status_code = 200 yield session_mock - -def test_send_email_stream(session_mock): - stream = Users(start_date="2020", authenticator=None) - stream_slice = StreamSlice(start_date=pendulum.parse("2020"), end_date=pendulum.parse("2021")) - _ = list(stream.read_records(sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=stream_slice, stream_state={})) - - assert session_mock.send.called - send_args = session_mock.send.call_args[1] - assert send_args.get("stream") is True - - @responses.activate def test_stream_correct(): stream_slice = StreamSlice(start_date=pendulum.parse("2020"), end_date=pendulum.parse("2021")) diff --git a/airbyte-integrations/connectors/source-jira/README.md b/airbyte-integrations/connectors/source-jira/README.md index be5d429e9d94..a9c7ce7bf481 100644 --- a/airbyte-integrations/connectors/source-jira/README.md +++ b/airbyte-integrations/connectors/source-jira/README.md @@ -1,67 +1,91 @@ -# News Api Source +# Jira source connector -This is the repository for the News Api configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/news-api). + +This is the repository for the Jira source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/jira). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/news-api) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_news_api/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source news-api test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name source-news-api build +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/jira) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_jira/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-jira spec +poetry run source-jira check --config secrets/config.json +poetry run source-jira discover --config secrets/config.json +poetry run source-jira read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -An image will be built with the tag `airbyte/source-news-api:dev`. +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -**Via `docker build`:** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-news-api:dev . +airbyte-ci connectors --name=source-jira build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-jira:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-news-api:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-news-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-jira:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-jira:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-jira:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-jira:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-jira test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-jira test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/jira.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/jira.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json index af6f2b9ee050..727d1fa0207e 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json @@ -6,7 +6,15 @@ "name": "board_issues" }, "stream_state": { - "updated": "2122-01-01T00:00:00Z" + "1": { + "updated": "2122-01-01T00:00:00Z" + }, + "17": { + "updated": "2122-01-01T00:00:00Z" + }, + "58": { + "updated": "2122-01-01T00:00:00Z" + } } } }, diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl index d5338e2d7ffd..62e84cc83433 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl @@ -21,9 +21,8 @@ {"stream": "groups", "data": {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87"}, "emitted_at": 1697453247031} {"stream": "groups", "data": {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1"}, "emitted_at": 1697453247032} {"stream": "groups", "data": {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e"}, "emitted_at": 1697453247033} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10625", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "key": "IT-25", "renderedFields": {"statuscategorychangedate": "17/May/22 4:06 AM", "created": "17/May/22 4:06 AM", "customfield_10017": "dark_yellow", "updated": "17/May/22 4:28 AM", "description": "

      Implement OAUth

      ", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "environment": "", "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "17/May/22 4:06 AM", "updated": "17/May/22 4:06 AM", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 1, "total": 1, "histories": [{"id": "15129", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-26", "toString": "This issue is cloned by IT-26"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:06:24.675-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "workratio": -1, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "created": "2022-05-17T04:06:24.048000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i0076v:", "customfield_10217": [], "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "inwardIssue": {"id": "10626", "key": "IT-26", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "fields": {"summary": "CLONE - Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2022-05-17T04:28:19.876000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "summary": "Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:06:24.048000-07:00", "updated": "2022-05-17T04:28:19.876000-07:00"}, "emitted_at": 1701283916831} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10080", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080", "key": "IT-24", "renderedFields": {"statuscategorychangedate": "11/Mar/21 6:17 AM", "timespent": "5 hours, 48 minutes", "aggregatetimespent": "5 hours, 48 minutes", "created": "11/Mar/21 6:17 AM", "customfield_10017": "", "timeestimate": "0 minutes", "updated": "05/Apr/23 4:58 AM", "description": "

      Test description 74

      ", "timetracking": {"remainingEstimate": "0 minutes", "timeSpent": "5 hours, 48 minutes", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "14/Apr/21 2:11 PM", "size": "7 kB", "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": "0 minutes", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 21 minutes", "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "37 minutes", "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 50 minutes", "id": "11710", "issueId": "10080"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 8, "total": 8, "histories": [{"id": "15179", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-04-05T04:58:35.333-0700", "items": [{"field": "Sprint", "fieldtype": "custom", "fieldId": "customfield_10020", "from": "", "fromString": "", "to": "10", "toString": "IT Sprint 9"}]}, {"id": "14989", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.917-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "10680", "fromString": "10680", "to": "20880", "toString": "20880"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11710", "toString": "11710"}]}, {"id": "14988", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.314-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "8460", "fromString": "8460", "to": "10680", "toString": "10680"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11709", "toString": "11709"}]}, {"id": "14987", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:46.691-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "8460", "toString": "8460"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11708", "toString": "11708"}]}, {"id": "14800", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T07:18:07.884-0700", "items": [{"field": "RemoteIssueLink", "fieldtype": "jira", "from": null, "fromString": null, "to": "10046", "toString": "This issue links to \"TSTSUP-111 (My Acme Tracker)\""}]}, {"id": "14718", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:54.455-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-22", "toString": "This issue is duplicated by IT-22"}]}, {"id": "14716", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-23", "toString": "This issue is duplicated by IT-23"}]}, {"id": "14596", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.899-0700", "items": [{"field": "Attachment", "fieldtype": "jira", "fieldId": "attachment", "from": null, "fromString": null, "to": "10123", "toString": "demo.xlsx"}]}]}, "fields": {"statuscategorychangedate": "2021-03-11T06:17:33.483-0800", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}, "timespent": 20880, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 20880, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "watchCount": 1, "isWatching": true}, "created": "2021-03-11T06:17:33.169000-08:00", "customfield_10020": [{"id": 10, "name": "IT Sprint 9", "state": "future", "boardId": 1, "startDate": "2022-09-06T11:25:59.072Z", "endDate": "2022-09-20T11:25:00.000Z"}], "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "labels": [], "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i000hr:", "timeestimate": 0, "versions": [], "issuelinks": [{"id": "10244", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10244", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10069", "key": "IT-22", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10069", "fields": {"summary": "Test 63", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}, {"id": "10243", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10243", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10075", "key": "IT-23", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075", "fields": {"summary": "Test 69", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}}}}], "updated": "2023-04-05T04:58:35.329000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "description": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description 74"}]}]}, "timetracking": {"remainingEstimate": "0m", "timeSpent": "5h 48m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.652-0700", "size": 7360, "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": 0, "summary": "Test 74", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 20880, "total": 20880, "percent": 100}, "progress": {"progress": 20880, "total": 20880, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 2", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.834-0700", "updated": "2021-04-15T11:39:47.834-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 50m", "timeSpentSeconds": 10200, "id": "11710", "issueId": "10080"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2021-03-11T06:17:33.169000-08:00", "updated": "2023-04-05T04:58:35.329000-07:00"}, "emitted_at": 1701283916937} -{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "renderedFields": {"statuscategorychangedate": "17/May/22 4:28 AM", "timespent": "1 day", "aggregatetimespent": "1 day", "lastViewed": "12/Oct/23 1:43 PM", "created": "17/May/22 4:28 AM", "customfield_10017": "dark_yellow", "timeestimate": "1 week, 1 day", "aggregatetimeoriginalestimate": "2 weeks, 4 days, 5 hours", "updated": "12/Oct/23 1:43 PM", "timeoriginalestimate": "2 weeks, 4 days, 5 hours", "description": "

      Implement OAUth

      ", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2 weeks, 4 days, 5 hours", "remainingEstimate": "1 week, 1 day", "timeSpent": "1 day", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": "1 week, 1 day", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "05/Apr/23 5:08 AM", "updated": "05/Apr/23 5:08 AM", "started": "05/Apr/23 1:00 AM", "timeSpent": "1 day", "id": "11820", "issueId": "10626"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 4, "total": 4, "histories": [{"id": "15198", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:15.036-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "172800", "toString": "172800"}]}, {"id": "15197", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:05.182-0700", "items": [{"field": "timeoriginalestimate", "fieldtype": "jira", "fieldId": "timeoriginalestimate", "from": null, "fromString": null, "to": "421200", "toString": "421200"}]}, {"id": "15186", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "2023-04-05T05:08:50.115-0700", "items": [{"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "28800", "toString": "28800"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11820", "toString": "11820"}]}, {"id": "15128", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.837-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-25", "toString": "This issue clones IT-25"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "workratio": 6, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "lastViewed": "2023-10-12T13:43:22.992-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "created": "2022-05-17T04:28:19.523000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i00773:", "timeestimate": 172800, "aggregatetimeoriginalestimate": 421200, "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-10-12T13:43:15.025000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": 421200, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2w 4d 5h", "remainingEstimate": "1w 1d", "timeSpent": "1d", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": 172800, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 201600, "percent": 14}, "progress": {"progress": 28800, "total": 201600, "percent": 14}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523000-07:00", "updated": "2023-10-12T13:43:15.025000-07:00"}, "emitted_at": 1701283916963} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10080", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080", "key": "IT-24", "renderedFields": {"statuscategorychangedate": "11/Mar/21 6:17 AM", "timespent": "5 hours, 48 minutes", "aggregatetimespent": "5 hours, 48 minutes", "created": "11/Mar/21 6:17 AM", "customfield_10017": "", "timeestimate": "0 minutes", "updated": "05/Apr/23 4:58 AM", "description": "

      Test description 74

      ", "timetracking": {"remainingEstimate": "0 minutes", "timeSpent": "5 hours, 48 minutes", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "14/Apr/21 2:11 PM", "size": "7 kB", "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": "0 minutes", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 21 minutes", "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "37 minutes", "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 50 minutes", "id": "11710", "issueId": "10080"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 8, "total": 8, "histories": [{"id": "15179", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-04-05T04:58:35.333-0700", "items": [{"field": "Sprint", "fieldtype": "custom", "fieldId": "customfield_10020", "from": "", "fromString": "", "to": "10", "toString": "IT Sprint 9"}]}, {"id": "14989", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.917-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "10680", "fromString": "10680", "to": "20880", "toString": "20880"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11710", "toString": "11710"}]}, {"id": "14988", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.314-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "8460", "fromString": "8460", "to": "10680", "toString": "10680"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11709", "toString": "11709"}]}, {"id": "14987", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:46.691-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "8460", "toString": "8460"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11708", "toString": "11708"}]}, {"id": "14800", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T07:18:07.884-0700", "items": [{"field": "RemoteIssueLink", "fieldtype": "jira", "from": null, "fromString": null, "to": "10046", "toString": "This issue links to \"TSTSUP-111 (My Acme Tracker)\""}]}, {"id": "14718", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:54.455-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-22", "toString": "This issue is duplicated by IT-22"}]}, {"id": "14716", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-23", "toString": "This issue is duplicated by IT-23"}]}, {"id": "14596", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.899-0700", "items": [{"field": "Attachment", "fieldtype": "jira", "fieldId": "attachment", "from": null, "fromString": null, "to": "10123", "toString": "demo.xlsx"}]}]}, "fields": {"statuscategorychangedate": "2021-03-11T06:17:33.483-0800", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}, "timespent": 20880, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 20880, "workratio": -1, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "created": "2021-03-11T06:17:33.169000-08:00", "customfield_10020": [{"id": 10, "name": "IT Sprint 9", "state": "future", "boardId": 1, "startDate": "2022-09-06T11:25:59.072Z", "endDate": "2022-09-20T11:25:00.000Z"}], "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "labels": [], "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i000hr:", "customfield_10217": [], "timeestimate": 0, "versions": [], "issuelinks": [{"id": "10244", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10244", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10069", "key": "IT-22", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10069", "fields": {"summary": "Test 63", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}, {"id": "10243", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10243", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10075", "key": "IT-23", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075", "fields": {"summary": "Test 69", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}}}}], "updated": "2023-04-05T04:58:35.329000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "description": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description 74"}]}]}, "timetracking": {"remainingEstimate": "0m", "timeSpent": "5h 48m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.652-0700", "size": 7360, "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": 0, "summary": "Test 74", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 20880, "total": 20880, "percent": 100}, "progress": {"progress": 20880, "total": 20880, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 2", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.834-0700", "updated": "2021-04-15T11:39:47.834-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 50m", "timeSpentSeconds": 10200, "id": "11710", "issueId": "10080"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2021-03-11T06:17:33.169000-08:00", "updated": "2023-04-05T04:58:35.329000-07:00"}, "emitted_at": 1706087956389} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "renderedFields": {"statuscategorychangedate": "17/May/22 4:28 AM", "timespent": "1 day", "aggregatetimespent": "1 day", "created": "17/May/22 4:28 AM", "customfield_10017": "dark_yellow", "timeestimate": "1 week, 1 day", "aggregatetimeoriginalestimate": "2 weeks, 4 days, 5 hours", "updated": "12/Oct/23 1:43 PM", "timeoriginalestimate": "2 weeks, 4 days, 5 hours", "description": "

      Implement OAUth

      ", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2 weeks, 4 days, 5 hours", "remainingEstimate": "1 week, 1 day", "timeSpent": "1 day", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": "1 week, 1 day", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "05/Apr/23 5:08 AM", "updated": "05/Apr/23 5:08 AM", "started": "05/Apr/23 1:00 AM", "timeSpent": "1 day", "id": "11820", "issueId": "10626"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 4, "total": 4, "histories": [{"id": "15198", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:15.036-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "172800", "toString": "172800"}]}, {"id": "15197", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:05.182-0700", "items": [{"field": "timeoriginalestimate", "fieldtype": "jira", "fieldId": "timeoriginalestimate", "from": null, "fromString": null, "to": "421200", "toString": "421200"}]}, {"id": "15186", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "2023-04-05T05:08:50.115-0700", "items": [{"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "28800", "toString": "28800"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11820", "toString": "11820"}]}, {"id": "15128", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.837-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-25", "toString": "This issue clones IT-25"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "workratio": 6, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "created": "2022-05-17T04:28:19.523000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i00773:", "timeestimate": 172800, "aggregatetimeoriginalestimate": 421200, "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-10-12T13:43:15.025000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": 421200, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2w 4d 5h", "remainingEstimate": "1w 1d", "timeSpent": "1d", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": 172800, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 201600, "percent": 14}, "progress": {"progress": 28800, "total": 201600, "percent": 14}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523000-07:00", "updated": "2023-10-12T13:43:15.025000-07:00"}, "emitted_at": 1706087956632} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076000-07:00", "updated": "2022-05-17T04:06:55.076000-07:00", "jsdPublic": true, "issueId": "IT-25"}, "emitted_at": 1697453253441} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10521", "id": "10521", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque eget venenatis elit. Duis eu justo eget augue iaculis fermentum. Sed semper quam laoreet nisi egestas at posuere augue semper.", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:32:43.099000-07:00", "updated": "2021-04-14T14:32:43.099000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254086} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10639", "id": "10639", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Linked related issue!", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.998000-07:00", "updated": "2021-04-15T00:08:48.998000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254087} diff --git a/airbyte-integrations/connectors/source-jira/main.py b/airbyte-integrations/connectors/source-jira/main.py index 9a560c777bf7..1885b3974def 100644 --- a/airbyte-integrations/connectors/source-jira/main.py +++ b/airbyte-integrations/connectors/source-jira/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_jira import SourceJira +from source_jira.run import run if __name__ == "__main__": - source = SourceJira() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index b8caaf3b41c2..654ccb7759c9 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 0.12.0 + dockerImageTag: 1.0.2 dockerRepository: airbyte/source-jira documentationUrl: https://docs.airbyte.com/integrations/sources/jira githubIssueLabel: source-jira @@ -18,12 +18,24 @@ data: license: MIT maxSecondsBetweenMessages: 21600 name: Jira + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-jira registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: "Stream state will be saved for every board in stream `Boards Issues`. Customers who use stream `Board Issues` in Incremental Sync mode must take action with their connections." + upgradeDeadline: "2024-01-25" + scopedImpact: + - scopeType: stream + impactedScopes: ["board_issues"] suggestedStreams: streams: - issues diff --git a/airbyte-integrations/connectors/source-jira/poetry.lock b/airbyte-integrations/connectors/source-jira/poetry.lock new file mode 100644 index 000000000000..41a088f72c9b --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/poetry.lock @@ -0,0 +1,1065 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.59.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.59.0.tar.gz", hash = "sha256:2f7bc07556cc7f42f0daf41d09be08fd22102864d087a27c8999f6f13fe67aad"}, + {file = "airbyte_cdk-0.59.0-py3-none-any.whl", hash = "sha256:94c561c053b8be3a66bfefe420812ced9237403441249408e2af5445214a6f7b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "types-toml" +version = "0.10.8.7" +description = "Typing stubs for toml" +optional = false +python-versions = "*" +files = [ + {file = "types-toml-0.10.8.7.tar.gz", hash = "sha256:58b0781c681e671ff0b5c0319309910689f4ab40e8a2431e205d70c94bb6efb1"}, + {file = "types_toml-0.10.8.7-py3-none-any.whl", hash = "sha256:61951da6ad410794c97bec035d59376ce1cbf4453dc9b6f90477e81e4442d631"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "044624d19c85475b787665da59cf1f03c423991291bd66013076f5cfb84d4e22" diff --git a/airbyte-integrations/connectors/source-jira/pyproject.toml b/airbyte-integrations/connectors/source-jira/pyproject.toml new file mode 100644 index 000000000000..d025905d1811 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.0.2" +name = "source-jira" +description = "Source implementation for Jira." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/jira" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_jira" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.59.0" + +[tool.poetry.scripts] +source-jira = "source_jira.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "==6.2.5" +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +responses = "^0.22.0" diff --git a/airbyte-integrations/connectors/source-jira/requirements.txt b/airbyte-integrations/connectors/source-jira/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-jira/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py deleted file mode 100644 index 800525b0a829..000000000000 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk>=0.51.19"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest==6.2.5", - "responses~=0.22.0", -] - -setup( - name="source_jira", - description="Source implementation for Jira.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/run.py b/airbyte-integrations/connectors/source-jira/source_jira/run.py new file mode 100644 index 000000000000..c5702721e0b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_jira import SourceJira + + +def run(): + source = SourceJira() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index 207177c2f79f..a34a74949070 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from typing import Any, List, Mapping, Optional, Tuple import pendulum @@ -74,13 +75,15 @@ ) from .utils import read_full_refresh +logger = logging.getLogger("airbyte") + class SourceJira(AbstractSource): def _validate_and_transform(self, config: Mapping[str, Any]): start_date = config.get("start_date") if start_date: config["start_date"] = pendulum.parse(start_date) - + config["lookback_window_minutes"] = pendulum.duration(minutes=config.get("lookback_window_minutes", 0)) config["projects"] = config.get("projects", []) return config @@ -90,18 +93,29 @@ def get_authenticator(config: Mapping[str, Any]): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: try: + original_config = config.copy() config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) kwargs = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} - labels_stream = Labels(**kwargs) - next(read_full_refresh(labels_stream), None) + # check projects projects_stream = Projects(**kwargs) projects = {project["key"] for project in read_full_refresh(projects_stream)} unknown_projects = set(config["projects"]) - projects if unknown_projects: return False, "unknown project(s): " + ", ".join(unknown_projects) - return True, None + + # Get streams to check access to any of them + streams = self.streams(original_config) + for stream in streams: + try: + next(read_full_refresh(stream), None) + except: + logger.warning("No access to stream: " + stream.name) + else: + logger.info(f"API Token have access to stream: {stream.name}, so check is successful.") + return True, None + return False, "This API Token does not have permission to read any of the resources." except ValidationError as validation_error: return False, validation_error except requests.exceptions.RequestException as request_error: @@ -131,7 +145,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} - incremental_args = {**args, "start_date": config.get("start_date")} + incremental_args = { + **args, + "start_date": config.get("start_date"), + "lookback_window_minutes": config.get("lookback_window_minutes"), + } issues_stream = Issues(**incremental_args) issue_fields_stream = IssueFields(**args) experimental_streams = [] diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json index 93d9fa13c71a..fff67ec349e6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/spec.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/spec.json @@ -82,12 +82,22 @@ "description": "Select fields to Expand the `Issues` stream when replicating with: ", "default": [] }, + "lookback_window_minutes": { + "title": "Lookback window", + "description": "When set to N, the connector will always refresh resources created within the past N minutes. By default, updated objects that are not newly created are not incrementally synced.", + "examples": [60], + "default": 0, + "minimum": 0, + "maximum": 576000, + "type": "integer", + "order": 5 + }, "enable_experimental_streams": { "type": "boolean", "title": "Enable Experimental Streams", "description": "Allow the use of experimental streams which rely on undocumented Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables for more info.", "default": false, - "order": 5 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 2bfed54b4e30..05c0d8381149 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -5,15 +5,17 @@ import re import urllib.parse as urlparse from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl import pendulum import requests -from airbyte_cdk.models import FailureType +from airbyte_cdk.logger import AirbyteLogger as Logger +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests.exceptions import HTTPError from source_jira.type_transfromer import DateTimeTransformer @@ -22,6 +24,20 @@ API_VERSION = 3 +class JiraAvailabilityStrategy(HttpAvailabilityStrategy): + """ + Inherit from HttpAvailabilityStrategy with slight modification to 403 and 401 error messages. + """ + + def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: + reasons_for_codes: Dict[int, str] = { + requests.codes.FORBIDDEN: "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access.", + requests.codes.UNAUTHORIZED: "Invalid creds were provided, please check your api token, domain and/or email.", + requests.codes.NOT_FOUND: "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access.", + } + return reasons_for_codes + + class JiraStream(HttpStream, ABC): """ Jira API Reference: https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ @@ -31,16 +47,15 @@ class JiraStream(HttpStream, ABC): primary_key: Optional[str] = "id" extract_field: Optional[str] = None api_v1 = False - skip_http_status_codes = [] + # Defines the HTTP status codes for which the slice should be skipped. + # Reference issue: https://github.com/airbytehq/oncall/issues/2133 + # we should skip the slice with `board id` which doesn't support `sprints` + # it's generally applied to all streams that might have the same error hit in the future. + skip_http_status_codes = [requests.codes.BAD_REQUEST] raise_on_http_errors = True - error_messages = { - requests.codes.UNAUTHORIZED: "Invalid creds were provided, please check your api token, domain and/or email.", - requests.codes.FORBIDDEN: "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access.", - } - config_error_status_codes = [ - requests.codes.UNAUTHORIZED, - ] transformer: TypeTransformer = DateTimeTransformer(TransformConfig.DefaultSchemaNormalization) + # emitting state message after every page read + state_checkpoint_interval = page_size def __init__(self, domain: str, projects: List[str], **kwargs): super().__init__(**kwargs) @@ -53,6 +68,14 @@ def url_base(self) -> str: return f"https://{self._domain}/rest/agile/1.0/" return f"https://{self._domain}/rest/api/{API_VERSION}/" + @property + def availability_strategy(self) -> HttpAvailabilityStrategy: + return JiraAvailabilityStrategy() + + def _get_custom_error(self, response: requests.Response) -> str: + """Method for specifying custom error messages for errors that will be skipped.""" + return "" + @property def max_retries(self) -> Union[int, None]: """Number of retries increased from default 5 to 10, based on issues with Jira. Max waiting time is still default 10 minutes.""" @@ -107,35 +130,22 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: try: yield from super().read_records(**kwargs) except HTTPError as e: - user_error_message = self.error_messages.get(e.response.status_code) - if user_error_message: - self.logger.error(user_error_message) - if e.response.status_code in self.config_error_status_codes: - raise AirbyteTracedException( - message="Config validation error: " + user_error_message, - internal_message=str(e), - failure_type=FailureType.config_error, - ) from e if not (self.skip_http_status_codes and e.response.status_code in self.skip_http_status_codes): raise e - - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == requests.codes.bad_request: - # Refernce issue: https://github.com/airbytehq/oncall/issues/2133 - # we should skip the slice with `board id` which doesn't support `sprints` - # it's generally applied to all streams that might have the same error hit in the future. - errors = response.json().get("errorMessages") - self.logger.error(f"Stream `{self.name}`. An error occured, details: {errors}. Skipping.") - setattr(self, "raise_on_http_errors", False) - return False - else: - # for all other HTTP errors the defaul handling is applied - return super().should_retry(response) + errors = e.response.json().get("errorMessages") + custom_error = self._get_custom_error(e.response) + self.logger.warning(f"Stream `{self.name}`. An error occurred, details: {errors}. Skipping for now. {custom_error}") class StartDateJiraStream(JiraStream, ABC): - def __init__(self, start_date: Optional[pendulum.DateTime] = None, **kwargs): + def __init__( + self, + start_date: Optional[pendulum.DateTime] = None, + lookback_window_minutes: pendulum.Duration = pendulum.duration(minutes=0), + **kwargs, + ): super().__init__(**kwargs) + self._lookback_window_minutes = lookback_window_minutes self._start_date = start_date @@ -167,7 +177,7 @@ def _get_starting_point(self, stream_state: Mapping[str, Any]) -> Optional[pendu if stream_state: stream_state_value = stream_state.get(self.cursor_field) if stream_state_value: - stream_state_value = pendulum.parse(stream_state_value) + stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes return safe_max(stream_state_value, self._start_date) return self._start_date @@ -191,10 +201,6 @@ class ApplicationRoles(JiraStream): """ primary_key = "key" - skip_http_status_codes = [ - # Application access permissions can only be edited or viewed by administrators. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "applicationrole" @@ -221,11 +227,6 @@ class Boards(JiraStream): https://developer.atlassian.com/cloud/jira/software/rest/api-group-other-operations/#api-agile-1-0-board-get """ - skip_http_status_codes = [ - # for user that have no valid license - requests.codes.FORBIDDEN - ] - extract_field = "values" use_cache = True api_v1 = True @@ -247,7 +248,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, return record -class BoardIssues(IncrementalJiraStream): +class BoardIssues(StartDateJiraStream): """ https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-issue-get """ @@ -258,6 +259,7 @@ class BoardIssues(IncrementalJiraStream): def __init__(self, **kwargs): super().__init__(**kwargs) + self._starting_point_cache = {} self.boards_stream = Boards(authenticator=self.authenticator, domain=self._domain, projects=self._projects) def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: @@ -271,11 +273,17 @@ def request_params( ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) params["fields"] = ["key", "created", "updated"] - jql = self.jql_compare_date(stream_state) + jql = self.jql_compare_date(stream_state, stream_slice) if jql: params["jql"] = jql return params + def jql_compare_date(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[str]: + compare_date = self.get_starting_point(stream_state, stream_slice) + if compare_date: + compare_date = compare_date.strftime("%Y/%m/%d %H:%M") + return f"{self.cursor_field} >= '{compare_date}'" + def _is_board_error(self, response): """Check if board has error and should be skipped""" if response.status_code == 500: @@ -289,17 +297,44 @@ def should_retry(self, response: requests.Response) -> bool: # for all other HTTP errors the default handling is applied return super().should_retry(response) + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from read_full_refresh(self.boards_stream) + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - for board in read_full_refresh(self.boards_stream): - try: - yield from super().read_records(stream_slice={"board_id": board["id"]}, **kwargs) - except HTTPError as e: - if self._is_board_error(e.response): - # Wrong board is skipped - self.logger.warning(f"Board {board['id']} has no columns with a mapped status. Skipping.") - continue - else: - raise + try: + yield from super().read_records(stream_slice={"board_id": stream_slice["id"]}, **kwargs) + except HTTPError as e: + if self._is_board_error(e.response): + # Wrong board is skipped + self.logger.warning(f"Board {stream_slice['id']} has no columns with a mapped status. Skipping.") + else: + raise + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + updated_state = latest_record[self.cursor_field] + board_id = str(latest_record["boardId"]) + stream_state_value = current_stream_state.get(board_id, {}).get(self.cursor_field) + if stream_state_value: + updated_state = max(updated_state, stream_state_value) + current_stream_state.setdefault(board_id, {})[self.cursor_field] = updated_state + return current_stream_state + + def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[pendulum.DateTime]: + board_id = str(stream_slice["board_id"]) + if self.cursor_field not in self._starting_point_cache: + self._starting_point_cache.setdefault(board_id, {})[self.cursor_field] = self._get_starting_point( + stream_state=stream_state, stream_slice=stream_slice + ) + return self._starting_point_cache[board_id][self.cursor_field] + + def _get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[pendulum.DateTime]: + if stream_state: + board_id = str(stream_slice["board_id"]) + stream_state_value = stream_state.get(board_id, {}).get(self.cursor_field) + if stream_state_value: + stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes + return safe_max(stream_state_value, self._start_date) + return self._start_date def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: record["boardId"] = stream_slice["board_id"] @@ -379,8 +414,9 @@ class Issues(IncrementalJiraStream): use_cache = True _expand_fields_list = ["renderedFields", "transitions", "changelog"] - skip_http_status_codes = [requests.codes.FORBIDDEN] - state_checkpoint_interval = 50 # default page size is 50 + # Issue: https://github.com/airbytehq/airbyte/issues/26712 + # we should skip the slice with wrong permissions on project level + skip_http_status_codes = [requests.codes.FORBIDDEN, requests.codes.BAD_REQUEST] def __init__(self, **kwargs): super().__init__(**kwargs) @@ -437,20 +473,10 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: else: yield from super().stream_slices(**kwargs) - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == requests.codes.bad_request: - # Issue: https://github.com/airbytehq/airbyte/issues/26712 - # we should skip the slice with wrong permissions on project level - errors = response.json().get("errorMessages") - self.logger.error( - f"Stream `{self.name}`. An error occurred, details: {errors}." - f"Check permissions for this project. Skipping for now. " - f"The user doesn't have permission to the project. Please grant the user to the project." - ) - setattr(self, "raise_on_http_errors", False) - return False - else: - return super().should_retry(response) + def _get_custom_error(self, response: requests.Response) -> str: + if response.status_code == requests.codes.BAD_REQUEST: + return "The user doesn't have permission to the project. Please grant the user to the project." + return "" class IssueComments(IncrementalJiraStream): @@ -508,10 +534,6 @@ class IssueFieldConfigurations(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access field configurations - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "fieldconfiguration" @@ -530,6 +552,7 @@ class IssueCustomFieldContexts(JiraStream): requests.codes.NOT_FOUND, # Only Jira administrators can access custom field contexts. requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, ] def __init__(self, **kwargs): @@ -561,6 +584,7 @@ class IssueCustomFieldOptions(JiraStream): requests.codes.NOT_FOUND, # Only Jira administrators can access custom field options. requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, ] extract_field = "values" @@ -602,10 +626,6 @@ class IssueNavigatorSettings(JiraStream): """ primary_key = None - skip_http_status_codes = [ - # You need Administrator permission to perform this operation. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "settings/columns" @@ -642,7 +662,8 @@ class IssuePropertyKeys(JiraStream): use_cache = True skip_http_status_codes = [ # Issue does not exist or you do not have permission to see it. - requests.codes.NOT_FOUND + requests.codes.NOT_FOUND, + requests.codes.BAD_REQUEST, ] def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: @@ -730,10 +751,6 @@ class IssueSecuritySchemes(JiraStream): """ extract_field = "issueSecuritySchemes" - skip_http_status_codes = [ - # You need to be a Jira administrator to perform this operation - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuesecurityschemes" @@ -754,10 +771,6 @@ class IssueTypeSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access issue type schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuetypescheme" @@ -769,10 +782,6 @@ class IssueTypeScreenSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access issue type screen schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuetypescreenscheme" @@ -855,7 +864,8 @@ class IssueWatchers(StartDateJiraStream): primary_key = None skip_http_status_codes = [ # Issue is not found or the user does not have permission to view it. - requests.codes.NOT_FOUND + requests.codes.NOT_FOUND, + requests.codes.BAD_REQUEST, ] def __init__(self, **kwargs): @@ -912,11 +922,6 @@ class JiraSettings(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jira-settings/#api-rest-api-3-application-properties-get """ - skip_http_status_codes = [ - # No permission - requests.codes.FORBIDDEN - ] - def path(self, **kwargs) -> str: return "application-properties" @@ -943,10 +948,6 @@ class Permissions(JiraStream): extract_field = "permissions" primary_key = "key" - skip_http_status_codes = [ - # You need to have Administer permissions to view this resource - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "permissions" @@ -996,6 +997,12 @@ class ProjectAvatars(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-avatars/#api-rest-api-3-project-projectidorkey-avatars-get """ + skip_http_status_codes = [ + # Project is not found or the user does not have permission to view the project. + requests.codes.UNAUTHORIZED, + requests.codes.NOT_FOUND, + ] + def __init__(self, **kwargs): super().__init__(**kwargs) self.projects_stream = Projects(authenticator=self.authenticator, domain=self._domain, projects=self._projects) @@ -1021,6 +1028,12 @@ class ProjectCategories(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-categories/#api-rest-api-3-projectcategory-get """ + skip_http_status_codes = [ + # Project is not found or the user does not have permission to view the project. + requests.codes.UNAUTHORIZED, + requests.codes.NOT_FOUND, + ] + def path(self, **kwargs) -> str: return "projectCategory" @@ -1052,7 +1065,8 @@ class ProjectEmail(JiraStream): primary_key = "projectId" skip_http_status_codes = [ # You cannot edit the configuration of this project. - requests.codes.FORBIDDEN + requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, ] def __init__(self, **kwargs): @@ -1100,10 +1114,6 @@ class ProjectRoles(JiraStream): """ primary_key = "id" - skip_http_status_codes = [ - # Application access permissions can only be edited or viewed by administrators. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "role" @@ -1212,10 +1222,6 @@ class Screens(JiraStream): extract_field = "values" use_cache = True - skip_http_status_codes = [ - # Only Jira administrators can manage screens. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "screens" @@ -1289,10 +1295,6 @@ class ScreenSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access screen schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "screenscheme" @@ -1311,28 +1313,17 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.boards_stream = Boards(authenticator=self.authenticator, domain=self._domain, projects=self._projects) - def get_user_message_from_error_message(self, errors: List[str]) -> str: - for error_message in errors: - if "The board does not support sprints" in error_message: - return ( - "The board does not support sprints. The board does not have a sprint board. if it's a team-managed one, " - "does it have sprints enabled under project settings? If it's a company-managed one," - " check that it has at least one Scrum board associated with it." - ) - - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == requests.codes.bad_request: + def _get_custom_error(self, response: requests.Response) -> str: + if response.status_code == requests.codes.BAD_REQUEST: errors = response.json().get("errorMessages") - message = self.get_user_message_from_error_message(errors) - if message: - self.logger.error( - f"Stream `{self.name}`. An error occurred, details: {errors}." - f"Skipping for now. {self.get_user_message_from_error_message(errors)}" - ) - setattr(self, "raise_on_http_errors", False) - return False - else: - return super().should_retry(response) + for error_message in errors: + if "The board does not support sprints" in error_message: + return ( + "The board does not support sprints. The board does not have a sprint board. if it's a team-managed one, " + "does it have sprints enabled under project settings? If it's a company-managed one," + " check that it has at least one Scrum board associated with it." + ) + return "" def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: return f"board/{stream_slice['board_id']}/sprint" @@ -1409,10 +1400,6 @@ class TimeTracking(JiraStream): """ primary_key = "key" - skip_http_status_codes = [ - # This resource is only available to administrators - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "configuration/timetracking/list" @@ -1466,10 +1453,6 @@ class Workflows(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access workflows. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "workflow/search" @@ -1481,10 +1464,6 @@ class WorkflowSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access workflow scheme associations. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "workflowscheme" @@ -1495,11 +1474,6 @@ class WorkflowStatuses(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-statuses/#api-rest-api-3-status-get """ - skip_http_status_codes = [ - # for user that have no valid license - requests.codes.FORBIDDEN - ] - def path(self, **kwargs) -> str: return "status" diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py index 595e1af857d8..4421dbc4641f 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py @@ -313,6 +313,11 @@ def issue_custom_field_contexts_response(): return json.loads(load_file("issue_custom_field_contexts.json")) +@fixture +def issue_custom_field_options_response(): + return json.loads(load_file("issue_custom_field_options.json")) + + @fixture def issue_property_keys_response(): return json.loads(load_file("issue_property_keys.json")) @@ -378,6 +383,17 @@ def mock_projects_responses(config, projects_response): ) +@fixture +def mock_projects_responses_additional_project(config, projects_response): + Projects.use_cache = False + projects_response["values"] += [{"id": "3", "key": "Project3"}, {"id": "4", "key": "Project4"}] + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", + json=projects_response, + ) + + @fixture def mock_issues_responses(config, issues_response): responses.add( @@ -410,6 +426,141 @@ def mock_issues_responses(config, issues_response): ], json={}, ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/search", + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (3) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], + json={"errorMessages": ["The value '3' does not exist for the field 'project'."]}, + status=400, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/search", + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (4) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], + json={ + "issues": [ + { + "key": "TESTKEY13-2", + "fields": { + "project": { + "id": "10016", + "key": "TESTKEY13", + }, + "created": "2022-06-09T16:29:31.871-0700", + "updated": "2022-12-08T02:22:18.889-0800", + }, + } + ] + }, + ) + + +@fixture +def mock_project_emails(config, project_email_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/1/email?maxResults=50", + json=project_email_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/2/email?maxResults=50", + json=project_email_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/3/email?maxResults=50", + json={"errorMessages": ["No access to emails for project 3"]}, + status=403, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/4/email?maxResults=50", + json=project_email_response, + ) + + +@fixture +def mock_issue_watchers_responses(config, issue_watchers_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/watchers?maxResults=50", + json=issue_watchers_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-2/watchers?maxResults=50", + json={"errorMessages": ["Not found watchers for issue TESTKEY13-2"]}, + status=404, + ) + + +@fixture +def mock_issue_custom_field_contexts_response(config, issue_custom_field_contexts_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", + json=issue_custom_field_contexts_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50", + json={}, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50", + json={}, + ) + + +@fixture +def mock_issue_custom_field_contexts_response_error(config, issue_custom_field_contexts_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", + json=issue_custom_field_contexts_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50", + json={"errorMessages": ["Not found issue custom field context for issue fields issuetype2"]}, + status=404, + ) + responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50", json={}) + + +@fixture +def mock_issue_custom_field_options_response(config, issue_custom_field_options_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context/10130/option?maxResults=50", + json=issue_custom_field_options_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context/10129/option?maxResults=50", + json={"errorMessages": ["Not found issue custom field options for issue fields issuetype3"]}, + status=404, + ) @fixture diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json index ead4a03913ba..39e40191fcd1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json @@ -5,14 +5,16 @@ "name": "Default Configuration Scheme for Account", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, - "isAnyIssueType": true + "isAnyIssueType": true, + "fieldType": "option" }, { "id": "10129", "name": "Default Configuration Scheme for Team", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, - "isAnyIssueType": true + "isAnyIssueType": true, + "fieldType": "option" } ] } diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json new file mode 100644 index 000000000000..2be8d7e1c7e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json @@ -0,0 +1,11 @@ +{ + "values": [ + { + "id": "10016", + "value": "To Do", + "disabled": false, + "fieldId": "customfield_10012", + "contextId": "10112" + } + ] +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json index 05a312fc78a8..37ac23f16ab3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json @@ -23,7 +23,7 @@ "searchable": true, "clauseNames": ["issuetype", "type"], "schema": { - "type": "issuetype", + "type": "option", "system": "issuetype" } }, @@ -36,5 +36,33 @@ "navigable": true, "searchable": false, "clauseNames": ["parent"] + }, + { + "id": "issuetype2", + "key": "issuetype2", + "name": "Issue Type2", + "custom": true, + "orderable": true, + "navigable": true, + "searchable": true, + "clauseNames": ["issuetype", "type"], + "schema": { + "type": "option", + "system": "issuetype" + } + }, + { + "id": "issuetype3", + "key": "issuetype3", + "name": "Issue Type3", + "custom": true, + "orderable": true, + "navigable": true, + "searchable": true, + "clauseNames": ["issuetype", "type"], + "schema": { + "type": "option", + "system": "issuetype" + } } ] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py index be4177b5a9bf..4cec82b00478 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py @@ -19,7 +19,7 @@ def test_streams(config): @responses.activate -def test_check_connection(config, projects_response, labels_response): +def test_check_connection_config_no_access_to_one_stream(config, caplog, projects_response, avatars_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", @@ -27,29 +27,18 @@ def test_check_connection(config, projects_response, labels_response): ) responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/label?maxResults=50", - json=labels_response, + f"https://{config['domain']}/rest/api/3/applicationrole?maxResults=50", + status=401, ) - source = SourceJira() - logger_mock = MagicMock() - - assert source.check_connection(logger=logger_mock, config=config) == (True, None) - - -@responses.activate -def test_check_connection_config_error(config, caplog): responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", - status=401, + f"https://{config['domain']}/rest/api/3/avatar/issuetype/system?maxResults=50", + json=avatars_response, ) responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/label?maxResults=50", status=401) source = SourceJira() logger_mock = MagicMock() - with pytest.raises(AirbyteTracedException): - source.check_connection(logger=logger_mock, config=config) - - assert "Invalid creds were provided, please check your api token, domain and/or email." in caplog.text + assert source.check_connection(logger=logger_mock, config=config) == (True, None) @responses.activate diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index c3166cceea9c..00675fa25ab1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -2,11 +2,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + +import pendulum import pytest import requests import responses from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests.exceptions import HTTPError from responses import matchers from source_jira.source import SourceJira @@ -21,6 +23,7 @@ Groups, IssueComments, IssueCustomFieldContexts, + IssueCustomFieldOptions, IssueFieldConfigurations, IssueFields, IssueLinkTypes, @@ -69,10 +72,14 @@ def test_application_roles_stream_401_error(config, caplog): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ApplicationRoles(**args) - with pytest.raises(AirbyteTracedException) as e: - [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert e.value.message == "Config validation error: Invalid creds were provided, please check your api token, domain and/or email." - assert "Invalid creds were provided, please check your api token, domain and/or email." in caplog.text + + is_available, reason = stream.check_availability(logger=logging.Logger, source=SourceJira()) + + assert is_available is False + + assert reason == ( + "Unable to read application_roles stream. The endpoint https://test_application_domain/rest/api/3/applicationrole?maxResults=50 returned 401: Unauthorized. Invalid creds were provided, please check your api token, domain and/or email.. Please visit https://docs.airbyte.com/integrations/sources/jira to learn more. " + ) @responses.activate @@ -128,9 +135,19 @@ def test_board_stream_forbidden(config, boards_response, caplog): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Boards(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert records == [] - assert "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access." in caplog.text + is_available, reason = stream.check_availability(logger=logging.Logger, source=SourceJira()) + + assert is_available is False + + assert reason == ( + "Unable to read boards stream. The endpoint " + "https://test_boards_domain/rest/agile/1.0/board?maxResults=50 returned 403: " + "Forbidden. Please check the 'READ' permission(Scopes for Connect apps) " + "and/or the user has Jira Software rights and access.. Please visit " + "https://docs.airbyte.com/integrations/sources/jira to learn more. " + "403 Client Error: Forbidden for url: " + "https://test_boards_domain/rest/agile/1.0/board?maxResults=50" + ) @responses.activate @@ -185,7 +202,7 @@ def test_issues_fields_stream(config, mock_fields_response): stream = IssueFields(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 3 + assert len(records) == 5 assert len(responses.calls) == 1 @@ -352,7 +369,7 @@ def test_board_issues_stream(config, mock_board_response, board_issues_response) responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/2/issue?maxResults=50&fields=key&fields=created&fields=updated", - json={'errorMessages': ['This board has no columns with a mapped status.'], 'errors': {}}, + json={"errorMessages": ["This board has no columns with a mapped status."], "errors": {}}, status=500, ) responses.add( @@ -364,7 +381,7 @@ def test_board_issues_stream(config, mock_board_response, board_issues_response) authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = BoardIssues(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.incremental)] + records = list(read_full_refresh(stream)) assert len(records) == 1 assert len(responses.calls) == 4 @@ -374,10 +391,10 @@ def test_stream_updated_state(config): args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = BoardIssues(**args) - current_stream_state = {"updated": "09.11.2023"} - latest_record = {"updated": "10.11.2023"} + current_stream_state = {"22": {"updated": "2023-10-01T00:00:00Z"}} + latest_record = {"boardId": 22, "updated": "2023-09-01T00:00:00Z"} - assert {"updated": "10.11.2023"} == stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) + assert {"22": {"updated": "2023-10-01T00:00:00Z"}} == stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) @responses.activate @@ -397,13 +414,7 @@ def test_filter_sharing_stream(config, mock_filter_response, filter_sharing_resp @responses.activate -def test_projects_stream(config, projects_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", - json=projects_response, - ) - +def test_projects_stream(config, mock_projects_responses): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Projects(**args) @@ -412,12 +423,7 @@ def test_projects_stream(config, projects_response): @responses.activate -def test_projects_avatars_stream(config, projects_response, projects_avatars_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", - json=projects_response, - ) +def test_projects_avatars_stream(config, mock_projects_responses, projects_avatars_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/project/Project1/avatars?maxResults=50", @@ -490,16 +496,26 @@ def test_sprints_stream(config, mock_board_response, mock_sprints_response): @responses.activate -def test_board_does_not_support_sprints(config, caplog): - url = f"https://{config['domain']}/rest/agile/1.0/board/4/sprint?maxResults=50" +def test_board_does_not_support_sprints(config, mock_board_response, sprints_response, caplog): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board/1/sprint?maxResults=50", + json=sprints_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board/3/sprint?maxResults=50", + json=sprints_response, + ) + url = f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50" error = {"errorMessages": ["The board does not support sprints"], "errors": {}} responses.add(responses.GET, url, json=error, status=400) authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Sprints(**args) - response = requests.get(url) - actual = stream.should_retry(response) - assert actual is False + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] + assert len(records) == 2 + assert ( "The board does not support sprints. The board does not have a sprint board. if it's a team-managed one, " "does it have sprints enabled under project settings? If it's a company-managed one," @@ -658,39 +674,16 @@ def test_avatars_stream_should_retry(config, caplog): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Avatars(**args) + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"avatar_type": "issuetype"})] + assert len(records) == 0 - response = requests.get(url) - actual = stream.should_retry(response) - assert actual is False assert "The error message" in caplog.text @responses.activate -def test_issues_stream(config, projects_response, mock_issues_responses, issues_response, caplog): - projects_response["values"].append({"id": "3", "key": "Project1"}) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", - json=projects_response, - ) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/search", - match=[ - matchers.query_param_matcher( - { - "maxResults": 50, - "fields": "*all", - "jql": "project in (3) ORDER BY updated asc", - "expand": "renderedFields,transitions,changelog", - } - ) - ], - json={"errorMessages": ["The value '3' does not exist for the field 'project'."]}, - status=400, - ) +def test_issues_stream(config, mock_projects_responses_additional_project, mock_issues_responses, caplog): authenticator = SourceJira().get_authenticator(config=config) - args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"]} stream = Issues(**args) records = list(read_full_refresh(stream)) assert len(records) == 1 @@ -700,9 +693,25 @@ def test_issues_stream(config, projects_response, mock_issues_responses, issues_ assert "non_empty_field" in records[0]["fields"] assert len(responses.calls) == 3 - error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"].Check permissions for this project. Skipping for now. The user doesn't have permission to the project. Please grant the user to the project." + error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"]. Skipping for now. The user doesn't have permission to the project. Please grant the user to the project." assert error_message in caplog.messages +@pytest.mark.parametrize( + "start_date, lookback_window, stream_state, expected_query", + [ + (pendulum.parse("2023-09-09T00:00:00Z"), 0, None, None), + (None, 10, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:37'"), + (None, 0, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:47'") + ] +) +def test_issues_stream_jql_compare_date(config, start_date, lookback_window, stream_state, expected_query, caplog): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"], + "lookback_window_minutes": pendulum.duration(minutes=lookback_window)} + stream = Issues(**args) + assert stream.jql_compare_date(stream_state) == expected_query + + @responses.activate def test_issue_comments_stream(config, mock_projects_responses, mock_issues_responses, issue_comments_response): @@ -721,19 +730,13 @@ def test_issue_comments_stream(config, mock_projects_responses, mock_issues_resp @responses.activate -def test_issue_custom_field_contexts_stream(config, mock_fields_response, issue_custom_field_contexts_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", - json=issue_custom_field_contexts_response, - ) - +def test_issue_custom_field_contexts_stream(config, mock_fields_response, mock_issue_custom_field_contexts_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssueCustomFieldContexts(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"field_id": "10130"})] + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 2 - assert len(responses.calls) == 2 + assert len(responses.calls) == 4 @responses.activate @@ -807,22 +810,11 @@ def test_project_permissions_stream(config, mock_projects_responses, project_per @responses.activate -def test_project_email_stream(config, mock_projects_responses, project_email_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/1/email?maxResults=50", - json=project_email_response, - ) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/2/email?maxResults=50", - json=project_email_response, - ) - +def test_project_email_stream(config, mock_projects_responses, mock_project_emails): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ProjectEmail(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "TESTKEY13-1"})] + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 2 assert len(responses.calls) == 2 @@ -897,13 +889,7 @@ def test_issue_worklogs_stream(config, mock_projects_responses, mock_issues_resp @responses.activate -def test_issue_watchers_stream(config, mock_projects_responses, mock_issues_responses, issue_watchers_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/watchers?maxResults=50", - json=issue_watchers_response, - ) - +def test_issue_watchers_stream(config, mock_projects_responses, mock_issues_responses, mock_issue_watchers_responses): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssueWatchers(**args) @@ -961,3 +947,66 @@ def test_project_versions_stream(config, mock_projects_responses, projects_versi assert len(records) == 2 assert len(responses.calls) == 2 + + +@pytest.mark.parametrize( + "stream, expected_records_number, expected_calls_number, log_message", + [ + ( + Issues, + 2, + 4, + "Stream `issues`. An error occurred, details: [\"The value '3' does not " + "exist for the field 'project'.\"]. Skipping for now. The user doesn't have " + "permission to the project. Please grant the user to the project.", + ), + ( + IssueCustomFieldContexts, + 2, + 4, + "Stream `issue_custom_field_contexts`. An error occurred, details: ['Not found issue custom field context for issue fields issuetype2']. Skipping for now. ", + ), + ( + IssueCustomFieldOptions, + 1, + 6, + "Stream `issue_custom_field_options`. An error occurred, details: ['Not found issue custom field options for issue fields issuetype3']. Skipping for now. ", + ), + ( + IssueWatchers, + 1, + 6, + "Stream `issue_watchers`. An error occurred, details: ['Not found watchers for issue TESTKEY13-2']. Skipping for now. ", + ), + ( + ProjectEmail, + 4, + 4, + "Stream `project_email`. An error occurred, details: ['No access to emails for project 3']. Skipping for now. ", + ), + ], +) +@responses.activate +def test_skip_slice( + config, + mock_projects_responses_additional_project, + mock_issues_responses, + mock_project_emails, + mock_issue_watchers_responses, + mock_issue_custom_field_contexts_response_error, + mock_issue_custom_field_options_response, + mock_fields_response, + caplog, + stream, + expected_records_number, + expected_calls_number, + log_message, +): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3", "Project4"]} + stream = stream(**args) + records = list(read_full_refresh(stream)) + assert len(records) == expected_records_number + + assert len(responses.calls) == expected_calls_number + assert log_message in caplog.messages diff --git a/airbyte-integrations/connectors/source-k6-cloud/main.py b/airbyte-integrations/connectors/source-k6-cloud/main.py index 5b694eda96f2..61325917a539 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/main.py +++ b/airbyte-integrations/connectors/source-k6-cloud/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_k6_cloud import SourceK6Cloud +from source_k6_cloud.run import run if __name__ == "__main__": - source = SourceK6Cloud() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml index f62b94eacbf6..f0c7664e3897 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml @@ -8,6 +8,10 @@ data: icon: k6cloud.svg license: MIT name: K6 Cloud + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-k6-cloud registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-k6-cloud/setup.py b/airbyte-integrations/connectors/source-k6-cloud/setup.py index 05908924fd5b..03ce14d72e5b 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/setup.py +++ b/airbyte-integrations/connectors/source-k6-cloud/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-k6-cloud=source_k6_cloud.run:run", + ], + }, name="source_k6_cloud", description="Source implementation for K6 Cloud.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py b/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py new file mode 100644 index 000000000000..c5103e21aa21 --- /dev/null +++ b/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_k6_cloud import SourceK6Cloud + + +def run(): + source = SourceK6Cloud() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index 4a3137bec286..b668bc3c575d 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.kafka.KafkaSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -29,7 +19,5 @@ dependencies { implementation 'org.apache.kafka:connect-json:3.2.1' implementation 'io.confluent:kafka-avro-serializer:7.2.1' - testImplementation libs.testcontainers.kafka - - integrationTestJavaImplementation libs.testcontainers.kafka + testImplementation 'org.testcontainers:kafka:1.19.4' } diff --git a/airbyte-integrations/connectors/source-kafka/metadata.yaml b/airbyte-integrations/connectors/source-kafka/metadata.yaml index 72575793c636..aedf1844ccaf 100644 --- a/airbyte-integrations/connectors/source-kafka/metadata.yaml +++ b/airbyte-integrations/connectors/source-kafka/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: source definitionId: d917a47b-8537-4d0d-8c10-36a9928d4265 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 dockerRepository: airbyte/source-kafka githubIssueLabel: source-kafka icon: kafka.svg diff --git a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java index 2ae90e827d74..7ecdfd5e8b41 100644 --- a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java @@ -10,9 +10,11 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.string.Strings; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -22,6 +24,7 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.AdminClient; @@ -32,16 +35,20 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.connect.json.JsonSerializer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.utility.DockerImageName; +@Disabled("need to fix docker container networking") public class KafkaSourceAcceptanceTest extends SourceAcceptanceTest { private static final ObjectMapper mapper = MoreMappers.initMapper(); - private static final String TOPIC_NAME = "test.topic"; private static KafkaContainer KAFKA; + private String topicName; + @Override protected String getImageName() { return "airbyte/source-kafka:dev"; @@ -53,10 +60,11 @@ protected JsonNode getConfig() { final ObjectNode subscriptionConfig = mapper.createObjectNode(); protocolConfig.put("security_protocol", KafkaProtocol.PLAINTEXT.toString()); subscriptionConfig.put("subscription_type", "subscribe"); - subscriptionConfig.put("topic_pattern", TOPIC_NAME); + subscriptionConfig.put("topic_pattern", topicName); + var bootstrapServers = String.format("PLAINTEXT://%s:%d", HostPortResolver.resolveHost(KAFKA), HostPortResolver.resolvePort(KAFKA)); return Jsons.jsonNode(ImmutableMap.builder() - .put("bootstrap_servers", KAFKA.getBootstrapServers()) + .put("bootstrap_servers", bootstrapServers) .put("subscription", subscriptionConfig) .put("client_dns_lookup", "use_all_dns_ips") .put("enable_auto_commit", false) @@ -67,11 +75,15 @@ protected JsonNode getConfig() { .build()); } - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + @BeforeAll + static public void setupContainer() { KAFKA = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.0")); KAFKA.start(); + } + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + topicName = Strings.addRandomSuffix("topic.test", "_", 10); createTopic(); sendEvent(); } @@ -87,7 +99,7 @@ private void sendEvent() throws ExecutionException, InterruptedException { final ObjectNode event = mapper.createObjectNode(); event.put("test", "value"); - producer.send(new ProducerRecord<>(TOPIC_NAME, event), (recordMetadata, exception) -> { + producer.send(new ProducerRecord<>(topicName, event), (recordMetadata, exception) -> { if (exception != null) { throw new RuntimeException("Cannot send message to Kafka. Error: " + exception.getMessage(), exception); } @@ -96,14 +108,18 @@ private void sendEvent() throws ExecutionException, InterruptedException { private void createTopic() throws Exception { try (final var admin = AdminClient.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()))) { - final NewTopic topic = new NewTopic(TOPIC_NAME, 1, (short) 1); + final NewTopic topic = new NewTopic(topicName, 1, (short) 1); admin.createTopics(Collections.singletonList(topic)).all().get(); } } @Override protected void tearDown(final TestDestinationEnv testEnv) { - KAFKA.close(); + try (final var admin = AdminClient.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()))) { + admin.deleteTopics(List.of(topicName)).all().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override @@ -114,7 +130,7 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { final ConfiguredAirbyteStream streams = - CatalogHelpers.createConfiguredAirbyteStream(TOPIC_NAME, null, Field.of("value", JsonSchemaType.STRING)); + CatalogHelpers.createConfiguredAirbyteStream(topicName, null, Field.of("value", JsonSchemaType.STRING)); streams.setSyncMode(SyncMode.FULL_REFRESH); return new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(streams)); } diff --git a/airbyte-integrations/connectors/source-klarna/main.py b/airbyte-integrations/connectors/source-klarna/main.py index 4abe77680d8b..566bf5c666d6 100644 --- a/airbyte-integrations/connectors/source-klarna/main.py +++ b/airbyte-integrations/connectors/source-klarna/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klarna import SourceKlarna +from source_klarna.run import run if __name__ == "__main__": - source = SourceKlarna() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klarna/metadata.yaml b/airbyte-integrations/connectors/source-klarna/metadata.yaml index 8eff9ee1fc7b..caf42f707603 100644 --- a/airbyte-integrations/connectors/source-klarna/metadata.yaml +++ b/airbyte-integrations/connectors/source-klarna/metadata.yaml @@ -5,6 +5,10 @@ data: - api.playground.klarna.com - api-${config.region}.klarna.com - api-${config.region}.playground.klarna.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klarna registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-klarna/setup.py b/airbyte-integrations/connectors/source-klarna/setup.py index a4742e88dd56..58609f375fe3 100644 --- a/airbyte-integrations/connectors/source-klarna/setup.py +++ b/airbyte-integrations/connectors/source-klarna/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-klarna=source_klarna.run:run", + ], + }, name="source_klarna", description="Source implementation for Klarna.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/run.py b/airbyte-integrations/connectors/source-klarna/source_klarna/run.py new file mode 100644 index 000000000000..8ae916c3c4e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klarna import SourceKlarna + + +def run(): + source = SourceKlarna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaus-api/main.py b/airbyte-integrations/connectors/source-klaus-api/main.py index 9be6460ab485..7896d99aec94 100644 --- a/airbyte-integrations/connectors/source-klaus-api/main.py +++ b/airbyte-integrations/connectors/source-klaus-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klaus_api import SourceKlausApi +from source_klaus_api.run import run if __name__ == "__main__": - source = SourceKlausApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klaus-api/metadata.yaml b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml index 24e538bf888a..2f4dbc01d8cf 100644 --- a/airbyte-integrations/connectors/source-klaus-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klaus-api registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-klaus-api/setup.py b/airbyte-integrations/connectors/source-klaus-api/setup.py index b4444f99dd06..815b95129848 100644 --- a/airbyte-integrations/connectors/source-klaus-api/setup.py +++ b/airbyte-integrations/connectors/source-klaus-api/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-klaus-api=source_klaus_api.run:run", + ], + }, name="source_klaus_api", description="Source implementation for Klaus Api.", author="Deke Li", author_email="deke.li@sendinblue.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py new file mode 100644 index 000000000000..c000d3864646 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klaus_api import SourceKlausApi + + +def run(): + source = SourceKlausApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaviyo/README.md b/airbyte-integrations/connectors/source-klaviyo/README.md index 61e657f87278..fa1d8bad1891 100644 --- a/airbyte-integrations/connectors/source-klaviyo/README.md +++ b/airbyte-integrations/connectors/source-klaviyo/README.md @@ -1,102 +1,91 @@ -# Klaviyo Source +# Klaviyo source connector -This is the repository for the Klaviyo source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/klaviyo). -## API Version Migration -This source is currently migrating from the v1 of Klaviyo API to the latest stable API version, v2023-02-22. Review the [api comparison chart](https://developers.klaviyo.com/en/v2022-10-17/docs/apis_comparison_chart) to see major changes. +This is the repository for the Klaviyo source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/klaviyo). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/klaviyo) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klaviyo/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/klaviyo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klaviyo/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source klaviyo test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-klaviyo spec +poetry run source-klaviyo check --config secrets/config.json +poetry run source-klaviyo discover --config secrets/config.json +poetry run source-klaviyo read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-klaviyo build ``` -An image will be built with the tag `airbyte/source-klaviyo:dev`. +An image will be available on your host with the tag `airbyte/source-klaviyo:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-klaviyo:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-klaviyo:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaviyo:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaviyo:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-klaviyo:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-klaviyo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-klaviyo test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-klaviyo test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/klaviyo.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/klaviyo.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl index 419da2f2429b..24cc25ea2160 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl @@ -10,35 +10,12 @@ {"stream": "events", "data": {"type": "event", "id": "3qvdgs9P", "attributes": {"timestamp": 1621295125, "event_properties": {"$event_id": "1621295125"}, "datetime": "2021-05-17 23:45:25+00:00", "uuid": "f3859880-b769-11eb-8001-f6a061424b91"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/"}, "datetime": "2021-05-17 23:45:25+00:00"}, "emitted_at": 1699980660457} {"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGPSXF1N23RBJZ947R1N1", "attributes": {"email": "some.email.that.dont.exist.8@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 8", "last_name": "Last Name 8", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1663367161413} {"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGQ6X21SSWPGRDK9QK97C", "attributes": {"email": "some.email.that.dont.exist.9@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 9", "last_name": "Last Name 9", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:28+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:28+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:20:01+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1663367161413} -{"stream": "lists", "data": {"type": "list", "id": "RnsiHB", "attributes": {"name": "Newsletter", "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/"}, "updated": "2021-03-31T10:50:36+00:00"}, "emitted_at": 1698942733516} -{"stream": "lists", "data": {"type": "list", "id": "TaSce6", "attributes": {"name": "Preview List", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TaSce6/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TaSce6/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698942733517} -{"stream": "lists", "data": {"type": "list", "id": "R2p3ry", "attributes": {"name": "Test2", "created": "2021-11-16T14:24:04+00:00", "updated": "2021-11-16T14:24:04+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/"}, "updated": "2021-11-16T14:24:04+00:00"}, "emitted_at": 1698942733517} -{"stream": "lists", "data": {"type": "list", "id": "S7aBY2", "attributes": {"name": "Test1", "created": "2021-11-16T14:24:07+00:00", "updated": "2021-11-16T14:24:07+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/S7aBY2/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/S7aBY2/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/"}, "updated": "2021-11-16T14:24:07+00:00"}, "emitted_at": 1698942733518} -{"stream": "lists", "data": {"type": "list", "id": "XpP2a5", "attributes": {"name": "Test4", "created": "2021-11-16T14:24:10+00:00", "updated": "2021-11-16T14:24:10+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XpP2a5/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XpP2a5/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/"}, "updated": "2021-11-16T14:24:10+00:00"}, "emitted_at": 1698942733518} -{"stream": "lists", "data": {"type": "list", "id": "TDGJsj", "attributes": {"name": "Test3", "created": "2021-11-16T14:24:14+00:00", "updated": "2021-11-16T14:24:14+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TDGJsj/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TDGJsj/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/"}, "updated": "2021-11-16T14:24:14+00:00"}, "emitted_at": 1698942733518} -{"stream": "lists", "data": {"type": "list", "id": "WBxsQE", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:17+00:00", "updated": "2021-11-16T14:24:17+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/WBxsQE/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/WBxsQE/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/"}, "updated": "2021-11-16T14:24:17+00:00"}, "emitted_at": 1698942733518} -{"stream": "lists", "data": {"type": "list", "id": "VmvmBq", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:18+00:00", "updated": "2021-11-16T14:24:18+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VmvmBq/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VmvmBq/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/"}, "updated": "2021-11-16T14:24:18+00:00"}, "emitted_at": 1698942733519} -{"stream": "lists", "data": {"type": "list", "id": "XGj3p8", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:20+00:00", "updated": "2021-11-16T14:24:20+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XGj3p8/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XGj3p8/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/"}, "updated": "2021-11-16T14:24:20+00:00"}, "emitted_at": 1698942733520} -{"stream": "lists", "data": {"type": "list", "id": "R4ZhCr", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:21+00:00", "updated": "2021-11-16T14:24:21+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/"}, "updated": "2021-11-16T14:24:21+00:00"}, "emitted_at": 1698942733520} -{"stream": "lists", "data": {"type": "list", "id": "Seq8wh", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:22+00:00", "updated": "2021-11-16T14:24:22+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/Seq8wh/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/Seq8wh/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/"}, "updated": "2021-11-16T14:24:22+00:00"}, "emitted_at": 1698942733520} -{"stream": "lists", "data": {"type": "list", "id": "TpNXq9", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:23+00:00", "updated": "2021-11-16T14:24:23+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TpNXq9/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TpNXq9/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/"}, "updated": "2021-11-16T14:24:23+00:00"}, "emitted_at": 1698942733521} -{"stream": "lists", "data": {"type": "list", "id": "UzdNhZ", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:24+00:00", "updated": "2021-11-16T14:24:24+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/UzdNhZ/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/UzdNhZ/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/"}, "updated": "2021-11-16T14:24:24+00:00"}, "emitted_at": 1698942733521} -{"stream": "lists", "data": {"type": "list", "id": "TWcKFn", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TWcKFn/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TWcKFn/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/"}, "updated": "2021-11-16T14:24:25+00:00"}, "emitted_at": 1698942733521} -{"stream": "lists", "data": {"type": "list", "id": "Ya5ziX", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/Ya5ziX/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/Ya5ziX/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/"}, "updated": "2021-11-16T14:24:25+00:00"}, "emitted_at": 1698942733521} -{"stream": "lists", "data": {"type": "list", "id": "RwKPyg", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/"}, "updated": "2021-11-16T14:24:26+00:00"}, "emitted_at": 1698942733522} -{"stream": "lists", "data": {"type": "list", "id": "VJCDbR", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VJCDbR/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VJCDbR/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/"}, "updated": "2021-11-16T14:24:26+00:00"}, "emitted_at": 1698942733522} -{"stream": "lists", "data": {"type": "list", "id": "TjbH4K", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:27+00:00", "updated": "2021-11-16T14:24:27+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TjbH4K/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TjbH4K/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/"}, "updated": "2021-11-16T14:24:27+00:00"}, "emitted_at": 1698942733522} -{"stream": "lists", "data": {"type": "list", "id": "VDZnQt", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VDZnQt/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VDZnQt/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/"}, "updated": "2021-11-16T14:24:28+00:00"}, "emitted_at": 1698942733523} -{"stream": "lists", "data": {"type": "list", "id": "WJLXnV", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/WJLXnV/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/WJLXnV/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/"}, "updated": "2021-11-16T14:24:28+00:00"}, "emitted_at": 1698942733523} -{"stream": "lists", "data": {"type": "list", "id": "XUbNgM", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:29+00:00", "updated": "2021-11-16T14:24:29+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XUbNgM/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XUbNgM/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/"}, "updated": "2021-11-16T14:24:29+00:00"}, "emitted_at": 1698942733523} -{"stream": "lists", "data": {"type": "list", "id": "RgS4w6", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/"}, "updated": "2021-11-16T14:24:30+00:00"}, "emitted_at": 1698942733524} -{"stream": "lists", "data": {"type": "list", "id": "UeGLUr", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/UeGLUr/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/UeGLUr/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/"}, "updated": "2021-11-16T14:24:30+00:00"}, "emitted_at": 1698942733524} -{"stream": "lists", "data": {"type": "list", "id": "SYEFFb", "attributes": {"name": "Test_5", "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T14:24:31+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SYEFFb/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SYEFFb/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/"}, "updated": "2021-11-16T14:24:31+00:00"}, "emitted_at": 1698942733524} -{"stream": "lists", "data": {"type": "list", "id": "SmDD4y", "attributes": {"name": "Test5__x", "created": "2021-11-16T14:24:32+00:00", "updated": "2021-11-16T14:24:32+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SmDD4y/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SmDD4y/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/"}, "updated": "2021-11-16T14:24:32+00:00"}, "emitted_at": 1698942733524} -{"stream": "lists", "data": {"type": "list", "id": "X7UeXn", "attributes": {"name": "Test5___", "created": "2021-11-16T14:24:34+00:00", "updated": "2021-11-16T14:24:34+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/X7UeXn/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/X7UeXn/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/"}, "updated": "2021-11-16T14:24:34+00:00"}, "emitted_at": 1698942733525} -{"stream": "lists", "data": {"type": "list", "id": "RPfQMj", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T15:01:15+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/"}, "updated": "2021-11-16T15:01:15+00:00"}, "emitted_at": 1698942733525} -{"stream": "lists", "data": {"type": "list", "id": "S8nmQ9", "attributes": {"name": "Test AAAB", "created": "2021-11-16T15:02:51+00:00", "updated": "2021-11-16T15:02:51+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/S8nmQ9/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/S8nmQ9/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/"}, "updated": "2021-11-16T15:02:51+00:00"}, "emitted_at": 1698942733525} -{"stream": "lists", "data": {"type": "list", "id": "SBYgiK", "attributes": {"name": "SMS Subscribers", "created": "2022-05-31T06:52:26+00:00", "updated": "2022-05-31T06:52:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SBYgiK/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SBYgiK/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/"}, "updated": "2022-05-31T06:52:26+00:00"}, "emitted_at": 1698942733525} +{"stream": "lists", "data": {"type": "list", "id": "R2p3ry", "attributes": {"name": "Test2", "created": "2021-11-16T14:24:04+00:00", "updated": "2021-11-16T14:24:04+00:00", "opt_in_process": "double_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/"}, "updated": "2021-11-16T14:24:04+00:00"}, "emitted_at": 1707338396895} +{"stream": "lists", "data": {"type": "list", "id": "R4ZhCr", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:21+00:00", "updated": "2021-11-16T14:24:21+00:00", "opt_in_process": "double_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/"}, "updated": "2021-11-16T14:24:21+00:00"}, "emitted_at": 1707338396896} +{"stream": "lists", "data": {"type": "list", "id": "RPfQMj", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T15:01:15+00:00", "opt_in_process": "single_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/"}, "updated": "2021-11-16T15:01:15+00:00"}, "emitted_at": 1707338396897} +{"stream": "lists", "data": {"type": "list", "id": "RgS4w6", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00", "opt_in_process": "double_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/"}, "updated": "2021-11-16T14:24:30+00:00"}, "emitted_at": 1707338396897} +{"stream": "lists", "data": {"type": "list", "id": "RnsiHB", "attributes": {"name": "Newsletter", "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00", "opt_in_process": "double_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/"}, "updated": "2021-03-31T10:50:36+00:00"}, "emitted_at": 1707338396897} +{"stream": "lists", "data": {"type": "list", "id": "RwKPyg", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00", "opt_in_process": "double_opt_in"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/"}, "updated": "2021-11-16T14:24:26+00:00"}, "emitted_at": 1707338396898} {"stream": "email_templates", "data": {"type": "template", "id": "RdbN2P", "attributes": {"name": "Newsletter #1 (Images & Text)", "editor_type": "SYSTEM_DRAGGABLE", "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n\n\n\n\n
      \n\n\n\n
      \n\n
      \n
      \n\n\n\n\n
      \n\n\n\n\n
      \n\n SHOP NOW\n \n
      \n
      \n
      \n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      \n

      \n\n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      This template starts with images.

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      Everyone loves pictures. They're more engaging that text by itself and the images in this template will neatly stack on mobile devices for the best viewing experience.

      \n

      Use the text area below to add additional content or add more images to create a larger image gallery. You can drag blocks from the left sidebar to add content to your template. You can customize this colors, fonts and styling of this template to match your brand by clicking the \"Styles\" button to the left.

      \n

      Happy emailing!

      \n

      The Klaviyo Team

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      \n\n
      \n\n
      \n\n\"Facebook\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"Twitter\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"LinkedIn\"\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      No longer want to receive these emails? {% unsubscribe %}.
      {{ organization.name }} {{ organization.full_address }}
      \n
      \n
      \n
      \n
      \n\n
      \n\n
      \n\n
      \n
      \n\n
      \n
      \n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\"Powered\n\n
      \n
      \n
      \n\n
      \n
      \n\n
      \n
      \n\n", "text": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "links": {"self": "https://a.klaviyo.com/api/templates/RdbN2P/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698938827838} {"stream": "metrics", "data": {"type": "metric", "id": "RUQ6YQ", "attributes": {"name": "Active on Site", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RUQ6YQ/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412889} {"stream": "metrics", "data": {"type": "metric", "id": "RhP4nd", "attributes": {"name": "Dropped Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RhP4nd/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412891} diff --git a/airbyte-integrations/connectors/source-klaviyo/main.py b/airbyte-integrations/connectors/source-klaviyo/main.py index 6285473bf2cd..5b8c871c3d7d 100644 --- a/airbyte-integrations/connectors/source-klaviyo/main.py +++ b/airbyte-integrations/connectors/source-klaviyo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klaviyo import SourceKlaviyo +from source_klaviyo.run import run if __name__ == "__main__": - source = SourceKlaviyo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml index 71bcca6adeb0..c647833c8583 100644 --- a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml @@ -8,12 +8,16 @@ data: definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c - dockerImageTag: 2.1.0 + dockerImageTag: 2.1.3 dockerRepository: airbyte/source-klaviyo githubIssueLabel: source-klaviyo icon: klaviyo.svg license: MIT name: Klaviyo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klaviyo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-klaviyo/poetry.lock b/airbyte-integrations/connectors/source-klaviyo/poetry.lock new file mode 100644 index 000000000000..40442646b18a --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.0.tar.gz", hash = "sha256:622f56bd7101493a74f11c33a45a31c251032333989996f137cac8370873c614"}, + {file = "airbyte_cdk-0.62.0-py3-none-any.whl", hash = "sha256:b21330a566b33dbdddde33243eb9855f086ad4272e3585ca626be1225451a3b8"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "d6958c9aa0a930adeb8df2c2a407dd6c5ea9495429478ef15b6fa9ff99a17471" diff --git a/airbyte-integrations/connectors/source-klaviyo/pyproject.toml b/airbyte-integrations/connectors/source-klaviyo/pyproject.toml new file mode 100644 index 000000000000..cec0355dfd2b --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.1.2" +name = "source-klaviyo" +description = "Source implementation for Klaviyo." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/klaviyo" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_klaviyo" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.0" + +[tool.poetry.scripts] +source-klaviyo = "source_klaviyo.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.12.0" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-klaviyo/requirements.txt b/airbyte-integrations/connectors/source-klaviyo/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-klaviyo/setup.py b/airbyte-integrations/connectors/source-klaviyo/setup.py deleted file mode 100644 index 32a31edb0848..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock~=1.8"] - -setup( - name="source_klaviyo", - description="Source implementation for Klaviyo.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py new file mode 100644 index 000000000000..afcae2272e29 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klaviyo import SourceKlaviyo + + +def run(): + source = SourceKlaviyo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json index d568f492604a..d4ac3b074b1a 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -15,22 +16,84 @@ "channel": { "type": "string" }, "audiences": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "included": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "excluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } }, "send_options": { "type": ["null", "object"], "properties": { - "use_smart_sending": { "type": "boolean" } + "ignore_unsubscribes": { "type": ["null", "boolean"] }, + "use_smart_sending": { "type": ["null", "boolean"] } } }, "message": { "type": "string" }, "tracking_options": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "is_tracking_opens": { "type": ["null", "boolean"] }, + "is_tracking_clicks": { "type": ["null", "boolean"] }, + "is_add_utm": { "type": ["null", "boolean"] }, + "utm_params": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { "type": "string" }, + "value": { "type": "string" } + } + } + } + } }, "send_strategy": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "method": { "type": "string" }, + "options_static": { + "type": ["null", "object"], + "properties": { + "datetime": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "is_local": { "type": ["null", "boolean"] }, + "send_past_recipients_immediately": { + "type": ["null", "boolean"] + } + } + }, + "options_throttled": { + "type": ["null", "object"], + "properties": { + "datetime": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "throttle_percentage": { "type": "integer" } + } + }, + "options_sto": { + "type": ["null", "object"], + "properties": { + "date": { "type": "string", "format": "date" } + } + } + } }, "created_at": { "type": ["null", "string"], "format": "date-time" }, "scheduled_at": { "type": ["null", "string"], "format": "date-time" }, @@ -40,11 +103,38 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { "type": "string" } + } }, "relationships": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "tags": { + "type": ["null", "object"], + "properties": { + "data": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": "string" }, + "related": { "type": "string" } + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json index c3590f804f83..ce477df9df88 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -20,7 +21,10 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { "type": "string" } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json index 23006e1f3c36..9a25f1351c55 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -38,7 +39,15 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { + "type": "string" + }, + "related": { + "type": "string" + } + } } } }, @@ -54,7 +63,15 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { + "type": "string" + }, + "related": { + "type": "string" + } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json index e65227382ac1..c5623ab0712c 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -19,11 +20,61 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { "type": "string" } + } }, "relationships": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "flow-actions": { + "type": ["null", "object"], + "properties": { + "data": { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": "string" }, + "related": { "type": "string" } + } + } + } + }, + "tags": { + "type": ["null", "object"], + "properties": { + "data": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": "string" }, + "related": { "type": "string" } + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json index 21f80313fd0a..fcffdb97cbbe 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -11,22 +12,176 @@ "properties": { "email": { "type": ["null", "string"] }, "phone_number": { "type": ["null", "string"] }, + "anonymous_id": { "type": ["null", "string"] }, + "external_id": { "type": ["null", "string"] }, "first_name": { "type": ["null", "string"] }, "last_name": { "type": ["null", "string"] }, - "properties": { - "type": ["null", "object"], - "additionalProperties": true - }, - "subscriptions": { "type": ["null", "object"] }, "organization": { "type": ["null", "string"] }, "title": { "type": ["null", "string"] }, + "image": { "type": ["null", "string"] }, "created": { "type": ["null", "string"], "format": "date-time" }, "updated": { "type": ["null", "string"], "format": "date-time" }, - "last_event_date": { "type": ["null", "string"], "format": "date-time" } + "last_event_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "location": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "latitude": { + "oneOf": [ + { "type": "null" }, + { "type": "number" }, + { "type": "string" } + ] + }, + "longitude": { + "oneOf": [ + { "type": "null" }, + { "type": "number" }, + { "type": "string" } + ] + }, + "region": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "timezone": { "type": ["null", "string"] }, + "ip": { "type": ["null", "string"] } + } + }, + "properties": { + "type": ["null", "object"], + "additionalProperties": true + }, + "subscriptions": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "object"], + "properties": { + "marketing": { + "type": ["null", "object"], + "properties": { + "can_receive_email_marketing": { "type": "boolean" }, + "consent": { "type": "string" }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_updated": { + "type": ["null", "string"], + "format": "date-time" + }, + "method": { "type": ["null", "string"] }, + "method_detail": { "type": ["null", "string"] }, + "custom_method_detail": { "type": ["null", "string"] }, + "double_optin": { "type": ["null", "boolean"] }, + "suppressions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "reason": { "type": "string" }, + "timestamp": { + "type": "string", + "format": "date-time" + } + } + } + }, + "list_suppressions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "list_id": { "type": "string" }, + "reason": { "type": "string" }, + "timestamp": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, + "sms": { + "type": ["null", "object"], + "properties": { + "marketing": { + "type": ["null", "object"], + "properties": { + "can_receive_sms_marketing": { "type": "boolean" }, + "consent": { "type": "string" }, + "consent_timestamp": { + "type": "string", + "format": "date-time" + }, + "method": { "type": "string" }, + "method_detail": { "type": ["null", "string"] }, + "last_updated": { "type": "string", "format": "date-time" } + } + } + } + } + } + }, + "predictive_analytics": { + "type": ["null", "object"], + "properties": { + "historic_clv": { "type": ["null", "number"] }, + "predicted_clv": { "type": ["null", "number"] }, + "total_clv": { "type": ["null", "number"] }, + "historic_number_of_orders": { "type": ["null", "integer"] }, + "predicted_number_of_orders": { "type": ["null", "number"] }, + "average_days_between_orders": { "type": ["null", "number"] }, + "average_order_value": { "type": ["null", "number"] }, + "churn_probability": { "type": ["null", "number"] }, + "expected_date_of_next_order": { "type": ["null", "string"] } + } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] } + } + }, + "relationships": { + "type": ["null", "object"], + "properties": { + "lists": { + "type": ["null", "object"], + "properties": { + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] }, + "related": { "type": ["null", "string"] } + } + } + } + }, + "segments": { + "type": ["null", "object"], + "properties": { + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] }, + "related": { "type": ["null", "string"] } + } + } + } + } } }, - "links": { "type": ["null", "object"] }, - "relationships": { "type": ["null", "object"] }, "segments": { "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json index 6adea37deef3..1f863d5c4feb 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -10,16 +11,56 @@ "properties": { "name": { "type": "string" }, "created": { "type": ["null", "string"], "format": "date-time" }, - "updated": { "type": ["null", "string"], "format": "date-time" } + "updated": { "type": ["null", "string"], "format": "date-time" }, + "opt_in_process": { "type": ["null", "string"] } } }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { "type": "string" } + } }, "relationships": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "profiles": { + "type": ["null", "object"], + "properties": { + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": "string" }, + "related": { "type": "string" } + } + } + } + }, + "tags": { + "type": ["null", "object"], + "properties": { + "data": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": "string" }, + "related": { "type": "string" } + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json index 1d6984e551f8..0afd53a3d6a2 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -19,7 +20,10 @@ }, "links": { "type": ["null", "object"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "self": { "type": "string" } + } } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json index 21f80313fd0a..fcffdb97cbbe 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -11,22 +12,176 @@ "properties": { "email": { "type": ["null", "string"] }, "phone_number": { "type": ["null", "string"] }, + "anonymous_id": { "type": ["null", "string"] }, + "external_id": { "type": ["null", "string"] }, "first_name": { "type": ["null", "string"] }, "last_name": { "type": ["null", "string"] }, - "properties": { - "type": ["null", "object"], - "additionalProperties": true - }, - "subscriptions": { "type": ["null", "object"] }, "organization": { "type": ["null", "string"] }, "title": { "type": ["null", "string"] }, + "image": { "type": ["null", "string"] }, "created": { "type": ["null", "string"], "format": "date-time" }, "updated": { "type": ["null", "string"], "format": "date-time" }, - "last_event_date": { "type": ["null", "string"], "format": "date-time" } + "last_event_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "location": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "latitude": { + "oneOf": [ + { "type": "null" }, + { "type": "number" }, + { "type": "string" } + ] + }, + "longitude": { + "oneOf": [ + { "type": "null" }, + { "type": "number" }, + { "type": "string" } + ] + }, + "region": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "timezone": { "type": ["null", "string"] }, + "ip": { "type": ["null", "string"] } + } + }, + "properties": { + "type": ["null", "object"], + "additionalProperties": true + }, + "subscriptions": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "object"], + "properties": { + "marketing": { + "type": ["null", "object"], + "properties": { + "can_receive_email_marketing": { "type": "boolean" }, + "consent": { "type": "string" }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_updated": { + "type": ["null", "string"], + "format": "date-time" + }, + "method": { "type": ["null", "string"] }, + "method_detail": { "type": ["null", "string"] }, + "custom_method_detail": { "type": ["null", "string"] }, + "double_optin": { "type": ["null", "boolean"] }, + "suppressions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "reason": { "type": "string" }, + "timestamp": { + "type": "string", + "format": "date-time" + } + } + } + }, + "list_suppressions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "list_id": { "type": "string" }, + "reason": { "type": "string" }, + "timestamp": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, + "sms": { + "type": ["null", "object"], + "properties": { + "marketing": { + "type": ["null", "object"], + "properties": { + "can_receive_sms_marketing": { "type": "boolean" }, + "consent": { "type": "string" }, + "consent_timestamp": { + "type": "string", + "format": "date-time" + }, + "method": { "type": "string" }, + "method_detail": { "type": ["null", "string"] }, + "last_updated": { "type": "string", "format": "date-time" } + } + } + } + } + } + }, + "predictive_analytics": { + "type": ["null", "object"], + "properties": { + "historic_clv": { "type": ["null", "number"] }, + "predicted_clv": { "type": ["null", "number"] }, + "total_clv": { "type": ["null", "number"] }, + "historic_number_of_orders": { "type": ["null", "integer"] }, + "predicted_number_of_orders": { "type": ["null", "number"] }, + "average_days_between_orders": { "type": ["null", "number"] }, + "average_order_value": { "type": ["null", "number"] }, + "churn_probability": { "type": ["null", "number"] }, + "expected_date_of_next_order": { "type": ["null", "string"] } + } + } + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] } + } + }, + "relationships": { + "type": ["null", "object"], + "properties": { + "lists": { + "type": ["null", "object"], + "properties": { + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] }, + "related": { "type": ["null", "string"] } + } + } + } + }, + "segments": { + "type": ["null", "object"], + "properties": { + "links": { + "type": ["null", "object"], + "properties": { + "self": { "type": ["null", "string"] }, + "related": { "type": ["null", "string"] } + } + } + } + } } }, - "links": { "type": ["null", "object"] }, - "relationships": { "type": ["null", "object"] }, "segments": { "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 10009335765e..5b7eca53a67b 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from .availability_strategy import KlaviyoAvailabilityStrategy from .exceptions import KlaviyoBackoffError @@ -143,7 +144,10 @@ def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token latest_cursor = pendulum.parse(latest_cursor) if stream_state_cursor_value: latest_cursor = max(latest_cursor, pendulum.parse(stream_state_cursor_value)) - latest_cursor = min(latest_cursor, pendulum.now()) + # Klaviyo API will throw an error if the request filter is set too close to the current time. + # Setting a minimum value of at least 3 seconds from the current time ensures this will never happen, + # and allows our 'abnormal_state' acceptance test to pass. + latest_cursor = min(latest_cursor, pendulum.now().subtract(seconds=3)) params["filter"] = f"greater-than({self.cursor_field},{latest_cursor.isoformat()})" params["sort"] = self.cursor_field return params @@ -239,6 +243,8 @@ def read_records( class Profiles(IncrementalKlaviyoStream): """Docs: https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles""" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + cursor_field = "updated" api_revision = "2023-02-22" page_size = 100 diff --git a/airbyte-integrations/connectors/source-kustomer-singer/main.py b/airbyte-integrations/connectors/source-kustomer-singer/main.py index 77ee51ee831c..00920dc58a36 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/main.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/main.py @@ -2,11 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys - -from airbyte_cdk.entrypoint import launch -from source_kustomer_singer import SourceKustomerSinger +from source_kustomer_singer.run import run if __name__ == "__main__": - source = SourceKustomerSinger() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml index ede0b9624645..0aa460d17c75 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml @@ -8,11 +8,16 @@ data: icon: kustomer.svg license: MIT name: Kustomer - registries: + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-kustomer-singer + registries: # Removed from registries due to LEGACY STATE cloud: - enabled: true + enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/kustomer-singer tags: diff --git a/airbyte-integrations/connectors/source-kustomer-singer/setup.py b/airbyte-integrations/connectors/source-kustomer-singer/setup.py index 5e14ee6e2e1d..bc8c57120807 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/setup.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/setup.py @@ -53,6 +53,11 @@ def run(self): TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1"] setup( + entry_points={ + "console_scripts": [ + "source-kustomer-singer=source_kustomer_singer.run:run", + ], + }, name="source_kustomer_singer", description="Source implementation for Kustomer, built on the Singer tap implementation.", author="Airbyte", @@ -64,7 +69,19 @@ def run(self): "develop": CustomDevelopCommand, "egg_info": CustomEggInfoCommand, }, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py new file mode 100644 index 000000000000..fba603c749e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kustomer_singer import SourceKustomerSinger + + +def run(): + source = SourceKustomerSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kyriba/Dockerfile b/airbyte-integrations/connectors/source-kyriba/Dockerfile deleted file mode 100644 index cf89dabee191..000000000000 --- a/airbyte-integrations/connectors/source-kyriba/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.7.11-alpine3.14 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_kyriba ./source_kyriba - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-kyriba diff --git a/airbyte-integrations/connectors/source-kyriba/README.md b/airbyte-integrations/connectors/source-kyriba/README.md index 5fc9f63ccd96..ba9253537574 100644 --- a/airbyte-integrations/connectors/source-kyriba/README.md +++ b/airbyte-integrations/connectors/source-kyriba/README.md @@ -8,7 +8,7 @@ For information about how to use this connector within Airbyte, see [the documen ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.10.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: @@ -50,19 +50,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + ```bash -airbyte-ci connectors --name=source-kyriba build +airbyte-ci connectors --name source-kyriba build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-kyriba:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -An image will be built with the tag `airbyte/source-kyriba:dev`. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-kyriba:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code -**Via `docker build`:** +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: ```bash docker build -t airbyte/source-kyriba:dev . +# Running the spec command against your patched connector +docker run airbyte/source-kyriba:dev spec ``` - #### Run Then run any of the connector commands as follows: ``` @@ -97,4 +148,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/source-kyriba/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kyriba/acceptance-test-config.yml index 2c1a1e1d9718..fa8d8c88fc81 100644 --- a/airbyte-integrations/connectors/source-kyriba/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-kyriba/acceptance-test-config.yml @@ -1,30 +1,38 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-kyriba:dev -tests: +acceptance_tests: spec: - - spec_path: "source_kyriba/spec.json" + tests: + - spec_path: "source_kyriba/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + timeout_seconds: 1200 + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: true + incremental: + tests: + - config_path: "secrets/config.json" + timeout_seconds: 2400 + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + timeout_seconds: 2400 + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-kyriba/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-kyriba/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..404df43027ff --- /dev/null +++ b/airbyte-integrations/connectors/source-kyriba/integration_tests/expected_records.jsonl @@ -0,0 +1,30 @@ +{"stream": "accounts", "data": {"uuid": "aff2fe86-2c13-4362-bc7b-9d2f2cacfd2e", "code": "012CITIAUD", "description": "Company 012 Citi AUD a/c", "description2": null, "accountType": "BANK_ACCOUNT", "company": {"uuid": "b79ffed5-dd18-4242-985f-b9ca607d7ede", "code": "COMPANY012"}, "bank": {"uuid": "1f71d0be-f1a0-4eb5-8dae-3871a84bd5f8", "code": "CITI"}, "branch": {"uuid": "14d748f7-0054-443a-8b32-a7c452542d64", "code": "CITI_US"}, "branchDescription": null, "currency": {"uuid": "a8ab2150-a094-4b1a-b7a0-e21749e7b60c", "code": "AUD"}, "countryCode": "US", "ibanCode": null, "banCode": "00000000000000000", "statementIdentifier": null, "zbaIdentifier": null, "glAccount": {"uuid": null, "code": null}, "internalAccountCode": null, "calendar": {"uuid": "fc315653-ca1a-40b6-82e3-31e0ca73be91", "code": "US"}, "signatoryUsers": 0, "documents": false, "closedAccount": false, "creationDate": "2022-11-22", "updateDate": null, "closingDate": null, "status": "CREATION_TO_VALIDATE", "defaultGroup": {"uuid": null, "code": null}, "accountCategory1": {"uuid": null, "code": null}, "accountCategory2": {"uuid": null, "code": null}, "accountCategory3": {"uuid": null, "code": null}, "accountCategory4": {"uuid": null, "code": null}, "accountCategory5": {"uuid": null, "code": null}, "accountCategory6": {"uuid": null, "code": null}, "accountCategory7": {"uuid": null, "code": null}, "accountCategory8": {"uuid": null, "code": null}, "accountCategory9": {"uuid": null, "code": null}, "accountCategory10": {"uuid": null, "code": null}, "marker1": false, "marker2": false, "marker3": false, "attachments": false, "activeStatus": "OPENED", "accountAvailableForPayments": true}, "emitted_at": 1706211479986} +{"stream": "accounts", "data": {"uuid": "ec258294-9384-4aff-9201-e4c78dece318", "code": "012CITICHF", "description": "Company 012 Citi CHF a/c", "description2": null, "accountType": "BANK_ACCOUNT", "company": {"uuid": "b79ffed5-dd18-4242-985f-b9ca607d7ede", "code": "COMPANY012"}, "bank": {"uuid": "1f71d0be-f1a0-4eb5-8dae-3871a84bd5f8", "code": "CITI"}, "branch": {"uuid": "14d748f7-0054-443a-8b32-a7c452542d64", "code": "CITI_US"}, "branchDescription": null, "currency": {"uuid": "3def955d-cee5-441c-81d2-514c9acaec49", "code": "CHF"}, "countryCode": "US", "ibanCode": null, "banCode": "00000000000000000", "statementIdentifier": null, "zbaIdentifier": null, "glAccount": {"uuid": null, "code": null}, "internalAccountCode": null, "calendar": {"uuid": "fc315653-ca1a-40b6-82e3-31e0ca73be91", "code": "US"}, "signatoryUsers": 0, "documents": true, "closedAccount": false, "creationDate": "2022-11-22", "updateDate": null, "closingDate": null, "status": "CREATION_TO_VALIDATE", "defaultGroup": {"uuid": null, "code": null}, "accountCategory1": {"uuid": null, "code": null}, "accountCategory2": {"uuid": null, "code": null}, "accountCategory3": {"uuid": null, "code": null}, "accountCategory4": {"uuid": null, "code": null}, "accountCategory5": {"uuid": null, "code": null}, "accountCategory6": {"uuid": null, "code": null}, "accountCategory7": {"uuid": null, "code": null}, "accountCategory8": {"uuid": null, "code": null}, "accountCategory9": {"uuid": null, "code": null}, "accountCategory10": {"uuid": null, "code": null}, "marker1": false, "marker2": false, "marker3": false, "attachments": true, "activeStatus": "OPENED", "accountAvailableForPayments": true}, "emitted_at": 1706211479986} +{"stream": "accounts", "data": {"uuid": "29a77473-bc11-4a7a-b6f1-e11ed993cf67", "code": "012CITIDKK", "description": "Company 012 Citi DKK a/c", "description2": null, "accountType": "BANK_ACCOUNT", "company": {"uuid": "b79ffed5-dd18-4242-985f-b9ca607d7ede", "code": "COMPANY012"}, "bank": {"uuid": "1f71d0be-f1a0-4eb5-8dae-3871a84bd5f8", "code": "CITI"}, "branch": {"uuid": "14d748f7-0054-443a-8b32-a7c452542d64", "code": "CITI_US"}, "branchDescription": null, "currency": {"uuid": "ede48ce3-991f-41fc-86ff-23f859efde58", "code": "DKK"}, "countryCode": "US", "ibanCode": null, "banCode": "00000000000000000", "statementIdentifier": null, "zbaIdentifier": null, "glAccount": {"uuid": null, "code": null}, "internalAccountCode": null, "calendar": {"uuid": "fc315653-ca1a-40b6-82e3-31e0ca73be91", "code": "US"}, "signatoryUsers": 0, "documents": false, "closedAccount": false, "creationDate": "2022-11-22", "updateDate": null, "closingDate": null, "status": "CREATION_TO_VALIDATE", "defaultGroup": {"uuid": null, "code": null}, "accountCategory1": {"uuid": null, "code": null}, "accountCategory2": {"uuid": null, "code": null}, "accountCategory3": {"uuid": null, "code": null}, "accountCategory4": {"uuid": null, "code": null}, "accountCategory5": {"uuid": null, "code": null}, "accountCategory6": {"uuid": null, "code": null}, "accountCategory7": {"uuid": null, "code": null}, "accountCategory8": {"uuid": null, "code": null}, "accountCategory9": {"uuid": null, "code": null}, "accountCategory10": {"uuid": null, "code": null}, "marker1": false, "marker2": false, "marker3": false, "attachments": false, "activeStatus": "OPENED", "accountAvailableForPayments": true}, "emitted_at": 1706211479987} +{"stream": "accounts", "data": {"uuid": "9f1170cb-e4e4-48c1-b3e8-0abc5dfbd6ba", "code": "012CITIGBP", "description": "Company 012 Citi GBP a/c", "description2": null, "accountType": "BANK_ACCOUNT", "company": {"uuid": "b79ffed5-dd18-4242-985f-b9ca607d7ede", "code": "COMPANY012"}, "bank": {"uuid": "1f71d0be-f1a0-4eb5-8dae-3871a84bd5f8", "code": "CITI"}, "branch": {"uuid": "14d748f7-0054-443a-8b32-a7c452542d64", "code": "CITI_US"}, "branchDescription": null, "currency": {"uuid": "b790622e-5fc7-4a05-aa4b-879c5a9613a6", "code": "GBP"}, "countryCode": "US", "ibanCode": null, "banCode": "00000000000000000", "statementIdentifier": null, "zbaIdentifier": null, "glAccount": {"uuid": null, "code": null}, "internalAccountCode": null, "calendar": {"uuid": "fc315653-ca1a-40b6-82e3-31e0ca73be91", "code": "US"}, "signatoryUsers": 0, "documents": false, "closedAccount": false, "creationDate": "2022-11-22", "updateDate": null, "closingDate": null, "status": "CREATION_TO_VALIDATE", "defaultGroup": {"uuid": null, "code": null}, "accountCategory1": {"uuid": null, "code": null}, "accountCategory2": {"uuid": null, "code": null}, "accountCategory3": {"uuid": null, "code": null}, "accountCategory4": {"uuid": null, "code": null}, "accountCategory5": {"uuid": null, "code": null}, "accountCategory6": {"uuid": null, "code": null}, "accountCategory7": {"uuid": null, "code": null}, "accountCategory8": {"uuid": null, "code": null}, "accountCategory9": {"uuid": null, "code": null}, "accountCategory10": {"uuid": null, "code": null}, "marker1": false, "marker2": false, "marker3": false, "attachments": false, "activeStatus": "OPENED", "accountAvailableForPayments": true}, "emitted_at": 1706211479987} +{"stream": "accounts", "data": {"uuid": "42e125ab-b9be-4fb2-82a5-104141afedc9", "code": "012CITIHKD", "description": "Company 012 Citi HKD a/c", "description2": null, "accountType": "BANK_ACCOUNT", "company": {"uuid": "b79ffed5-dd18-4242-985f-b9ca607d7ede", "code": "COMPANY012"}, "bank": {"uuid": "1f71d0be-f1a0-4eb5-8dae-3871a84bd5f8", "code": "CITI"}, "branch": {"uuid": "14d748f7-0054-443a-8b32-a7c452542d64", "code": "CITI_US"}, "branchDescription": null, "currency": {"uuid": "4d0966be-c684-41b5-9782-a671391ef8b3", "code": "HKD"}, "countryCode": "US", "ibanCode": null, "banCode": "00000000000000000", "statementIdentifier": null, "zbaIdentifier": null, "glAccount": {"uuid": null, "code": null}, "internalAccountCode": null, "calendar": {"uuid": "fc315653-ca1a-40b6-82e3-31e0ca73be91", "code": "US"}, "signatoryUsers": 0, "documents": false, "closedAccount": false, "creationDate": "2022-11-22", "updateDate": null, "closingDate": null, "status": "CREATION_TO_VALIDATE", "defaultGroup": {"uuid": null, "code": null}, "accountCategory1": {"uuid": null, "code": null}, "accountCategory2": {"uuid": null, "code": null}, "accountCategory3": {"uuid": null, "code": null}, "accountCategory4": {"uuid": null, "code": null}, "accountCategory5": {"uuid": null, "code": null}, "accountCategory6": {"uuid": null, "code": null}, "accountCategory7": {"uuid": null, "code": null}, "accountCategory8": {"uuid": null, "code": null}, "accountCategory9": {"uuid": null, "code": null}, "accountCategory10": {"uuid": null, "code": null}, "marker1": false, "marker2": false, "marker3": false, "attachments": false, "activeStatus": "OPENED", "accountAvailableForPayments": true}, "emitted_at": 1706211479988} +{"stream": "cash_flows", "data": {"uuid": "03740dec-d26b-4860-9e94-c3ff812a01cd", "account": {"uuid": "88a86d41-76f2-4f89-a8b8-ba4b3abf9f49", "code": "C01-SCB-USD"}, "flowCode": {"uuid": "c217bfea-eb19-37bf-e053-f60311ac43de", "code": "+AR"}, "budgetCode": {"uuid": null, "code": null}, "flowAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "accountAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "description": null, "reference": "Kyriba Test invoice 146", "origin": "CA/COPY/24 01 010000057", "number": 1, "glStatus": "EXPORTED_TO_GL", "userZones": {"userZone1": "US-026", "userZone2": "USMF", "userZone3": "GNJL000949", "userZone4": "20210420 05 48 51 pm", "userZone5": "D365"}, "actualMode": "MANUAL", "status": "CONFIRMED", "transactionDate": "2024-01-01", "valueDate": "2021-04-19", "accountingDate": "2021-04-19", "updateDateTime": "2024-01-01T16:00:02Z"}, "emitted_at": 1706211495950} +{"stream": "cash_flows", "data": {"uuid": "f74f0cef-4555-47f7-9d30-324666600870", "account": {"uuid": "88a86d41-76f2-4f89-a8b8-ba4b3abf9f49", "code": "C01-SCB-USD"}, "flowCode": {"uuid": "c217bfea-eb19-37bf-e053-f60311ac43de", "code": "+AR"}, "budgetCode": {"uuid": null, "code": null}, "flowAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "accountAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "description": null, "reference": "Kyriba Test invoice 207", "origin": "CA/COPY/24 01 010000096", "number": 1, "glStatus": "EXPORTED_TO_GL", "userZones": {"userZone1": "US-026", "userZone2": "USMF", "userZone3": "GNJL001010", "userZone4": "20210420 05 48 55 pm", "userZone5": "D365"}, "actualMode": "MANUAL", "status": "CONFIRMED", "transactionDate": "2024-01-01", "valueDate": "2021-04-19", "accountingDate": "2021-04-19", "updateDateTime": "2024-01-01T16:00:02Z"}, "emitted_at": 1706211495951} +{"stream": "cash_flows", "data": {"uuid": "e82cd190-a922-4f79-8bf2-7818cc0e8d55", "account": {"uuid": "88a86d41-76f2-4f89-a8b8-ba4b3abf9f49", "code": "C01-SCB-USD"}, "flowCode": {"uuid": "c217bfea-eb19-37bf-e053-f60311ac43de", "code": "+AR"}, "budgetCode": {"uuid": null, "code": null}, "flowAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "accountAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "description": null, "reference": "Kyriba Test invoice 155", "origin": "CA/COPY/24 01 010000046", "number": 1, "glStatus": "EXPORTED_TO_GL", "userZones": {"userZone1": "US-026", "userZone2": "USMF", "userZone3": "GNJL000958", "userZone4": "20210420 05 48 51 pm", "userZone5": "D365"}, "actualMode": "MANUAL", "status": "CONFIRMED", "transactionDate": "2024-01-01", "valueDate": "2021-04-19", "accountingDate": "2021-04-19", "updateDateTime": "2024-01-01T16:00:02Z"}, "emitted_at": 1706211495951} +{"stream": "cash_flows", "data": {"uuid": "18bacf36-f8fd-459d-bd8b-f62277ec7ba1", "account": {"uuid": "88a86d41-76f2-4f89-a8b8-ba4b3abf9f49", "code": "C01-SCB-USD"}, "flowCode": {"uuid": "c217bfea-eb19-37bf-e053-f60311ac43de", "code": "+AR"}, "budgetCode": {"uuid": null, "code": null}, "flowAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "accountAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "description": null, "reference": "Kyriba Test invoice 145", "origin": "CA/COPY/24 01 010000079", "number": 1, "glStatus": "EXPORTED_TO_GL", "userZones": {"userZone1": "US-026", "userZone2": "USMF", "userZone3": "GNJL000948", "userZone4": "20210420 05 48 51 pm", "userZone5": "D365"}, "actualMode": "MANUAL", "status": "CONFIRMED", "transactionDate": "2024-01-01", "valueDate": "2021-04-19", "accountingDate": "2021-04-19", "updateDateTime": "2024-01-01T16:00:02Z"}, "emitted_at": 1706211495951} +{"stream": "cash_flows", "data": {"uuid": "e2f0a797-9233-4888-a65e-933bb7abb30b", "account": {"uuid": "88a86d41-76f2-4f89-a8b8-ba4b3abf9f49", "code": "C01-SCB-USD"}, "flowCode": {"uuid": "c217bfea-eb19-37bf-e053-f60311ac43de", "code": "+AR"}, "budgetCode": {"uuid": null, "code": null}, "flowAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "accountAmount": {"currency": {"uuid": "cb00aa87-fc0c-4712-a86b-062b1b77b3e6", "code": "USD"}, "amount": 1000.0, "signedAmount": 1000.0}, "description": null, "reference": "Kyriba Test invoice 243", "origin": "CA/COPY/24 01 010000137", "number": 1, "glStatus": "EXPORTED_TO_GL", "userZones": {"userZone1": "US-026", "userZone2": "USMF", "userZone3": "GNJL001046", "userZone4": "20210420 05 48 58 pm", "userZone5": "D365"}, "actualMode": "MANUAL", "status": "CONFIRMED", "transactionDate": "2024-01-01", "valueDate": "2021-04-19", "accountingDate": "2021-04-19", "updateDateTime": "2024-01-01T16:00:02Z"}, "emitted_at": 1706211495952} +{"stream": "bank_balances_eod", "data": {"account": {"uuid": "aff2fe86-2c13-4362-bc7b-9d2f2cacfd2e", "code": "012CITIAUD", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "AUD"}}, "emitted_at": 1706211527093} +{"stream": "bank_balances_eod", "data": {"account": {"uuid": "ec258294-9384-4aff-9201-e4c78dece318", "code": "012CITICHF", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "CHF"}}, "emitted_at": 1706211527234} +{"stream": "bank_balances_eod", "data": {"account": {"uuid": "29a77473-bc11-4a7a-b6f1-e11ed993cf67", "code": "012CITIDKK", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "DKK"}}, "emitted_at": 1706211527377} +{"stream": "bank_balances_eod", "data": {"account": {"uuid": "9f1170cb-e4e4-48c1-b3e8-0abc5dfbd6ba", "code": "012CITIGBP", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "GBP"}}, "emitted_at": 1706211527515} +{"stream": "bank_balances_eod", "data": {"account": {"uuid": "42e125ab-b9be-4fb2-82a5-104141afedc9", "code": "012CITIHKD", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "HKD"}}, "emitted_at": 1706211527741} +{"stream": "bank_balances_intraday", "data": {"account": {"uuid": "aff2fe86-2c13-4362-bc7b-9d2f2cacfd2e", "code": "012CITIAUD", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "AUD"}}, "emitted_at": 1706211626413} +{"stream": "bank_balances_intraday", "data": {"account": {"uuid": "ec258294-9384-4aff-9201-e4c78dece318", "code": "012CITICHF", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "CHF"}}, "emitted_at": 1706211626557} +{"stream": "bank_balances_intraday", "data": {"account": {"uuid": "29a77473-bc11-4a7a-b6f1-e11ed993cf67", "code": "012CITIDKK", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "DKK"}}, "emitted_at": 1706211626702} +{"stream": "bank_balances_intraday", "data": {"account": {"uuid": "9f1170cb-e4e4-48c1-b3e8-0abc5dfbd6ba", "code": "012CITIGBP", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "GBP"}}, "emitted_at": 1706211626891} +{"stream": "bank_balances_intraday", "data": {"account": {"uuid": "42e125ab-b9be-4fb2-82a5-104141afedc9", "code": "012CITIHKD", "statementIdentifier": null}, "bankBalance": {"balanceDate": "2024-01-01", "amount": 0, "lastKnownBalanceDate": null, "currencyCode": "HKD"}}, "emitted_at": 1706211627036} +{"stream": "cash_balances_eod", "data": {"account": {"uuid": "aff2fe86-2c13-4362-bc7b-9d2f2cacfd2e", "code": "012CITIAUD", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": false, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "AUD"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "AUD"}]}, "emitted_at": 1706211790985} +{"stream": "cash_balances_eod", "data": {"account": {"uuid": "ec258294-9384-4aff-9201-e4c78dece318", "code": "012CITICHF", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": false, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "CHF"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "CHF"}]}, "emitted_at": 1706211791131} +{"stream": "cash_balances_eod", "data": {"account": {"uuid": "29a77473-bc11-4a7a-b6f1-e11ed993cf67", "code": "012CITIDKK", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": false, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "DKK"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "DKK"}]}, "emitted_at": 1706211791278} +{"stream": "cash_balances_eod", "data": {"account": {"uuid": "9f1170cb-e4e4-48c1-b3e8-0abc5dfbd6ba", "code": "012CITIGBP", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": false, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "GBP"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "GBP"}]}, "emitted_at": 1706211791422} +{"stream": "cash_balances_eod", "data": {"account": {"uuid": "42e125ab-b9be-4fb2-82a5-104141afedc9", "code": "012CITIHKD", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": false, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "HKD"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "HKD"}]}, "emitted_at": 1706211791566} +{"stream": "cash_balances_intraday", "data": {"account": {"uuid": "aff2fe86-2c13-4362-bc7b-9d2f2cacfd2e", "code": "012CITIAUD", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": true, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "AUD"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "AUD"}]}, "emitted_at": 1706211865338} +{"stream": "cash_balances_intraday", "data": {"account": {"uuid": "ec258294-9384-4aff-9201-e4c78dece318", "code": "012CITICHF", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": true, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "CHF"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "CHF"}]}, "emitted_at": 1706211865475} +{"stream": "cash_balances_intraday", "data": {"account": {"uuid": "29a77473-bc11-4a7a-b6f1-e11ed993cf67", "code": "012CITIDKK", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": true, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "DKK"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "DKK"}]}, "emitted_at": 1706211865614} +{"stream": "cash_balances_intraday", "data": {"account": {"uuid": "9f1170cb-e4e4-48c1-b3e8-0abc5dfbd6ba", "code": "012CITIGBP", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": true, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "GBP"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "GBP"}]}, "emitted_at": 1706211865762} +{"stream": "cash_balances_intraday", "data": {"account": {"uuid": "42e125ab-b9be-4fb2-82a5-104141afedc9", "code": "012CITIHKD", "statementIdentifier": null}, "cashFlowStatus": {"estimatedForecasts": false, "confirmedForecasts": false, "intraday": true, "actual": true}, "cashBalance": [{"balanceDate": {"dateType": "VALUE", "date": "2024-01-01"}, "amount": 0.0, "currencyCode": "HKD"}, {"balanceDate": {"dateType": "VALUE", "date": "2024-01-02"}, "amount": 0.0, "currencyCode": "HKD"}]}, "emitted_at": 1706211865899} diff --git a/airbyte-integrations/connectors/source-kyriba/main.py b/airbyte-integrations/connectors/source-kyriba/main.py index 771b738d1b1b..cd0b8f1f2f3e 100644 --- a/airbyte-integrations/connectors/source-kyriba/main.py +++ b/airbyte-integrations/connectors/source-kyriba/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_kyriba import SourceKyriba +from source_kyriba.run import run if __name__ == "__main__": - source = SourceKyriba() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kyriba/metadata.yaml b/airbyte-integrations/connectors/source-kyriba/metadata.yaml index 347d7b22f2fa..49bfb45ac7e3 100644 --- a/airbyte-integrations/connectors/source-kyriba/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyriba/metadata.yaml @@ -1,24 +1,30 @@ data: + ab_internal: + ql: 100 + sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 547dc08e-ab51-421d-953b-8f3745201a8c - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-kyriba + documentationUrl: https://docs.airbyte.com/integrations/sources/kyriba githubIssueLabel: source-kyriba icon: kyriba.svg license: MIT name: Kyriba + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-kyriba registries: cloud: enabled: false oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/kyriba + supportLevel: community tags: - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-kyriba/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-kyriba/sample_files/configured_catalog.json deleted file mode 100644 index 4375ab4a8946..000000000000 --- a/airbyte-integrations/connectors/source-kyriba/sample_files/configured_catalog.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "bank_balances_intraday", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "cash_balances_intraday", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "accounts", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "source_defined_primary_key": [["uuid"], ["code"]], - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "cash_flows", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updateDateTime"] - }, - "sync_mode": "incremental", - "source_defined_primary_key": [["uuid"]], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "cash_balances_eod", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "bank_balances_eod", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updateDateTime"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "cash_balances_eod", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "bank_balances_eod", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-kyriba/setup.py b/airbyte-integrations/connectors/source-kyriba/setup.py index c81f4d4e1e0d..541552d2fbf1 100644 --- a/airbyte-integrations/connectors/source-kyriba/setup.py +++ b/airbyte-integrations/connectors/source-kyriba/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-kyriba=source_kyriba.run:run", + ], + }, name="source_kyriba", description="Source implementation for Kyriba.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py b/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py new file mode 100644 index 000000000000..e12facaaab47 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kyriba import SourceKyriba + + +def run(): + source = SourceKyriba() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/accounts.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/accounts.json index 9eaf67af886a..617951e2767d 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/accounts.json @@ -1,461 +1,270 @@ { - "type": "object", - "required": [ - "bank", - "branch", - "calendar", - "code", - "company", - "countryCode", - "currency", - "uuid" - ], + "type": ["object", "null"], + "title": "AccountSearchModel", + "description": "Account search", + "$schema": "http://json-schema.org/schema#", "properties": { "accountCategory1": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory2": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory3": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory4": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory5": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory6": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory7": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory8": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory9": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountCategory10": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "accountAvailableForPayments": { - "type": "boolean", - "example": true, - "description": "Flag to show if account is available for payments (true) or no (false)." + "type": ["boolean", "null"] }, "accountType": { - "type": "string", - "example": "BANK_ACCOUNT", - "description": "Account type", - "enum": [ - "BANK_ACCOUNT", - "INTERCOMPANY_ACCOUNT", - "OTHER_ACCOUNT", - "SHARED_ACCOUNT" - ] + "type": ["string", "null"] }, "activeStatus": { - "type": "string", - "example": "OPENED", - "description": "Account status.", - "enum": ["OPENED", "CLOSED"] + "type": ["string", "null"] }, "attachments": { - "type": "boolean", - "description": "Attachments" + "type": ["boolean", "null"] }, "banCode": { - "type": "string", - "description": "Account BAN code" + "type": ["string", "null"] }, "bank": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "CODE_ID", - "description": "Code identifier." + "type": ["string", "null"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["string", "null"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "branch": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "CODE_ID", - "description": "Code identifier." + "type": ["string", "null"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["string", "null"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "branchDescription": { - "type": ["null", "string"], - "description": "Account branch description" + "type": ["null", "string"] }, "calendar": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "CODE_ID", - "description": "Code identifier." + "type": ["string", "null"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["string", "null"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "closedAccount": { - "type": "boolean", - "example": false, - "description": "Is account closed. Default value false" + "type": ["boolean", "null"] }, "closingDate": { - "type": ["null", "string"], - "format": "date", - "example": "2020-06-23", - "description": "Closing date" + "type": ["null", "string"] }, "code": { - "type": "string", - "example": "COMPANYCODE", - "description": "Account code." + "type": ["string", "null"] }, "company": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "CODE_ID", - "description": "Code identifier." + "type": ["string", "null"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["string", "null"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "countryCode": { - "type": "string", - "description": "Account country code" + "type": ["string", "null"] }, "creationDate": { - "type": "string", - "format": "date", - "example": "2020-06-21", - "description": "Creation date" + "type": ["string", "null"] }, "currency": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "CODE_ID", - "description": "Code identifier." + "type": ["string", "null"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["string", "null"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "defaultGroup": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "description": { - "type": ["null", "string"], - "example": "Some description", - "description": "Account description." + "type": ["null", "string"] }, "description2": { - "type": ["null", "string"], - "example": "Some description2", - "description": "Account description2." + "type": ["null", "string"] }, "documents": { - "type": "boolean", - "description": "Documents." + "type": ["boolean", "null"] }, "glAccount": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": ["null", "string"], - "example": "CODE_ID", - "description": "Code identifier." + "type": ["null", "string"] }, "uuid": { - "type": ["null", "string"], - "format": "uuid", - "example": "123e4567-e89b-12d3-a456-426655440001", - "description": "UUID identifier. Has priority over the code" + "type": ["null", "string"] } - }, - "title": "ReferenceModel", - "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." + } }, "ibanCode": { - "type": ["null", "string"], - "description": "Account IBAN code" + "type": ["null", "string"] }, "internalAccountCode": { - "type": ["null", "string"], - "description": "Internal account code" + "type": ["null", "string"] }, "marker1": { - "type": "boolean", - "example": false, - "description": "Marker1." + "type": ["boolean", "null"] }, "marker2": { - "type": "boolean", - "example": false, - "description": "Marker2." + "type": ["boolean", "null"] }, "marker3": { - "type": "boolean", - "example": false, - "description": "Marker3." + "type": ["boolean", "null"] }, "signatoryUsers": { - "type": "integer", - "format": "int64", - "description": "Signatory users." + "type": ["integer", "null"] }, "statementIdentifier": { - "type": ["null", "string"], - "description": "Account statement identifier" + "type": ["null", "string"] }, "status": { - "type": "string", - "example": "CREATION_TO_VALIDATE", - "description": "Account status", - "enum": [ - "STANDARD", - "CREATION_TO_VALIDATE", - "CREATION_VALIDATED", - "AVAILABLE_FOR_BANK_INTEGRATION", - "CLOSURE_REQUESTED", - "CLOSURE_VALIDATE", - "BANK_CONTRACT_TO_INACTIVATE", - "CLOSED" - ] + "type": ["string", "null"] }, "updateDate": { - "type": ["null", "string"], - "format": "date", - "example": "2020-06-22", - "description": "Update date" + "type": ["null", "string"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "UUID of the account." + "type": ["string", "null"] }, "zbaIdentifier": { - "type": ["null", "string"], - "description": "Account ZBA identifier" + "type": ["null", "string"] } - }, - "title": "AccountSearchModel", - "description": "Account search", - "$schema": "http://json-schema.org/schema#" + } } diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_eod.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_eod.json index 06dbe17767f1..f17fe8e24f47 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_eod.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_eod.json @@ -1,63 +1,36 @@ { "type": "object", - "required": ["account", "bankBalance"], "properties": { "account": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "ACCOUNT001", - "description": "Code that represents the account.", - "minLength": 1, - "maxLength": 12 + "type": ["string", "null"] }, "statementIdentifier": { - "type": ["null", "string"], - "example": "Statement identifier", - "description": "Account Statement Identifier of the account.", - "minLength": 0, - "maxLength": 50 + "type": ["null", "string"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "Unique internal identifier that represents the account." + "type": ["string", "null"] } - }, - "title": "BK-BankAccountModel" + } }, "bankBalance": { - "type": "object", + "type": ["object", "null"], "properties": { "amount": { - "type": "number", - "format": "double", - "example": 1.123, - "description": "End of day balance value." + "type": ["number", "null"] }, "balanceDate": { - "type": "string", - "format": "date", - "example": "2019-11-11", - "description": "Date of the end of day balance." + "type": ["string", "null"] }, "currencyCode": { - "type": "string", - "example": "EUR", - "description": "Account currency code.", - "minLength": 1, - "maxLength": 3 + "type": ["string", "null"] }, "lastKnownBalanceDate": { - "type": ["null", "string"], - "format": "date", - "example": "2019-11-11", - "description": "Date of last known balance within the last 365 days." + "type": ["null", "string"] } - }, - "title": "BK-BalanceModel" + } } }, "title": "BK-BankStatementBalanceModel", diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_intraday.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_intraday.json index 446b0322f8e8..fe05c9ecf299 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_intraday.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/bank_balances_intraday.json @@ -1,54 +1,34 @@ { "type": "object", - "required": ["account", "bankBalance"], "properties": { "account": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "ACCOUNT001", - "description": "Code that represents the account.", - "minLength": 1, - "maxLength": 12 + "type": ["string", "null"] }, "statementIdentifier": { - "type": ["null", "string"], - "example": "Statement identifier", - "description": "Account Statement Identifier of the account.", - "minLength": 0, - "maxLength": 50 + "type": ["null", "string"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "Unique internal identifier that represents the account." + "type": ["string", "null"] } - }, - "title": "BK-BankAccountModel" + } }, "bankBalance": { - "type": "object", + "type": ["object", "null"], "properties": { "amount": { - "type": "number", - "format": "double", - "example": 1.123, - "description": "End of day balance value." + "type": ["number", "null"] }, "balanceDate": { - "type": "string", - "format": "date", - "example": "2019-11-11", - "description": "Date of the end of day balance." + "type": ["string", "null"] }, "currencyCode": { - "type": "string", - "example": "EUR", - "description": "Account currency code.", - "minLength": 1, - "maxLength": 3 + "type": ["string", "null"] + }, + "lastKnownBalanceDate": { + "type": ["string", "null"] } }, "title": "BK-BalanceModel" diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_eod.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_eod.json index b311f38ce9df..809cc9429d0e 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_eod.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_eod.json @@ -2,98 +2,60 @@ "type": "object", "properties": { "account": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "ACCOUNT001", - "description": "Code that represents the account.", - "minLength": 1, - "maxLength": 12 + "type": ["string", "null"] }, "statementIdentifier": { - "type": ["null", "string"], - "example": "Statement identifier", - "description": "Account Statement Identifier of the account.", - "minLength": 0, - "maxLength": 50 + "type": ["null", "string"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "Unique internal identifier that represents the account." + "type": ["string", "null"] } - }, - "title": "BK-BankAccountModel" + } }, "cashBalance": { - "type": "array", - "description": "Cash balance.", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "amount": { - "type": "number", - "format": "double", - "example": -50000.0, - "description": "Cash balance amount of the selected date, Amount can be positive and negative." + "type": ["number", "null"] }, "balanceDate": { - "type": "object", + "type": ["object", "null"], "properties": { "date": { - "type": "string", - "format": "date", - "example": "2020-05-25", - "description": "The date based on which the cash balance is calculated." + "type": ["string", "null"] }, "dateType": { - "type": "string", - "example": "TRANSACTION", - "description": "The date type based on which the cash balance is calculated.", - "enum": ["TRANSACTION", "VALUE"] + "type": ["string", "null"] } - }, - "title": "BK-BalanceDateModel" + } }, "currencyCode": { - "type": "string", - "example": "EUR", - "description": "Currency code.", - "minLength": 1, - "maxLength": 3 + "type": ["string", "null"] } - }, - "title": "BK-CashBalanceModel" + } } }, "cashFlowStatus": { - "type": "object", + "type": ["object", "null"], "properties": { "actual": { - "type": "boolean", - "example": false, - "description": "Select the status Actual to calculate the cash balance." + "type": ["boolean", "null"] }, "confirmedForecasts": { - "type": "boolean", - "example": false, - "description": "Select the status Confirmed forecast to calculate the cash balance." + "type": ["boolean", "null"] }, "estimatedForecasts": { - "type": "boolean", - "example": false, - "description": "Select the status Estimated forecast to calculate the cash balance." + "type": ["boolean", "null"] }, "intraday": { - "type": "boolean", - "example": false, - "description": "Select the status Intraday to calculate the cash balance." + "type": ["boolean", "null"] } - }, - "title": "BK-CashFlowStatusModel", - "description": "Cash flow status." + } } }, "title": "BK-AccountCashBalanceModel", diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_intraday.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_intraday.json index b311f38ce9df..4ffbdf43f695 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_intraday.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_balances_intraday.json @@ -2,98 +2,61 @@ "type": "object", "properties": { "account": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", - "example": "ACCOUNT001", - "description": "Code that represents the account.", - "minLength": 1, - "maxLength": 12 + "type": ["string", "null"] }, "statementIdentifier": { - "type": ["null", "string"], - "example": "Statement identifier", - "description": "Account Statement Identifier of the account.", - "minLength": 0, - "maxLength": 50 + "type": ["null", "string"] }, "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "Unique internal identifier that represents the account." + "type": ["string", "null"] } - }, - "title": "BK-BankAccountModel" + } }, "cashBalance": { - "type": "array", + "type": ["array", "null"], "description": "Cash balance.", "items": { - "type": "object", + "type": ["object", "null"], "properties": { "amount": { - "type": "number", - "format": "double", - "example": -50000.0, - "description": "Cash balance amount of the selected date, Amount can be positive and negative." + "type": ["number", "null"] }, "balanceDate": { - "type": "object", + "type": ["object", "null"], "properties": { "date": { - "type": "string", - "format": "date", - "example": "2020-05-25", - "description": "The date based on which the cash balance is calculated." + "type": ["string", "null"] }, "dateType": { - "type": "string", - "example": "TRANSACTION", - "description": "The date type based on which the cash balance is calculated.", - "enum": ["TRANSACTION", "VALUE"] + "type": ["string", "null"] } - }, - "title": "BK-BalanceDateModel" + } }, "currencyCode": { - "type": "string", - "example": "EUR", - "description": "Currency code.", - "minLength": 1, - "maxLength": 3 + "type": ["string", "null"] } - }, - "title": "BK-CashBalanceModel" + } } }, "cashFlowStatus": { - "type": "object", + "type": ["object", "null"], "properties": { "actual": { - "type": "boolean", - "example": false, - "description": "Select the status Actual to calculate the cash balance." + "type": ["boolean", "null"] }, "confirmedForecasts": { - "type": "boolean", - "example": false, - "description": "Select the status Confirmed forecast to calculate the cash balance." + "type": ["boolean", "null"] }, "estimatedForecasts": { - "type": "boolean", - "example": false, - "description": "Select the status Estimated forecast to calculate the cash balance." + "type": ["boolean", "null"] }, "intraday": { - "type": "boolean", - "example": false, - "description": "Select the status Intraday to calculate the cash balance." + "type": ["boolean", "null"] } - }, - "title": "BK-CashFlowStatusModel", - "description": "Cash flow status." + } } }, "title": "BK-AccountCashBalanceModel", diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_flows.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_flows.json index a16b1e884e9e..68194b1ce435 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_flows.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/cash_flows.json @@ -3,37 +3,21 @@ "required": ["uuid", "account", "flowAmount", "flowCode", "status"], "properties": { "uuid": { - "type": "string", - "format": "uuid", - "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "description": "UUID of the cash flow." + "type": ["string", "null"] }, "transactionDate": { - "type": "string", - "format": "date", - "example": "2021-04-01", - "description": "Transaction date" + "type": ["string", "null"] }, "valueDate": { - "type": "string", - "format": "date", - "example": "2021-04-02", - "description": "Value date" + "type": ["string", "null"] }, "accountingDate": { - "type": "string", - "format": "date", - "example": "2021-04-03", - "description": "Accounting date" + "type": ["string", "null"] }, "updateDateTime": { - "type": "string", - "format": "date-time", - "example": "2020-01-02T13:04:35Z", - "description": "Update date time" + "type": ["string", "null"] }, "account": { - "description": "Bank account", "$ref": "_definitions.json#/definitions/ReferenceModel" }, "flowCode": { @@ -45,9 +29,7 @@ "$ref": "_definitions.json#/definitions/ReferenceModel" }, "status": { - "type": "string", - "example": "CONFIRMED", - "description": "Cash flow status" + "type": ["string", "null"] }, "flowAmount": { "description": "Flow amount", @@ -58,43 +40,26 @@ "$ref": "_definitions.json#/definitions/AmountModel" }, "description": { - "type": ["null", "string"], - "description": "Cash flow's description", - "minLength": 0, - "maxLength": 2500 + "type": ["null", "string"] }, "reference": { - "type": ["null", "string"], - "description": "Cash flow's reference", - "minLength": 0, - "maxLength": 250 + "type": ["null", "string"] }, "origin": { - "type": "string", - "description": "Cash flow's origin", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"] }, "number": { - "type": "integer", - "format": "int32", - "description": "Cash flow's number", - "minimum": 0, - "maximum": 250 + "type": ["integer", "null"] }, "glStatus": { - "type": "string", - "example": "BALANCED", - "description": "Cash flow GL status" + "type": ["string", "null"] }, "userZones": { "description": "User zones of the cash flow", "$ref": "_definitions.json#/definitions/UserZonesModel" }, "actualMode": { - "type": "string", - "example": "Manual", - "description": "Cash flow actual mode" + "type": ["string", "null"] } }, "title": "CashFlowSearchModel", diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/shared/_definitions.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/shared/_definitions.json index f2c720540145..a7486dbe06e1 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/shared/_definitions.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/schemas/shared/_definitions.json @@ -1,15 +1,14 @@ { "definitions": { "AggregatedGroupDto": { - "type": "object", - "required": ["aggregation"], + "type": ["object", "null"], "properties": { "entity": { "description": "Entity", "$ref": "#/definitions/ReferenceModel" }, "date": { - "type": "string", + "type": ["string", "null"], "format": "date", "description": "Date" }, @@ -25,16 +24,15 @@ "title": "AggregatedGroupDto" }, "AggregationDto": { - "type": "object", - "required": ["amount"], + "type": ["object", "null"], "properties": { "amount": { - "type": "number", + "type": ["number", "null"], "example": 10580.123, "description": "Calculated aggregation amount of the cash flows expressed in conversion currency" }, "count": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", "description": "Count of the aggregated cash flows." } @@ -42,39 +40,21 @@ "title": "AggregationDto" }, "AggregationLevelDto": { - "type": "object", - "required": ["groupingCriterion", "groups", "level"], + "type": ["object", "null"], "properties": { "level": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", "example": 1, - "description": "The level of the aggregation", - "enum": [1, 2, 3] + "description": "The level of the aggregation" }, "groupingCriterion": { - "type": "string", + "type": ["string", "null"], "example": "BANK", - "description": "The grouping criterion for this level", - "enum": [ - "ACCOUNT", - "ACCOUNT_GROUP", - "BANK", - "BANK_GROUP", - "COMPANY", - "COMPANY_GROUP", - "CURRENCY", - "COUNTRY", - "COUNTRY_GROUP", - "FLOW_CODE", - "BUDGET_CODE", - "TRANSACTION_DATE", - "VALUE_DATE", - "ACCOUNTING_DATE" - ] + "description": "The grouping criterion for this level" }, "groups": { - "type": "array", + "type": ["array", "null"], "description": "List of aggregated groups", "items": { "$ref": "#/definitions/AggregatedGroupDto" @@ -84,7 +64,7 @@ "title": "AggregationLevelDto" }, "AmountModel": { - "type": "object", + "type": ["object", "null"], "properties": { "currency": { "example": "USD", @@ -92,19 +72,22 @@ "$ref": "#/definitions/ReferenceModel" }, "amount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Amount value" + }, + "signedAmount": { + "type": ["number", "null"] } }, "title": "AmountModel" }, "AmountResponseDTO": { - "type": "object", + "type": ["object", "null"], "properties": { "amount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Amount" @@ -118,28 +101,28 @@ "title": "AmountResponseDTO" }, "CashAmountModel": { - "type": "object", + "type": ["object", "null"], "properties": { "accountAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Account amount" }, "feeAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Fee amount" }, "interestAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Interest amount" }, "commissionAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Commission amount" @@ -148,16 +131,7 @@ "title": "CashAmountModel" }, "CashAmountResponseDTO": { - "type": "object", - "required": [ - "account", - "accountAmount", - "date", - "flowAmount", - "flowCode", - "number", - "status" - ], + "type": ["object", "null"], "properties": { "account": { "description": "Bank account", @@ -176,10 +150,8 @@ "$ref": "#/definitions/CashFlowDateModel" }, "description": { - "type": "string", - "description": "Cash flow's description", - "minLength": 0, - "maxLength": 2500 + "type": ["string", "null"], + "description": "Cash flow's description" }, "flowAmount": { "description": "Flow amount", @@ -190,41 +162,33 @@ "$ref": "#/definitions/AmountResponseDTO" }, "status": { - "type": "string", + "type": ["string", "null"], "example": "CONFIRMED", - "description": "Cash flow status", - "minLength": 0, - "maxLength": 2500 + "description": "Cash flow status" }, "reference": { - "type": "string", - "description": "Cash flow's reference", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's reference" }, "origin": { - "type": "string", - "description": "Cash flow's origin", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's origin" }, "number": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", "description": "Cash flow's number" }, "glStatus": { - "type": "string", - "description": "Cash flow's GL status", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's GL status" }, "userZones": { "description": "User zones of the cash flow", "$ref": "#/definitions/UserZonesModel" }, "actualMode": { - "type": "string", + "type": ["string", "null"], "example": "Manual", "description": "Cash flow actual mode" } @@ -232,20 +196,18 @@ "title": "CashAmountResponseDTO" }, "CashFlowAggregationFilteringDto": { - "type": "object", - "required": ["entityFilters"], + "type": ["object", "null"], "properties": { "entityFilters": { - "type": "array", + "type": ["array", "null"], "description": "The list of the filters by entities", "items": { "$ref": "#/definitions/EntityFilterDto" } }, "accountType": { - "type": "string", - "description": "The account type selected to filter cash flows", - "enum": ["ALL", "BANK", "INTERCO"] + "type": ["string", "null"], + "description": "The account type selected to filter cash flows" }, "period": { "description": "The period used to filter cash flows", @@ -260,132 +222,53 @@ "$ref": "#/definitions/ReferenceModel" }, "origin": { - "type": "string", - "description": "Origin of a cash flow", - "enum": [ - "BANK", - "BANK_FLOAT", - "BANK_INTRADAY", - "BANK_SHARE", - "CASH_FLOW", - "CASH_INTEGRATION", - "CASH_LAW", - "CASH_COPY", - "CASH_FEE", - "CASH_INTEREST", - "CASH_BALANCE", - "CASH_SCENARIO", - "CASH_SERIES", - "BALANCING", - "BORROWING", - "FX", - "PAYMENT", - "PAYMENT_CAMT054", - "PAYMENT_ERP", - "PAYABLE_DRAFTS", - "RECEIVABLE_DRAFTS" - ] + "type": ["string", "null"], + "description": "Origin of a cash flow" }, "description": { - "type": "string", - "description": "Description of the cash flow", - "minLength": 0, - "maxLength": 100 + "type": ["string", "null"], + "description": "Description of the cash flow" }, "reference": { - "type": "string", + "type": ["string", "null"], "example": "123-ABC", - "description": "Reference of the cash flow", - "minLength": 0, - "maxLength": 50 + "description": "Reference of the cash flow" }, "status": { - "type": "string", - "description": "Status of a cash flow", - "enum": ["ESTIMATED", "CONFIRMED", "ACTUAL", "INTRADAY"] + "type": ["string", "null"], + "description": "Status of a cash flow" }, "actualMode": { - "type": "string", - "description": "Select the actual mode to filter the cash flows", - "enum": ["ALL", "CASHREC", "MANUAL"] + "type": ["string", "null"], + "description": "Select the actual mode to filter the cash flows" }, "glStatus": { - "type": "string", - "description": "GL status", - "enum": ["EMPTY", "NOT_BALANCED", "NOT_SENT_TO_GL", "SENT_TO_GL"] + "type": ["string", "null"], + "description": "GL status" } }, "title": "CashFlowAggregationFilteringDto" }, "CashFlowAggregationLevelsDto": { - "type": "object", + "type": ["object", "null"], "properties": { "level1": { - "type": "string", - "description": "The first level of aggregation by the grouping criterion", - "enum": [ - "ACCOUNT", - "ACCOUNT_GROUP", - "BANK", - "BANK_GROUP", - "COMPANY", - "COMPANY_GROUP", - "CURRENCY", - "COUNTRY", - "COUNTRY_GROUP", - "FLOW_CODE", - "BUDGET_CODE", - "TRANSACTION_DATE", - "VALUE_DATE", - "ACCOUNTING_DATE" - ] + "type": ["string", "null"], + "description": "The first level of aggregation by the grouping criterion" }, "level2": { - "type": "string", - "description": "The second level of aggregation by the grouping criterion", - "enum": [ - "ACCOUNT", - "ACCOUNT_GROUP", - "BANK", - "BANK_GROUP", - "COMPANY", - "COMPANY_GROUP", - "CURRENCY", - "COUNTRY", - "COUNTRY_GROUP", - "FLOW_CODE", - "BUDGET_CODE", - "TRANSACTION_DATE", - "VALUE_DATE", - "ACCOUNTING_DATE" - ] + "type": ["string", "null"], + "description": "The second level of aggregation by the grouping criterion" }, "level3": { - "type": "string", - "description": "The third level of aggregation by the grouping criterion", - "enum": [ - "ACCOUNT", - "ACCOUNT_GROUP", - "BANK", - "BANK_GROUP", - "COMPANY", - "COMPANY_GROUP", - "CURRENCY", - "COUNTRY", - "COUNTRY_GROUP", - "FLOW_CODE", - "BUDGET_CODE", - "TRANSACTION_DATE", - "VALUE_DATE", - "ACCOUNTING_DATE" - ] + "type": ["string", "null"], + "description": "The third level of aggregation by the grouping criterion" } }, "title": "CashFlowAggregationLevelsDto" }, "CashFlowAggregationRequestDto": { - "type": "object", - "required": ["currencyConversion", "filtering"], + "type": ["object", "null"], "properties": { "filtering": { "description": "Filtering", @@ -403,8 +286,7 @@ "title": "CashFlowAggregationRequestDto" }, "CashFlowAggregationResponseDto": { - "type": "object", - "required": ["accountAmountAggregationTotal", "currency"], + "type": ["object", "null"], "properties": { "accountAmountAggregationTotal": { "description": "Calculated aggregation total of the cash flows", @@ -422,28 +304,28 @@ "title": "CashFlowAggregationResponseDto" }, "CashFlowDateModel": { - "type": "object", + "type": ["object", "null"], "properties": { "transactionDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2021-04-01", "description": "Transaction date" }, "valueDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2021-04-02", "description": "Value date" }, "accountingDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2021-04-03", "description": "Accounting date" }, "updateDateTime": { - "type": "string", + "type": ["string", "null"], "format": "date-time", "example": "2020-01-02T13:04:35Z", "description": "Update date time" @@ -452,8 +334,7 @@ "title": "CashFlowDateModel" }, "CashFlowModel": { - "type": "object", - "required": ["account", "date", "flowAmount", "flowCode", "status"], + "type": ["object", "null"], "properties": { "account": { "description": "Bank account", @@ -468,7 +349,7 @@ "$ref": "#/definitions/ReferenceModel" }, "status": { - "type": "string", + "type": ["string", "null"], "example": "CONFIRMED", "description": "Cash flow status" }, @@ -485,16 +366,12 @@ "$ref": "#/definitions/CashAmountModel" }, "description": { - "type": "string", - "description": "Cash flow's description", - "minLength": 0, - "maxLength": 2500 + "type": ["string", "null"], + "description": "Cash flow's description" }, "reference": { - "type": "string", - "description": "Cash flow's reference", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's reference" }, "userZones": { "description": "User zones of the cash flow", @@ -504,17 +381,7 @@ "title": "CashFlowModel" }, "CashFlowResponseDetailed": { - "type": "object", - "required": [ - "account", - "bank", - "branch", - "company", - "companyGlAmount", - "counterpartyFlow", - "flowCode", - "forecastGlCountervalue" - ], + "type": ["object", "null"], "properties": { "currency": { "example": "USD", @@ -522,40 +389,36 @@ "$ref": "#/definitions/ReferenceModel" }, "feeAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Fee amount" }, "interestAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Interest amount" }, "commissionAmount": { - "type": "number", + "type": ["number", "null"], "format": "double", "example": 10580.123, "description": "Commission amount" }, "updateDateTime": { - "type": "string", + "type": ["string", "null"], "format": "date-time", "example": "2021-04-02T12:12:12Z", "description": "Last update date of the cash flow." }, "flowID": { - "type": "string", - "description": "Flow ID", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Flow ID" }, "companyConsolidationCode": { - "type": "string", - "description": "Company consolidation code", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Company consolidation code" }, "company": { "description": "Company", @@ -578,10 +441,8 @@ "$ref": "#/definitions/AmountResponseDTO" }, "debitCreditType": { - "type": "string", - "description": "Type of GL", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Type of GL" }, "counterpartyFlow": { "description": "Counterparty flow", @@ -599,8 +460,7 @@ "title": "CashFlowResponseDetailed" }, "CashFlowSearchModel": { - "type": "object", - "required": ["account", "date", "flowAmount", "flowCode", "status"], + "type": ["object", "null"], "properties": { "account": { "description": "Bank account", @@ -619,7 +479,7 @@ "$ref": "#/definitions/CashFlowDateModel" }, "status": { - "type": "string", + "type": ["string", "null"], "example": "CONFIRMED", "description": "Cash flow status" }, @@ -632,32 +492,24 @@ "$ref": "#/definitions/AmountModel" }, "description": { - "type": "string", - "description": "Cash flow's description", - "minLength": 0, - "maxLength": 2500 + "type": ["string", "null"], + "description": "Cash flow's description" }, "reference": { - "type": "string", - "description": "Cash flow's reference", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's reference" }, "origin": { - "type": "string", - "description": "Cash flow's origin", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's origin" }, "number": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", - "description": "Cash flow's number", - "minimum": 0, - "maximum": 250 + "description": "Cash flow's number" }, "glStatus": { - "type": "string", + "type": ["string", "null"], "example": "BALANCED", "description": "Cash flow GL status" }, @@ -666,7 +518,7 @@ "$ref": "#/definitions/UserZonesModel" }, "actualMode": { - "type": "string", + "type": ["string", "null"], "example": "Manual", "description": "Cash flow actual mode" } @@ -674,8 +526,7 @@ "title": "CashFlowSearchModel" }, "CashFlowUpdateModel": { - "type": "object", - "required": ["account", "date", "flowAmount", "flowCode", "status"], + "type": ["object", "null"], "properties": { "account": { "description": "Bank account", @@ -690,7 +541,7 @@ "$ref": "#/definitions/ReferenceModel" }, "status": { - "type": "string", + "type": ["string", "null"], "example": "CONFIRMED", "description": "Cash flow status" }, @@ -707,16 +558,12 @@ "$ref": "#/definitions/CashAmountModel" }, "description": { - "type": "string", - "description": "Cash flow's description", - "minLength": 0, - "maxLength": 2500 + "type": ["string", "null"], + "description": "Cash flow's description" }, "reference": { - "type": "string", - "description": "Cash flow's reference", - "minLength": 0, - "maxLength": 250 + "type": ["string", "null"], + "description": "Cash flow's reference" }, "userZones": { "description": "User zones of the cash flow", @@ -726,23 +573,21 @@ "title": "CashFlowUpdateModel" }, "CounterpartyFlowModel": { - "type": "object", - "required": ["company", "consolidationCode"], + "type": ["object", "null"], "properties": { "company": { "description": "Company", "$ref": "#/definitions/ReferenceModel" }, "consolidationCode": { - "type": "string", + "type": ["string", "null"], "description": "Consolidation code" } }, "title": "CounterpartyFlowModel" }, "CurrencyConversionDto": { - "type": "object", - "required": ["currency", "currencyRate"], + "type": ["object", "null"], "properties": { "currency": { "description": "Currency used for conversion", @@ -756,24 +601,21 @@ "title": "CurrencyConversionDto" }, "CurrencyConversionRateDto": { - "type": "object", - "required": ["mode"], + "type": ["object", "null"], "properties": { "mode": { - "type": "string", - "description": "Currency rate mode to use for currency conversion", - "enum": ["FIXING", "HISTORICAL"] + "type": ["string", "null"], + "description": "Currency rate mode to use for currency conversion" }, "fixingDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2020-02-01", "description": "The date of the currency rate to be used if selected mode is FIXING." }, "historicalDateType": { - "type": "string", - "description": "The dates of that type of the cash flows will be used as the currency rate datesif selected mode is HISTORICAL.", - "enum": ["TRANSACTION", "VALUE", "ACCOUNTING"] + "type": ["string", "null"], + "description": "The dates of that type of the cash flows will be used as the currency rate datesif selected mode is HISTORICAL." }, "type": { "description": "Currency rate type", @@ -783,26 +625,14 @@ "title": "CurrencyConversionRateDto" }, "EntityFilterDto": { - "type": "object", - "required": ["entities", "entityType"], + "type": ["object", "null"], "properties": { "entityType": { - "type": "string", - "description": "The entity type to filter cash flows", - "enum": [ - "ACCOUNT", - "ACCOUNT_GROUP", - "BANK", - "BANK_GROUP", - "COMPANY", - "COMPANY_GROUP", - "CURRENCY", - "COUNTRY", - "COUNTRY_GROUP" - ] + "type": ["string", "null"], + "description": "The entity type to filter cash flows" }, "entities": { - "type": "array", + "type": ["array", "null"], "description": "The entities to filter cash flows of the specified type", "items": { "$ref": "#/definitions/ReferenceModel" @@ -812,22 +642,21 @@ "title": "EntityFilterDto" }, "FilteringPeriodDto": { - "type": "object", + "type": ["object", "null"], "properties": { "dateType": { - "type": "string", + "type": ["string", "null"], "example": "TRANSACTION", - "description": "The date type based on which the period is defined", - "enum": ["TRANSACTION", "VALUE", "ACCOUNTING", "UPDATE", "CREATION"] + "description": "The date type based on which the period is defined" }, "startDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2021-04-02", "description": "The start date of the filtering period" }, "endDate": { - "type": "string", + "type": ["string", "null"], "format": "date", "example": "2021-04-03", "description": "The end date of the filtering period" @@ -836,10 +665,10 @@ "title": "FilteringPeriodDto" }, "InnerError": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { - "type": "string", + "type": ["string", "null"], "example": "Invalid", "description": "One of a server-defined set of error codes." }, @@ -852,7 +681,7 @@ "description": "An object containing more specific information than the current object about the error." }, "ReferenceModel": { - "type": "object", + "type": ["object", "null"], "properties": { "code": { "type": ["null", "string"], @@ -870,11 +699,10 @@ "description": "Represents possible identifiers for resource. Should be provided at least one identifier (code or uuid). In the case of providing uuid and code, uuid will be used for resolving the reference." }, "ResponseUUIDModel": { - "type": "object", - "required": ["uuid"], + "type": ["object", "null"], "properties": { "uuid": { - "type": "string", + "type": ["string", "null"], "format": "uuid", "example": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "description": "UUID of the created resource." @@ -883,16 +711,15 @@ "title": "ResponseUUIDModel" }, "RestApiError": { - "type": "object", - "required": ["code", "message"], + "type": ["object", "null"], "properties": { "code": { - "type": "string", + "type": ["string", "null"], "example": "Invalid", "description": "One of a server-defined set of error codes." }, "details": { - "type": "array", + "type": ["array", "null"], "example": [ { "code": "Invalid", @@ -910,12 +737,12 @@ "$ref": "#/definitions/InnerError" }, "message": { - "type": "string", + "type": ["string", "null"], "example": "The request failed because it contained invalid values", "description": "A human-readable representation of the error." }, "target": { - "type": "string", + "type": ["string", "null"], "example": "target", "description": "The target of the error." } @@ -924,7 +751,7 @@ "description": "REST API Error." }, "RestApiErrorResponse": { - "type": "object", + "type": ["object", "null"], "properties": { "error": { "description": "The error object.", @@ -939,52 +766,42 @@ "properties": { "userZone1": { "type": ["null", "string"], - "description": "Free field whose type (Alphanumeric, Numeric, Date, List) is defined in the application setup. The imported value must be consistent with the field type. And the total length across all user zone values must be of 250 characters max", - "minLength": 0, - "maxLength": 100 + "description": "Free field whose type (Alphanumeric, Numeric, Date, List) is defined in the application setup. The imported value must be consistent with the field type. And the total length across all user zone values must be of 250 characters max" }, "userZone2": { "type": ["null", "string"], - "description": "Free field. Same behavior as field User zone 1", - "minLength": 0, - "maxLength": 100 + "description": "Free field. Same behavior as field User zone 1" }, "userZone3": { "type": ["null", "string"], - "description": "Free field. Same behavior as field User zone 1", - "minLength": 0, - "maxLength": 100 + "description": "Free field. Same behavior as field User zone 1" }, "userZone4": { "type": ["null", "string"], - "description": "Free field. Same behavior as field User zone 1", - "minLength": 0, - "maxLength": 100 + "description": "Free field. Same behavior as field User zone 1" }, "userZone5": { "type": ["null", "string"], - "description": "Free field. Same behavior as field User zone 1", - "minLength": 0, - "maxLength": 100 + "description": "Free field. Same behavior as field User zone 1" } }, "title": "UserZonesModel" }, "_links": { - "type": "object", + "type": ["object", "null"], "properties": { "current": { - "type": "string", + "type": ["string", "null"], "example": "https://host/gateway/api/resources?page.limit=10&page.offset=10", "description": "Current page." }, "next": { - "type": "string", + "type": ["string", "null"], "example": "https://host/gateway/api/resources?page.limit=10&page.offset=20", "description": "Next page." }, "prev": { - "type": "string", + "type": ["string", "null"], "example": "https://host/gateway/api/resources?page.limit=10&page.offset=0", "description": "Previous page." } @@ -992,55 +809,51 @@ "title": "_links" }, "_metadata": { - "type": "object", + "type": ["object", "null"], "properties": { "links": { "description": "Links for pages.", "$ref": "#/definitions/_links" }, "numberOfTotalResults": { - "type": "integer", + "type": ["integer", "null"], "format": "int64", "example": 100, "description": "The number of total records.", - "minimum": 0.0, "exclusiveMinimum": false }, "pageLimit": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", "example": 100, "description": "Limit the number of records per page. By default 100.", - "minimum": 1.0, "exclusiveMinimum": false }, "pageOffset": { - "type": "integer", + "type": ["integer", "null"], "format": "int32", "example": 0, "description": "Page Offset means the number of records you want to skip before starting reading. By default 0.", - "minimum": 0.0, "exclusiveMinimum": false }, "pageResults": { - "type": "integer", + "type": ["integer", "null"], "format": "int64", "example": 10, "description": "The number of records in current page.", - "minimum": 0.0, "exclusiveMinimum": false } }, "title": "_metadata" }, "_pageOfCashFlowSearchModel": { - "type": "object", + "type": ["object", "null"], "properties": { "metadata": { "$ref": "#/definitions/_metadata" }, "results": { - "type": "array", + "type": ["array", "null"], "items": { "$ref": "#/definitions/CashFlowSearchModel" } diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py index 17ea43ac3414..cac3eb31f5cf 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py @@ -270,9 +270,15 @@ def gateway_url(self, config: Mapping[str, Any]) -> str: return f"https://{config['domain']}/gateway" def check_connection(self, logger, config) -> Tuple[bool, any]: - client = KyribaClient(config["username"], config["password"], self.gateway_url(config)) - client.login() - return True, None + try: + client = KyribaClient(config["username"], config["password"], self.gateway_url(config)) + client.login() + return True, None + except Exception as e: + if isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 401: + err_message = f"Please check your `username` and `password`. Error: {repr(e)}" + return False, err_message + return False, repr(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: gateway_url = self.gateway_url(config) diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/spec.json b/airbyte-integrations/connectors/source-kyriba/source_kyriba/spec.json index 37050136e4c0..2d507d94a818 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/spec.json +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/spec.json @@ -5,32 +5,36 @@ "title": "Kyriba Spec", "type": "object", "required": ["domain", "username", "password", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "domain": { "type": "string", "description": "Kyriba domain", "title": "Domain", "examples": ["demo.kyriba.com"], - "pattern": "^[a-zA-Z0-9._-]*\\.[a-zA-Z0-9._-]*\\.[a-z]*" + "pattern": "^[a-zA-Z0-9._-]*\\.[a-zA-Z0-9._-]*\\.[a-z]*", + "order": 2 }, "username": { "type": "string", "description": "Username to be used in basic auth", - "title": "Username" + "title": "Username", + "order": 0 }, "password": { "type": "string", "description": "Password to be used in basic auth", "title": "Password", - "airbyte_secret": true + "airbyte_secret": true, + "order": 1 }, "start_date": { "type": "string", "description": "The date the sync should start from.", "title": "Start Date", "examples": ["2021-01-10"], - "pattern": "^\\d{4}\\-(0[1-9]|1[012])\\-(0[1-9]|[12][0-9]|3[01])$" + "pattern": "^\\d{4}\\-(0[1-9]|1[012])\\-(0[1-9]|[12][0-9]|3[01])$", + "order": 3 }, "end_date": { "type": "string", diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py index c2e26c3eea7c..58ff037abc67 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pytest -from source_kyriba.source import CashBalancesStream +from source_kyriba.source import BankBalancesStream from .test_streams import config @@ -14,64 +14,58 @@ @pytest.fixture def patch_base_class(mocker): # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(CashBalancesStream, "primary_key", "test_primary_key") - mocker.patch.object(CashBalancesStream, "__abstractmethods__", set()) + mocker.patch.object(BankBalancesStream, "primary_key", "test_primary_key") + mocker.patch.object(BankBalancesStream, "__abstractmethods__", set()) def test_stream_slices(patch_base_class): - stream = CashBalancesStream(**config()) - account_uuids = [{"account_uuid": "first"}, {"account_uuid": "second"}] + stream = BankBalancesStream(**config()) + account_uuids = [ + {"account_uuid": "first"}, + {"account_uuid": "second"} + ] stream.get_account_uuids = MagicMock(return_value=account_uuids) stream.start_date = date(2022, 1, 1) - stream.end_date = date(2022, 3, 1) + stream.end_date = date(2022, 1, 2) expected = [ { "account_uuid": "first", - "startDate": "2022-01-01", - "endDate": "2022-02-01", + "date": "2022-01-01", }, { "account_uuid": "second", - "startDate": "2022-01-01", - "endDate": "2022-02-01", + "date": "2022-01-01", }, { "account_uuid": "first", - "startDate": "2022-02-02", - "endDate": "2022-03-01", + "date": "2022-01-02", }, { "account_uuid": "second", - "startDate": "2022-02-02", - "endDate": "2022-03-01", - }, + "date": "2022-01-02", + } ] slices = stream.stream_slices() assert slices == expected def test_path(patch_base_class): - stream = CashBalancesStream(**config()) + stream = BankBalancesStream(**config()) inputs = {"stream_slice": {"account_uuid": "uuid"}} path = stream.path(**inputs) - assert path == "cash-balances/accounts/uuid/balances" + assert path == "bank-balances/accounts/uuid/balances" def test_request_params(patch_base_class): - stream = CashBalancesStream(**config()) + stream = BankBalancesStream(**config()) inputs = { - "stream_slice": {"account_uuid": "uuid", "endDate": "2022-02-01", "startDate": "2022-01-01"}, + "stream_slice": {"account_uuid": "uuid", "date": "2022-02-01"}, "stream_state": {}, } - stream.intraday = False + stream.balance_type = "END_OF_DAY" params = stream.request_params(**inputs) expected = { - "endDate": "2022-02-01", - "startDate": "2022-01-01", - "intraday": False, - "actual": True, - "estimatedForecasts": False, - "confirmedForecasts": False, - "dateType": "VALUE", + "date": inputs["stream_slice"]["date"], + "type": stream.balance_type, } assert params == expected diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py index 0bb0d10f8a18..1bda3981cbd6 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py @@ -13,14 +13,6 @@ "start_date": "2022-01-01", } -config = { - "username": "username", - "password": "password", - "domain": "demo.kyriba.com", - "start_date": "2022-01-01", -} - - def test_check_connection(mocker): source = SourceKyriba() KyribaClient.login = MagicMock() diff --git a/airbyte-integrations/connectors/source-kyve/main.py b/airbyte-integrations/connectors/source-kyve/main.py index f4055d71e80e..a3740b34d958 100644 --- a/airbyte-integrations/connectors/source-kyve/main.py +++ b/airbyte-integrations/connectors/source-kyve/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_kyve import SourceKyve +from source_kyve.run import run if __name__ == "__main__": - source = SourceKyve() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kyve/metadata.yaml b/airbyte-integrations/connectors/source-kyve/metadata.yaml index 6f2b6cfc27ec..c95fa5e33341 100644 --- a/airbyte-integrations/connectors/source-kyve/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyve/metadata.yaml @@ -9,6 +9,10 @@ data: icon: icon.svg license: MIT name: KYVE + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-kyve registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-kyve/setup.py b/airbyte-integrations/connectors/source-kyve/setup.py index 1d6d5bbde662..709638a8574a 100644 --- a/airbyte-integrations/connectors/source-kyve/setup.py +++ b/airbyte-integrations/connectors/source-kyve/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-kyve=source_kyve.run:run", + ], + }, name="source_kyve", description="Source implementation for KYVE.", author="KYVE Core Team", author_email="security@kyve.network", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/run.py b/airbyte-integrations/connectors/source-kyve/source_kyve/run.py new file mode 100644 index 000000000000..fc9753ebf2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kyve import SourceKyve + + +def run(): + source = SourceKyve() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-launchdarkly/main.py b/airbyte-integrations/connectors/source-launchdarkly/main.py index beb7cab5ae2f..997cfda7e441 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/main.py +++ b/airbyte-integrations/connectors/source-launchdarkly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_launchdarkly import SourceLaunchdarkly +from source_launchdarkly.run import run if __name__ == "__main__": - source = SourceLaunchdarkly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml index d55516006941..c93a438bd591 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml +++ b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml @@ -8,6 +8,10 @@ data: icon: launchdarkly.svg license: MIT name: LaunchDarkly + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-launchdarkly registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-launchdarkly/setup.py b/airbyte-integrations/connectors/source-launchdarkly/setup.py index 07d48e22e847..722dfc7a0768 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/setup.py +++ b/airbyte-integrations/connectors/source-launchdarkly/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-launchdarkly=source_launchdarkly.run:run", + ], + }, name="source_launchdarkly", description="Source implementation for Launchdarkly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py b/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py new file mode 100644 index 000000000000..9807f36d4a6e --- /dev/null +++ b/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_launchdarkly import SourceLaunchdarkly + + +def run(): + source = SourceLaunchdarkly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lemlist/main.py b/airbyte-integrations/connectors/source-lemlist/main.py index 3d71a2f889d8..5d7573f45162 100644 --- a/airbyte-integrations/connectors/source-lemlist/main.py +++ b/airbyte-integrations/connectors/source-lemlist/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lemlist import SourceLemlist +from source_lemlist.run import run if __name__ == "__main__": - source = SourceLemlist() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lemlist/metadata.yaml b/airbyte-integrations/connectors/source-lemlist/metadata.yaml index 1b9d60189c19..6fbfb68fb760 100644 --- a/airbyte-integrations/connectors/source-lemlist/metadata.yaml +++ b/airbyte-integrations/connectors/source-lemlist/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.lemlist.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lemlist registries: cloud: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/lemlist tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py index b202001b47e2..53550286882a 100644 --- a/airbyte-integrations/connectors/source-lemlist/setup.py +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lemlist=source_lemlist.run:run", + ], + }, name="source_lemlist", description="Source implementation for Lemlist.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py new file mode 100644 index 000000000000..092c7e718f2b --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lemlist import SourceLemlist + + +def run(): + source = SourceLemlist() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lever-hiring/main.py b/airbyte-integrations/connectors/source-lever-hiring/main.py index b79de94f75ee..ca4773e96a03 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/main.py +++ b/airbyte-integrations/connectors/source-lever-hiring/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lever_hiring import SourceLeverHiring +from source_lever_hiring.run import run if __name__ == "__main__": - source = SourceLeverHiring() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml index 9a49336207cb..93c0e331a63b 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml +++ b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml @@ -8,6 +8,10 @@ data: icon: leverhiring.svg license: MIT name: Lever Hiring + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lever-hiring registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-lever-hiring/setup.py b/airbyte-integrations/connectors/source-lever-hiring/setup.py index 3ceb890c8e93..b87f0ba2038a 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/setup.py +++ b/airbyte-integrations/connectors/source-lever-hiring/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lever-hiring=source_lever_hiring.run:run", + ], + }, name="source_lever_hiring", description="Source implementation for Lever Hiring.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py new file mode 100644 index 000000000000..cd8dc4e7bfc3 --- /dev/null +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lever_hiring import SourceLeverHiring + + +def run(): + source = SourceLeverHiring() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc b/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc new file mode 100644 index 000000000000..6b0b0af5e2ce --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_linkedin_ads/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-ads/README.md b/airbyte-integrations/connectors/source-linkedin-ads/README.md index 09a8052cdf6a..6d6a5d6b6b92 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/README.md +++ b/airbyte-integrations/connectors/source-linkedin-ads/README.md @@ -1,118 +1,55 @@ -# Linkedin Ads Source Connector +# Linkedin-Ads source connector -This is the repository for the Linkedin Ads source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/linkedin-ads). + +This is the repository for the Linkedin-Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-ads). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python3 -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-ads) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_ads/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/linkedin-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_ads/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-ads test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-linkedin-ads spec +poetry run source-linkedin-ads check --config secrets/config.json +poetry run source-linkedin-ads discover --config secrets/config.json +poetry run source-linkedin-ads read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-linkedin-ads build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-linkedin-ads:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-linkedin-ads:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-linkedin-ads:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-linkedin-ads:dev . -# Running the spec command against your patched connector -docker run airbyte/source-linkedin-ads:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-linkedin-ads:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-ads:dev disco docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-linkedin-ads test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linkedin-ads test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-ads.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-ads.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl index c40775868b68..2ccd9635d7f7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl @@ -4,10 +4,10 @@ {"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508720451", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-06-14T10:09:22+00:00"}, "emitted_at": 1697196559364} {"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508774356", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-21T21:28:19+00:00"}, "emitted_at": 1697196559760} {"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508777244", "created": "2021-08-21T21:27:55+00:00", "lastModified": "2021-08-21T21:27:55+00:00"}, "emitted_at": 1697196560036} -{"stream": "ad_campaign_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": -2e-18, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": -2e-18, "conversionValueInLocalCurrency": 0.0, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "approximateUniqueImpressions": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-26", "end_date": "2023-08-26", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "postClickJobApplyClicks": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 1.0, "postClickJobApplications": 0.0, "otherEngagements": 0.0, "jobApplyClicks": 0.0, "jobApplications": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:sponsoredCampaign:252074216"], "likes": 0.0, "postClickRegistrations": 0.0}, "emitted_at": 1697196575455} -{"stream": "ad_campaign_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 100.00000000000004, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 100.00000000000004, "conversionValueInLocalCurrency": 0.0, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 106.0, "cardClicks": 0.0, "approximateUniqueImpressions": 17392.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "postClickJobApplyClicks": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 106.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 19464.0, "postClickJobApplications": 0.0, "otherEngagements": 0.0, "jobApplyClicks": 0.0, "jobApplications": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:sponsoredCampaign:252074216"], "likes": 0.0, "postClickRegistrations": 0.0, "videoCompletions": 0.0, "registrations": 0.0, "talentLeads": 0.0, "viralCardImpressions": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "postViewJobApplyClicks": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "viralCardClicks": 0.0, "postViewRegistrations": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 106.0, "reactions": 0.0, "postViewJobApplications": 0.0, "videoViews": 0.0}, "emitted_at": 1697196575461} -{"stream": "ad_creative_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": -2e-18, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": -2e-18, "conversionValueInLocalCurrency": 0.0, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "approximateUniqueImpressions": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-26", "end_date": "2023-08-26", "pivotValue": "urn:li:sponsoredCreative:287513206", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "postClickJobApplyClicks": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 1.0, "postClickJobApplications": 0.0, "otherEngagements": 0.0, "jobApplyClicks": 0.0, "jobApplications": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:sponsoredCreative:287513206"], "likes": 0.0, "postClickRegistrations": 0.0}, "emitted_at": 1697196599913} -{"stream": "ad_creative_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 100.00000000000004, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 100.00000000000004, "conversionValueInLocalCurrency": 0.0, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 106.0, "cardClicks": 0.0, "approximateUniqueImpressions": 17392.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCreative:287513206", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "postClickJobApplyClicks": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 106.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 19464.0, "postClickJobApplications": 0.0, "otherEngagements": 0.0, "jobApplyClicks": 0.0, "jobApplications": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:sponsoredCreative:287513206"], "likes": 0.0, "postClickRegistrations": 0.0, "videoCompletions": 0.0, "registrations": 0.0, "talentLeads": 0.0, "viralCardImpressions": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "postViewJobApplyClicks": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "viralCardClicks": 0.0, "postViewRegistrations": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 106.0, "reactions": 0.0, "postViewJobApplications": 0.0, "videoViews": 0.0}, "emitted_at": 1697196599918} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":-2E-18,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":-2E-18,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":0.0,"cardClicks":0.0,"approximateUniqueImpressions":0.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-26","end_date":"2023-08-26","pivotValue":"urn:li:sponsoredCampaign:252074216","oneClickLeads":0.0,"landingPageClicks":0.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":1.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCampaign:252074216"],"likes":0.0},"emitted_at":1702655286996} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":100.00000000000004,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":100.00000000000004,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":106.0,"cardClicks":0.0,"approximateUniqueImpressions":17392.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-25","end_date":"2023-08-25","pivotValue":"urn:li:sponsoredCampaign:252074216","oneClickLeads":0.0,"landingPageClicks":106.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":19464.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCampaign:252074216"],"likes":0.0,"videoCompletions":0.0,"viralCardImpressions":0.0,"videoFirstQuartileCompletions":0.0,"textUrlClicks":0.0,"videoStarts":0.0,"sends":0.0,"shares":0.0,"videoMidpointCompletions":0.0,"validWorkEmailLeads":0.0,"viralCardClicks":0.0,"videoThirdQuartileCompletions":0.0,"totalEngagements":106.0,"reactions":0.0,"videoViews":0.0},"emitted_at":1702655287003} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":-2E-18,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":-2E-18,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":0.0,"cardClicks":0.0,"approximateUniqueImpressions":0.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-26","end_date":"2023-08-26","pivotValue":"urn:li:sponsoredCreative:287513206","oneClickLeads":0.0,"landingPageClicks":0.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":1.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCreative:287513206"],"likes":0.0},"emitted_at":1702656821471} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":100.00000000000004,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":100.00000000000004,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":106.0,"cardClicks":0.0,"approximateUniqueImpressions":17392.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-25","end_date":"2023-08-25","pivotValue":"urn:li:sponsoredCreative:287513206","oneClickLeads":0.0,"landingPageClicks":106.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":19464.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCreative:287513206"],"likes":0.0,"videoCompletions":0.0,"viralCardImpressions":0.0,"videoFirstQuartileCompletions":0.0,"textUrlClicks":0.0,"videoStarts":0.0,"sends":0.0,"shares":0.0,"videoMidpointCompletions":0.0,"validWorkEmailLeads":0.0,"viralCardClicks":0.0,"videoThirdQuartileCompletions":0.0,"totalEngagements":106.0,"reactions":0.0,"videoViews":0.0},"emitted_at":1702656821475} {"stream": "ad_impression_device_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 2.29, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 2.29, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "oneClickLeadFormOpens": 0.0, "follows": 0.0, "impressions": 498.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["UNDETECTED"], "likes": 0.0, "videoCompletions": 0.0, "viralCardImpressions": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "viralCardClicks": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 20.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196622374} {"stream": "ad_impression_device_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": -2e-18, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": -2e-18, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-26", "end_date": "2023-08-26", "pivotValue": "urn:li:sponsoredCampaign:252074216", "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "oneClickLeadFormOpens": 0.0, "follows": 0.0, "impressions": 1.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["MOBILE_WEB"], "likes": 0.0}, "emitted_at": 1697196622395} {"stream": "ad_member_company_size_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 24.457317520310493, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 24.457317520310493, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 9.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 9.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 1480.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["SIZE_2_TO_10"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 8.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196644434} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/main.py b/airbyte-integrations/connectors/source-linkedin-ads/main.py index c51fcd1a5cc5..899a7e8614a4 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/main.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linkedin_ads import SourceLinkedinAds +from source_linkedin_ads.run import run if __name__ == "__main__": - source = SourceLinkedinAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml index 78698dbe4944..e7a9a9212eb0 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml @@ -7,11 +7,11 @@ data: - linkedin.com - api.linkedin.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 137ece28-5434-455c-8f34-69dc3782f451 - dockerImageTag: 0.6.4 + dockerImageTag: 0.7.0 dockerRepository: airbyte/source-linkedin-ads documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads githubIssueLabel: source-linkedin-ads @@ -19,6 +19,10 @@ data: license: MIT maxSecondsBetweenMessages: 86400 name: LinkedIn Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-linkedin-ads registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-linkedin-ads/poetry.lock b/airbyte-integrations/connectors/source-linkedin-ads/poetry.lock new file mode 100644 index 000000000000..4c1cbe822ae2 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "861d01a7b883a61e01367a1c883fbc1699ef39984c364c6bd9984703c1747375" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml b/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml new file mode 100644 index 000000000000..bcbb55b92573 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.7.0" +name = "source-linkedin-ads" +description = "Source implementation for Linkedin Ads." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/linkedin-ads" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_linkedin_ads" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.63.2" + +[tool.poetry.scripts] +source-linkedin-ads = "source_linkedin_ads.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.6.1" +requests-mock = "^1.11.0" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt b/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py deleted file mode 100644 index 1c15f41abf5f..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.50", -] - -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "requests-mock", -] - -setup( - name="source_linkedin_ads", - description="Source implementation for Linkedin Ads.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py deleted file mode 100644 index 6963ecf1ac9f..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py +++ /dev/null @@ -1,207 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from collections import defaultdict -from typing import Any, Iterable, List, Mapping - -import pendulum as pdm - -from .utils import get_parent_stream_values - -# LinkedIn has a max of 20 fields per request. We make chunks by size of 19 fields -# to have the `dateRange` be included as well. -FIELDS_CHUNK_SIZE = 19 -# Number of days ahead for date slices, from start date. -WINDOW_IN_DAYS = 30 -# List of Reporting Metrics fields available for fetch -ANALYTICS_FIELDS_V2: List = [ - "actionClicks", - "adUnitClicks", - "approximateUniqueImpressions", - "cardClicks", - "cardImpressions", - "clicks", - "commentLikes", - "comments", - "companyPageClicks", - "conversionValueInLocalCurrency", - "costInLocalCurrency", - "costInUsd", - "dateRange", - "documentCompletions", - "documentFirstQuartileCompletions", - "documentMidpointCompletions", - "documentThirdQuartileCompletions", - "downloadClicks", - "externalWebsiteConversions", - "externalWebsitePostClickConversions", - "externalWebsitePostViewConversions", - "follows", - "fullScreenPlays", - "impressions", - "jobApplications", - "jobApplyClicks", - "landingPageClicks", - "leadGenerationMailContactInfoShares", - "leadGenerationMailInterestedClicks", - "likes", - "oneClickLeadFormOpens", - "oneClickLeads", - "opens", - "otherEngagements", - "pivotValues", - "postClickJobApplications", - "postClickJobApplyClicks", - "postClickRegistrations", - "postViewJobApplications", - "postViewJobApplyClicks", - "postViewRegistrations", - "reactions", - "registrations", - "sends", - "shares", - "talentLeads", - "textUrlClicks", - "totalEngagements", - "validWorkEmailLeads", - "videoCompletions", - "videoFirstQuartileCompletions", - "videoMidpointCompletions", - "videoStarts", - "videoThirdQuartileCompletions", - "videoViews", - "viralCardClicks", - "viralCardImpressions", - "viralClicks", - "viralCommentLikes", - "viralComments", - "viralCompanyPageClicks", - "viralDocumentCompletions", - "viralDocumentFirstQuartileCompletions", - "viralDocumentMidpointCompletions", - "viralDocumentThirdQuartileCompletions", - "viralDownloadClicks", - "viralExternalWebsiteConversions", - "viralExternalWebsitePostClickConversions", - "viralExternalWebsitePostViewConversions", - "viralFollows", - "viralFullScreenPlays", - "viralImpressions", - "viralJobApplications", - "viralJobApplyClicks", - "viralLandingPageClicks", - "viralLikes", - "viralOneClickLeadFormOpens", - "viralOneClickLeads", - "viralOtherEngagements", - "viralPostClickJobApplications", - "viralPostClickJobApplyClicks", - "viralPostClickRegistrations", - "viralPostViewJobApplications", - "viralPostViewJobApplyClicks", - "viralPostViewRegistrations", - "viralReactions", - "viralRegistrations", - "viralShares", - "viralTotalEngagements", - "viralVideoCompletions", - "viralVideoFirstQuartileCompletions", - "viralVideoMidpointCompletions", - "viralVideoStarts", - "viralVideoThirdQuartileCompletions", - "viralVideoViews", -] -# Fields that are always present in fields_set chunks -BASE_ANALLYTICS_FIELDS = ["dateRange"] - - -def chunk_analytics_fields( - fields: List = ANALYTICS_FIELDS_V2, - base_fields: List = BASE_ANALLYTICS_FIELDS, - fields_chunk_size: int = FIELDS_CHUNK_SIZE, -) -> Iterable[List]: - """ - Chunks the list of available fields into the chunks of equal size. - """ - # Make chunks - chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) - # Make sure base_fields are within the chunks - for chunk in chunks: - for field in base_fields: - if field not in chunk: - chunk.append(field) - yield from chunks - - -def make_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[List]: - """ - Produces date slices from start_date to end_date (if specified), - otherwise end_date will be present time. - """ - start = pdm.parse(start_date) - end = pdm.parse(end_date) if end_date else pdm.now() - date_slices = [] - while start < end: - slice_end_date = start.add(days=window_in_days) - date_slice = { - "start.day": start.day, - "start.month": start.month, - "start.year": start.year, - "end.day": slice_end_date.day, - "end.month": slice_end_date.month, - "end.year": slice_end_date.year, - } - date_slices.append({"dateRange": date_slice}) - start = slice_end_date - yield from date_slices - - -def make_analytics_slices( - record: Mapping[str, Any], key_value_map: Mapping[str, Any], start_date: str, end_date: str = None -) -> Iterable[Mapping[str, Any]]: - """ - We drive the ability to directly pass the prepared parameters inside the stream_slice. - The output of this method is ready slices for analytics streams: - """ - # define the base_slice - base_slice = get_parent_stream_values(record, key_value_map) - # add chunked fields, date_slices to the base_slice - analytics_slices = [] - for fields_set in chunk_analytics_fields(): - base_slice["fields"] = ",".join(map(str, fields_set)) - for date_slice in make_date_slices(start_date, end_date): - base_slice.update(**date_slice) - analytics_slices.append(base_slice.copy()) - yield from analytics_slices - - -def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Produces the date range parameters from input stream_slice - """ - date_range = stream_slice["dateRange"] - return { - "dateRange": f"(start:(year:{date_range['start.year']},month:{date_range['start.month']},day:{date_range['start.day']})," - f"end:(year:{date_range['end.year']},month:{date_range['end.month']},day:{date_range['end.day']}))", - # Chunk of fields - "fields": stream_slice["fields"], - } - - -def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) -> Iterable[Mapping[str, Any]]: - """ - We need to merge the chunked API responses - into the single structure using any available unique field. - """ - # Merge the pieces together - merged = defaultdict(dict) - for chunk in chunked_result: - for item in chunk: - merged[item[merge_by_key]].update(item) - # Clean up the result by getting out the values of the merged keys - result = [] - for item in merged: - result.append(merged.get(item)) - yield from result diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py new file mode 100644 index 000000000000..f58da0e25c8b --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py @@ -0,0 +1,373 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from urllib.parse import urlencode + +import pendulum +import requests +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils import casing +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_protocol.models import SyncMode +from source_linkedin_ads.streams import Campaigns, Creatives, IncrementalLinkedinAdsStream + +from .utils import get_parent_stream_values, transform_data + +# Number of days ahead for date slices, from start date. +WINDOW_IN_DAYS = 30 +# List of Reporting Metrics fields available for fetch +ANALYTICS_FIELDS_V2: List = [ + "actionClicks", + "adUnitClicks", + "approximateUniqueImpressions", + "cardClicks", + "cardImpressions", + "clicks", + "commentLikes", + "comments", + "companyPageClicks", + "conversionValueInLocalCurrency", + "costInLocalCurrency", + "costInUsd", + "dateRange", + "documentCompletions", + "documentFirstQuartileCompletions", + "documentMidpointCompletions", + "documentThirdQuartileCompletions", + "downloadClicks", + "externalWebsiteConversions", + "externalWebsitePostClickConversions", + "externalWebsitePostViewConversions", + "follows", + "fullScreenPlays", + "impressions", + "jobApplications", + "jobApplyClicks", + "landingPageClicks", + "leadGenerationMailContactInfoShares", + "leadGenerationMailInterestedClicks", + "likes", + "oneClickLeadFormOpens", + "oneClickLeads", + "opens", + "otherEngagements", + "pivotValues", + "postClickJobApplications", + "postClickJobApplyClicks", + "postClickRegistrations", + "postViewJobApplications", + "postViewJobApplyClicks", + "postViewRegistrations", + "reactions", + "registrations", + "sends", + "shares", + "talentLeads", + "textUrlClicks", + "totalEngagements", + "validWorkEmailLeads", + "videoCompletions", + "videoFirstQuartileCompletions", + "videoMidpointCompletions", + "videoStarts", + "videoThirdQuartileCompletions", + "videoViews", + "viralCardClicks", + "viralCardImpressions", + "viralClicks", + "viralCommentLikes", + "viralComments", + "viralCompanyPageClicks", + "viralDocumentCompletions", + "viralDocumentFirstQuartileCompletions", + "viralDocumentMidpointCompletions", + "viralDocumentThirdQuartileCompletions", + "viralDownloadClicks", + "viralExternalWebsiteConversions", + "viralExternalWebsitePostClickConversions", + "viralExternalWebsitePostViewConversions", + "viralFollows", + "viralFullScreenPlays", + "viralImpressions", + "viralJobApplications", + "viralJobApplyClicks", + "viralLandingPageClicks", + "viralLikes", + "viralOneClickLeadFormOpens", + "viralOneClickLeads", + "viralOtherEngagements", + "viralPostClickJobApplications", + "viralPostClickJobApplyClicks", + "viralPostClickRegistrations", + "viralPostViewJobApplications", + "viralPostViewJobApplyClicks", + "viralPostViewRegistrations", + "viralReactions", + "viralRegistrations", + "viralShares", + "viralTotalEngagements", + "viralVideoCompletions", + "viralVideoFirstQuartileCompletions", + "viralVideoMidpointCompletions", + "viralVideoStarts", + "viralVideoThirdQuartileCompletions", + "viralVideoViews", +] + + +class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): + """ + AdAnalytics Streams more info: + https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#analytics-finder + """ + + endpoint = "adAnalytics" + # For Analytics streams, the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc.] + `end_date` + primary_key = ["pivotValue", "end_date"] + cursor_field = "end_date" + records_limit = 15000 + FIELDS_CHUNK_SIZE = 19 + + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics") + + def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs): + self.user_stream_name = name + if pivot_by: + self.pivot_by = pivot_by + if time_granularity: + self.time_granularity = time_granularity + super().__init__(**kwargs) + + @property + @abstractmethod + def search_param(self) -> str: + """ + :return: Search parameters for the request + """ + + @property + @abstractmethod + def search_param_value(self) -> str: + """ + :return: Name field to filter by + """ + + @property + @abstractmethod + def parent_values_map(self) -> Mapping[str, str]: + """ + :return: Mapping for parent child relation + """ + + @property + def name(self) -> str: + """We override the stream name to let the user change it via configuration.""" + name = self.user_stream_name or self.__class__.__name__ + return casing.camel_to_snake(name) + + @property + def base_analytics_params(self) -> MutableMapping[str, Any]: + """Define the base parameters for analytics streams""" + return {"q": "analytics", "pivot": f"(value:{self.pivot_by})", "timeGranularity": f"(value:{self.time_granularity})"} + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + headers = super().request_headers(stream_state, stream_slice, next_page_token) + return headers | {"X-Restli-Protocol-Version": "2.0.0"} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = self.base_analytics_params + params.update(**self.update_analytics_params(stream_slice)) + params[self.search_param] = f"List(urn%3Ali%3A{self.search_param_value}%3A{self.get_primary_key_from_slice(stream_slice)})" + return urlencode(params, safe="():,%") + + @staticmethod + def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Produces the date range parameters from input stream_slice + """ + date_range = stream_slice["dateRange"] + return { + "dateRange": f"(start:(year:{date_range['start.year']},month:{date_range['start.month']},day:{date_range['start.day']})," + f"end:(year:{date_range['end.year']},month:{date_range['end.month']},day:{date_range['end.day']}))", + # Chunk of fields + "fields": stream_slice["fields"], + } + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Pagination is not supported + (See Restrictions: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-09&tabs=http#restrictions) + """ + parsed_response = response.json() + if len(parsed_response.get("elements")) < self.records_limit: + return None + raise Exception( + f"Limit {self.records_limit} elements exceeded. " + f"Try to request your data in more granular pieces. " + f"(For example switch `Time Granularity` from MONTHLY to DAILY)" + ) + + def get_primary_key_from_slice(self, stream_slice) -> str: + return stream_slice.get(self.primary_slice_key) + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[List[Mapping[str, Any]]]: + """ + LinkedIn has a max of 20 fields per request. We make chunks by size of 19 fields to have the `dateRange` be included as well. + https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-05&tabs=http#requesting-specific-metrics-in-the-analytics-finder + + :param sync_mode: + :param cursor_field: + :param stream_state: + :return: Iterable with List of stream slices within the same date range and chunked fields, example + [{'campaign_id': 123, 'fields': 'field_1,field_2,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}, + {'campaign_id': 123, 'fields': 'field_2,field_3,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}, + {'campaign_id': 123, 'fields': 'field_4,field_5,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}] + + """ + parent_stream = self.parent_stream(config=self.config) + stream_state = stream_state or {self.cursor_field: self.config.get("start_date")} + for record in parent_stream.read_records(sync_mode=sync_mode): + base_slice = get_parent_stream_values(record, self.parent_values_map) + for date_slice in self.get_date_slices(stream_state.get(self.cursor_field), self.config.get("end_date")): + date_slice_with_fields: List = [] + for fields_set in self.chunk_analytics_fields(): + base_slice["fields"] = ",".join(fields_set) + date_slice_with_fields.append(base_slice | date_slice) + yield date_slice_with_fields + + @staticmethod + def get_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[Mapping[str, Any]]: + """ + Produces date slices from start_date to end_date (if specified), + otherwise end_date will be present time. + """ + start = pendulum.parse(start_date) + end = pendulum.parse(end_date) if end_date else pendulum.now() + date_slices = [] + while start < end: + slice_end_date = start.add(days=window_in_days) + date_slice = { + "start.day": start.day, + "start.month": start.month, + "start.year": start.year, + "end.day": slice_end_date.day, + "end.month": slice_end_date.month, + "end.year": slice_end_date.year, + } + date_slices.append({"dateRange": date_slice}) + start = slice_end_date + yield from date_slices + + @staticmethod + def chunk_analytics_fields( + fields: List = ANALYTICS_FIELDS_V2, + fields_chunk_size: int = FIELDS_CHUNK_SIZE, + ) -> Iterable[List]: + """ + Chunks the list of available fields into the chunks of equal size. + """ + # Make chunks + chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) + # Make sure base_fields are within the chunks + for chunk in chunks: + if "dateRange" not in chunk: + chunk.append("dateRange") + yield from chunks + + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + merged_records = defaultdict(dict) + for field_slice in stream_slice: + for rec in super().read_records(stream_slice=field_slice, **kwargs): + merged_records[rec[self.cursor_field]].update(rec) + yield from merged_records.values() + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + We need to get out the nested complex data structures for further normalization, so the transform_data method is applied. + """ + for rec in transform_data(response.json().get("elements")): + yield rec | {"pivotValue": f"urn:li:{self.search_param_value}:{self.get_primary_key_from_slice(kwargs.get('stream_slice'))}"} + + +class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): + """ + Campaign Analytics stream. + """ + + endpoint = "adAnalytics" + + parent_stream = Campaigns + parent_values_map = {"campaign_id": "id"} + search_param = "campaigns" + search_param_value = "sponsoredCampaign" + pivot_by = "CAMPAIGN" + time_granularity = "DAILY" + + +class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): + """ + Creative Analytics stream. + """ + + parent_stream = Creatives + parent_values_map = {"creative_id": "id"} + search_param = "creatives" + search_param_value = "sponsoredCreative" + pivot_by = "CREATIVE" + time_granularity = "DAILY" + + def get_primary_key_from_slice(self, stream_slice) -> str: + creative_id = stream_slice.get(self.primary_slice_key).split(":")[-1] + return creative_id + + +class AdImpressionDeviceAnalytics(AdCampaignAnalytics): + pivot_by = "IMPRESSION_DEVICE_TYPE" + + +class AdMemberCompanySizeAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY_SIZE" + + +class AdMemberIndustryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_INDUSTRY" + + +class AdMemberSeniorityAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_SENIORITY" + + +class AdMemberJobTitleAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_TITLE" + + +class AdMemberJobFunctionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_FUNCTION" + + +class AdMemberCountryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COUNTRY_V2" + + +class AdMemberRegionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_REGION_V2" + + +class AdMemberCompanyAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py new file mode 100644 index 000000000000..e37dbe66f17f --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_ads import SourceLinkedinAds + + +def run(): + source = SourceLinkedinAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 8a89a468eaab..4716fe245093 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -11,9 +11,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType -from source_linkedin_ads.streams import ( - Accounts, - AccountUsers, +from source_linkedin_ads.analytics_streams import ( AdCampaignAnalytics, AdCreativeAnalytics, AdImpressionDeviceAnalytics, @@ -25,11 +23,8 @@ AdMemberJobTitleAnalytics, AdMemberRegionAnalytics, AdMemberSeniorityAnalytics, - CampaignGroups, - Campaigns, - Conversions, - Creatives, ) +from source_linkedin_ads.streams import Accounts, AccountUsers, CampaignGroups, Campaigns, Conversions, Creatives logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json index e130c0a073de..8bfe86125ad1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json @@ -111,9 +111,9 @@ "MEMBER_COMPANY_SIZE", "MEMBER_INDUSTRY", "MEMBER_SENIORITY", - "MEMBER_JOB_TITLE ", - "MEMBER_JOB_FUNCTION ", - "MEMBER_COUNTRY_V2 ", + "MEMBER_JOB_TITLE", + "MEMBER_JOB_FUNCTION", + "MEMBER_COUNTRY_V2", "MEMBER_REGION_V2", "MEMBER_COMPANY", "PLACEMENT_NAME", diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index ca13c1856b35..5151d52a961d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -10,13 +10,9 @@ import pendulum import requests -from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils import casing -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from .analytics import make_analytics_slices, merge_chunks, update_analytics_params from .utils import get_parent_stream_values, transform_data logger = logging.getLogger("airbyte") @@ -32,7 +28,6 @@ class LinkedinAdsStream(HttpStream, ABC): url_base = "https://api.linkedin.com/rest/" primary_key = "id" records_limit = 500 - endpoint = None transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) def __init__(self, config: Dict): @@ -52,6 +47,11 @@ def accounts(self): """Property to return the list of the user Account Ids from input""" return ",".join(map(str, self.config.get("account_ids", []))) + @property + @abstractmethod + def endpoint(self) -> str: + """Endpoint associated with the current stream""" + def path( self, *, @@ -92,7 +92,7 @@ def request_params( def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ - We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. + We need to get out the nested complex data structures for further normalization, so the transform_data method is applied. """ for record in transform_data(response.json().get("elements")): yield self._date_time_to_rfc3339(record) @@ -126,6 +126,7 @@ class Accounts(LinkedinAdsStream): """ endpoint = "adAccounts" + use_cache = True def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: """ @@ -169,12 +170,12 @@ def primary_slice_key(self) -> str: @property @abstractmethod - def parent_stream(self) -> object: - """Defines the parrent stream for slicing, the class object should be provided.""" + def parent_stream(self) -> LinkedinAdsStream: + """Defines the parent stream for slicing, the class object should be provided.""" @property def state_checkpoint_interval(self) -> Optional[int]: - """Define the checkpoint from the records output size.""" + """Define the checkpoint from the record output size.""" return 100 def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: @@ -182,11 +183,11 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field))} -class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream): +class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream, ABC): """ This class stands for provide stream slicing for other dependent streams. :: `parent_stream` - the reference to the parent stream class, - by default it's referenced to the Accounts stream class, as far as majority of streams are using it. + by default it's referenced to the Accounts stream class, as far as a majority of streams are using it. :: `parent_values_map` - key_value map for stream slices in a format: {: } :: `search_param` - the query param to pass with request_params """ @@ -315,7 +316,7 @@ class Creatives(LinkedInAdsStreamSlicing): endpoint = "creatives" parent_stream = Accounts cursor_field = "lastModifiedAt" - # standard records_limit=500 returns error 400: Request would return too many entities; https://github.com/airbytehq/oncall/issues/2159 + # standard records_limit=500 returns error 400: Request would return too many entities; https://github.com/airbytehq/oncall/issues/2159 records_limit = 100 def path( @@ -388,161 +389,3 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late else current_stream_state ) return {self.cursor_field: max(latest_record.get(self.cursor_field), int(current_stream_state.get(self.cursor_field)))} - - -class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): - """ - AdAnalytics Streams more info: - https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#analytics-finder - """ - - endpoint = "adAnalytics" - # For Analytics streams the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc] + `end_date` - primary_key = ["pivotValue", "end_date"] - cursor_field = "end_date" - records_limit = 15000 - - def get_json_schema(self) -> Mapping[str, Any]: - return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics") - - def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs): - self.user_stream_name = name - if pivot_by: - self.pivot_by = pivot_by - if time_granularity: - self.time_granularity = time_granularity - super().__init__(**kwargs) - - @property - def name(self) -> str: - """We override stream name to let the user change it via configuration.""" - name = self.user_stream_name or self.__class__.__name__ - return casing.camel_to_snake(name) - - @property - def base_analytics_params(self) -> MutableMapping[str, Any]: - """Define the base parameters for analytics streams""" - return {"q": "analytics", "pivot": f"(value:{self.pivot_by})", "timeGranularity": f"(value:{self.time_granularity})"} - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - headers = super().request_headers(stream_state, stream_slice, next_page_token) - return headers | {"X-Restli-Protocol-Version": "2.0.0"} - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = self.base_analytics_params - params.update(**update_analytics_params(stream_slice)) - params[self.search_param] = f"List(urn%3Ali%3A{self.search_param_value}%3A{self.get_primary_key_from_slice(stream_slice)})" - return urlencode(params, safe="():,%") - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Pagination is not supported - (See Restrictions: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-09&tabs=http#restrictions) - """ - parsed_response = response.json() - if len(parsed_response.get("elements")) < self.records_limit: - return None - raise Exception( - f"Limit {self.records_limit} elements exceeded. " - f"Try to request your data in more granular pieces. " - f"(For example switch `Time Granularity` from MONTHLY to DAILY)" - ) - - def get_primary_key_from_slice(self, stream_slice) -> str: - return stream_slice.get(self.primary_slice_key) - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {self.cursor_field: self.config.get("start_date")} - parent_stream = self.parent_stream(config=self.config) - for record in parent_stream.read_records(**kwargs): - result_chunks = [] - for analytics_slice in make_analytics_slices( - record, self.parent_values_map, stream_state.get(self.cursor_field), self.config.get("end_date") - ): - child_stream_slice = super().read_records(stream_slice=analytics_slice, **kwargs) - result_chunks.append(child_stream_slice) - yield from merge_chunks(result_chunks, self.cursor_field) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. - """ - for rec in transform_data(response.json().get("elements")): - yield rec | {"pivotValue": f"urn:li:{self.search_param_value}:{self.get_primary_key_from_slice(kwargs.get('stream_slice'))}"} - - -class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): - """ - Campaign Analytics stream. - """ - - endpoint = "adAnalytics" - - parent_stream = Campaigns - parent_values_map = {"campaign_id": "id"} - search_param = "campaigns" - search_param_value = "sponsoredCampaign" - pivot_by = "CAMPAIGN" - time_granularity = "DAILY" - - -class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): - """ - Creative Analytics stream. - """ - - parent_stream = Creatives - parent_values_map = {"creative_id": "id"} - search_param = "creatives" - search_param_value = "sponsoredCreative" - pivot_by = "CREATIVE" - time_granularity = "DAILY" - - def get_primary_key_from_slice(self, stream_slice) -> str: - creative_id = stream_slice.get(self.primary_slice_key).split(":")[-1] - return creative_id - - -class AdImpressionDeviceAnalytics(AdCampaignAnalytics): - pivot_by = "IMPRESSION_DEVICE_TYPE" - - -class AdMemberCompanySizeAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COMPANY_SIZE" - - -class AdMemberIndustryAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_INDUSTRY" - - -class AdMemberSeniorityAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_SENIORITY" - - -class AdMemberJobTitleAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_JOB_TITLE" - - -class AdMemberJobFunctionAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_JOB_FUNCTION" - - -class AdMemberCountryAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COUNTRY_V2" - - -class AdMemberRegionAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_REGION_V2" - - -class AdMemberCompanyAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COMPANY" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index f000f88f343a..9872ea0055b7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -14,18 +14,11 @@ DESTINATION_RESERVED_KEYWORDS: list = ["pivot"] -def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: +def get_parent_stream_values(record: Mapping[str, Any], key_value_map: Mapping[str, str]) -> Mapping[str, Any]: """ - Outputs the Dict with key:value slices for the stream. - :: EXAMPLE: - Input: - records = [{dict}, {dict}, ...], - key_value_map = {: } - - Output: - { - : records..value, - } + :param record: Mapping[str, Any] + :param key_value_map: Mapping[str, str] {: } + :return: Mapping[str, str] { : records..value} """ result = {} for key in key_value_map: diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py deleted file mode 100644 index 2b4f603356f4..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Dict, List - -""" -This is the example of input record for the test_make_analytics_slices. -""" -test_input_record: Dict = { - "id": 123, - "audienceExpansionEnabled": True, - "test": False, - "format": "STANDARD_UPDATE", - "servingStatuses": ["CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], - "version": {"versionTag": "2"}, - "objectiveType": "TEST_TEST", - "associatedEntity": "urn:li:organization:456", - "offsitePreferences": { - "iabCategories": {"exclude": []}, - "publisherRestrictionFiles": {"exclude": []}, - }, - "campaignGroup": "urn:li:sponsoredCampaignGroup:1234567", - "account": "urn:li:sponsoredAccount:123456", - "status": "ACTIVE", - "created": "2021-08-06 06:03:52", - "lastModified": "2021-08-06 06:09:04", -} - -""" -This is the expected output from the `make_analytics_slices` method. -VALID PARAMETERS FOR THE OUTPUT ARE: -: TEST_KEY_VALUE_MAP = {"campaign_id": "id"} -: TEST_START_DATE = "2021-08-01" -: TEST_END_DATE = "2021-09-30" - -Change the input parameters inside of test_make_analytics_slices.py unit test. -Make sure for valid KEY_VALUE_MAP references inside of the `test_input_record` -""" -test_output_slices: List = [ - { - "camp_id": 123, - "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021, - }, - }, - { - "camp_id": 123, - "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions", - "dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange", - "dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange", - "dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange", - "dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange", - "dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange", - "dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange", - "dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange", - "dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}, - }, - { - "camp_id": 123, - "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange", - "dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}, - }, -] - -""" This is the example of the input chunks for the `test_merge_chunks` """ -test_input_result_record_chunks = [ - [ - { - "field_1": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_1": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_1": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], - [ - { - "field_2": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_2": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_2": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], - [ - { - "field_3": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_3": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_3": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], -] - -""" This is the expected test ouptput from the `merge_chunks` method from analytics module """ -test_output_merged_chunks = [ - { - "field_1": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - "field_2": "test1", - "field_3": "test1", - }, - { - "field_1": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - "field_2": "test2", - "field_3": "test2", - }, - { - "field_1": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - "field_2": "test3", - "field_3": "test3", - }, -] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py deleted file mode 100644 index c360c249159f..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_linkedin_ads.analytics import chunk_analytics_fields - -# Test chunk size for each field set -TEST_FIELDS_CHUNK_SIZE = 3 -# Test fields assuming they are really available for the fetch -TEST_ANALYTICS_FIELDS = [ - "field_1", - "base_field_1", - "field_2", - "base_field_2", - "field_3", - "field_4", - "field_5", - "field_6", - "field_7", - "field_8", -] -# Fields that are always present in fields_set chunks -TEST_BASE_ANALLYTICS_FIELDS = ["base_field_1", "base_field_2"] - - -def test_chunk_analytics_fields(): - """ - We expect to truncate the fields list into the chunks of equal size, - with TEST_BASE_ANALLYTICS_FIELDS presence in each chunk, - order is not matter. - """ - expected_output = [ - ["field_1", "base_field_1", "field_2", "base_field_2"], - ["base_field_2", "field_3", "field_4", "base_field_1"], - ["field_5", "field_6", "field_7", "base_field_1", "base_field_2"], - ["field_8", "base_field_1", "base_field_2"], - ] - - assert list(chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_BASE_ANALLYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py deleted file mode 100644 index f2579852d87c..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from samples.test_data_for_analytics import test_input_record, test_output_slices -from source_linkedin_ads.analytics import make_analytics_slices - -# Test input arguments for the `make_analytics_slices` -TEST_KEY_VALUE_MAP = {"camp_id": "id"} -TEST_START_DATE = "2021-08-01" -TEST_END_DATE = "2021-09-30" - -# This is the mock of the request_params -TEST_REQUEST_PRAMS = {} - - -def test_make_analytics_slices(): - assert list(make_analytics_slices(test_input_record, TEST_KEY_VALUE_MAP, TEST_START_DATE, TEST_END_DATE)) == test_output_slices diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py deleted file mode 100644 index ec5e5c9d5d85..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_linkedin_ads.analytics import make_date_slices - -TEST_START_DATE = "2021-08-01" -TEST_END_DATE = "2021-10-01" - - -def test_make_date_slices(): - """ - : By default we use the `WINDOW_SIZE = 30`, as it set in the analytics module - : This value could be changed by setting the corresponding argument in the method. - : The `end_date` is not specified by default, but for this test it was specified to have the test static. - """ - - expected_output = [ - {"dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}}, - {"dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}}, - {"dateRange": {"start.day": 30, "start.month": 9, "start.year": 2021, "end.day": 30, "end.month": 10, "end.year": 2021}}, - ] - - assert list(make_date_slices(TEST_START_DATE, TEST_END_DATE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py deleted file mode 100644 index 65036b99d06a..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from samples.test_data_for_analytics import test_input_result_record_chunks, test_output_merged_chunks -from source_linkedin_ads.analytics import merge_chunks - -TEST_MERGE_BY_KEY = "end_date" - - -def test_merge_chunks(): - """`merge_chunks` is the generator object, to get the output the list() function is applied""" - assert list(merge_chunks(test_input_result_record_chunks, TEST_MERGE_BY_KEY)) == test_output_merged_chunks diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json new file mode 100644 index 000000000000..edab65234171 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json @@ -0,0 +1,126 @@ +[ + [ + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange" + } + ], + [ + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange" + } + ] +] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json new file mode 100644 index 000000000000..a68474329ce0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json @@ -0,0 +1,71 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "documentFirstQuartileCompletions": 0, + "actionClicks": 0, + "comments": 0, + "costInUsd": "-2E-18", + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "commentLikes": 0, + "adUnitClicks": 0, + "companyPageClicks": 0, + "costInLocalCurrency": "-2E-18", + "documentThirdQuartileCompletions": 0, + "externalWebsiteConversions": 0, + "cardImpressions": 0, + "documentCompletions": 0, + "clicks": 0, + "cardClicks": 0, + "approximateUniqueImpressions": 0, + "documentMidpointCompletions": 0, + "downloadClicks": 0 + }, + { + "documentFirstQuartileCompletions": 0, + "actionClicks": 0, + "comments": 0, + "costInUsd": "100", + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "commentLikes": 0, + "adUnitClicks": 0, + "companyPageClicks": 0, + "costInLocalCurrency": "100", + "documentThirdQuartileCompletions": 0, + "externalWebsiteConversions": 0, + "cardImpressions": 0, + "documentCompletions": 0, + "clicks": 106, + "cardClicks": 0, + "approximateUniqueImpressions": 17392, + "documentMidpointCompletions": 0, + "downloadClicks": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json new file mode 100644 index 000000000000..ac6433682403 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json @@ -0,0 +1,61 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "oneClickLeads": 0, + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2021 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "landingPageClicks": 0, + "fullScreenPlays": 0, + "oneClickLeadFormOpens": 0, + "follows": 0, + "impressions": 1, + "otherEngagements": 0, + "leadGenerationMailContactInfoShares": 0, + "opens": 0, + "leadGenerationMailInterestedClicks": 0, + "pivotValues": ["urn:li:sponsoredCreative:1"], + "likes": 0 + }, + { + "oneClickLeads": 0, + "dateRange": { + "start": { + "month": 1, + "day": 1, + "year": 2021 + }, + "end": { + "month": 1, + "day": 1, + "year": 2021 + } + }, + "landingPageClicks": 106, + "fullScreenPlays": 0, + "oneClickLeadFormOpens": 0, + "follows": 0, + "impressions": 19464, + "otherEngagements": 0, + "leadGenerationMailContactInfoShares": 0, + "opens": 0, + "leadGenerationMailInterestedClicks": 0, + "pivotValues": ["urn:li:sponsoredCreative:1"], + "likes": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json new file mode 100644 index 000000000000..5e128840ccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json @@ -0,0 +1,37 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "videoCompletions": 0, + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "viralCardImpressions": 0, + "videoFirstQuartileCompletions": 0, + "textUrlClicks": 0, + "videoStarts": 0, + "sends": 0, + "shares": 0, + "videoMidpointCompletions": 0, + "validWorkEmailLeads": 0, + "viralCardClicks": 0, + "videoThirdQuartileCompletions": 0, + "totalEngagements": 105, + "reactions": 0, + "videoViews": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py new file mode 100644 index 000000000000..3936c7fad7e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json +import os +from typing import Any, Mapping + +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from source_linkedin_ads.analytics_streams import AdMemberCountryAnalytics, LinkedInAdsAnalyticsStream + +# Test input arguments for the `make_analytics_slices` +TEST_KEY_VALUE_MAP = {"camp_id": "id"} +TEST_START_DATE = "2021-08-01" +TEST_END_DATE = "2021-09-30" + +# This is the mock of the request_params +TEST_REQUEST_PRAMS = {} + + +TEST_CONFIG: dict = { + "start_date": "2021-01-01", + "end_date": "2021-02-01", + "account_ids": [1, 2], + "credentials": { + "auth_method": "access_token", + "access_token": "access_token", + "authenticator": TokenAuthenticator(token="123"), + }, +} + +# Test chunk size for each field set +TEST_FIELDS_CHUNK_SIZE = 3 +# Test fields assuming they are really available for the fetch +TEST_ANALYTICS_FIELDS = [ + "field_1", + "base_field_1", + "field_2", + "base_field_2", + "field_3", + "field_4", + "field_5", + "field_6", + "field_7", + "field_8", +] + + +# HELPERS +def load_json_file(file_name: str) -> Mapping[str, Any]: + with open(f"{os.path.dirname(__file__)}/{file_name}", "r") as data: + return json.load(data) + + +def test_analytics_stream_slices(requests_mock): + requests_mock.get("https://api.linkedin.com/rest/adAccounts", json={"elements": [{"id": 1}]}) + requests_mock.get("https://api.linkedin.com/rest/adAccounts/1/adCampaigns", json={"elements": [{"id": 123}]}) + assert list( + AdMemberCountryAnalytics(config=TEST_CONFIG).stream_slices( + sync_mode=None, + ) + ) == load_json_file("output_slices.json") + + +def test_read_records(requests_mock): + requests_mock.get( + "https://api.linkedin.com/rest/adAnalytics", + [ + {"json": load_json_file("responses/ad_member_country_analytics/response_1.json")}, + {"json": load_json_file("responses/ad_member_country_analytics/response_2.json")}, + {"json": load_json_file("responses/ad_member_country_analytics/response_3.json")}, + ], + ) + stream_slice = load_json_file("output_slices.json")[0] + records = list(AdMemberCountryAnalytics(config=TEST_CONFIG).read_records(stream_slice=stream_slice, sync_mode=None)) + assert len(records) == 2 + + +def test_chunk_analytics_fields(): + """ + We expect to truncate the field list into the chunks of equal size, + with "dateRange" field presented in each chunk. + """ + expected_output = [ + ["field_1", "base_field_1", "field_2", "dateRange"], + ["base_field_2", "field_3", "field_4", "dateRange"], + ["field_5", "field_6", "field_7", "dateRange"], + ["field_8", "dateRange"], + ] + + assert list(LinkedInAdsAnalyticsStream.chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output + + +def test_get_date_slices(): + """ + By default, we use the `WINDOW_SIZE = 30`, as it set in the analytics module + This value could be changed by setting the corresponding argument in the method. + The `end_date` is not specified by default, but for this test it was specified to have the test static. + """ + + test_start_date = "2021-08-01" + test_end_date = "2021-10-01" + + expected_output = [ + {"dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}}, + {"dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}}, + {"dateRange": {"start.day": 30, "start.month": 9, "start.year": 2021, "end.day": 30, "end.month": 10, "end.year": 2021}}, + ] + + assert list(LinkedInAdsAnalyticsStream.get_date_slices(test_start_date, test_end_date)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py similarity index 100% rename from airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py rename to airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py index 74ee9b7342f2..a6068720a6a8 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/main.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linkedin_pages import SourceLinkedinPages +from source_linkedin_pages.run import run if __name__ == "__main__": - source = SourceLinkedinPages() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml index f2c1338be532..f5d1196e7b6c 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml @@ -12,6 +12,10 @@ data: icon: linkedin.svg license: MIT name: LinkedIn Pages + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-linkedin-pages registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py index f690842da795..1b491a1f3e95 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-linkedin-pages=source_linkedin_pages.run:run", + ], + }, name="source_linkedin_pages", description="Source implementation for Linkedin Company Pages.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py new file mode 100644 index 000000000000..4f0787573f2e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + + +def run(): + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linnworks/Dockerfile b/airbyte-integrations/connectors/source-linnworks/Dockerfile deleted file mode 100644 index c8b96529f82a..000000000000 --- a/airbyte-integrations/connectors/source-linnworks/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_linnworks ./source_linnworks - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.5 -LABEL io.airbyte.name=airbyte/source-linnworks diff --git a/airbyte-integrations/connectors/source-linnworks/README.md b/airbyte-integrations/connectors/source-linnworks/README.md index b5395f00515c..d8557c27c0ce 100644 --- a/airbyte-integrations/connectors/source-linnworks/README.md +++ b/airbyte-integrations/connectors/source-linnworks/README.md @@ -6,23 +6,28 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development ### Prerequisites + **To iterate on this connector, make sure to complete this prerequisites section.** #### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies + From this connector directory, create a virtual environment: -``` + +```bash python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: -``` + +```bash source .venv/bin/activate pip install -r requirements.txt pip install '.[tests]' ``` + If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is @@ -31,6 +36,7 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu should work as you expect. #### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linnworks) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linnworks/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. @@ -40,7 +46,8 @@ See `integration_tests/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector -``` + +```bash python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json @@ -49,23 +56,82 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image +#### Use `airbyte-ci` to build your connector + +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name=source-linnworks build +airbyte-ci connectors --name source-linnworks build +``` + +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-linnworks:dev`. + +##### Customizing our build process + +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: + +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image + +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. + +```Dockerfile +FROM airbyte/source-linnworks:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -An image will be built with the tag `airbyte/source-linnworks:dev`. +Please use this as an example. This is not optimized. + +2. Build your image: -**Via `docker build`:** ```bash docker build -t airbyte/source-linnworks:dev . +# Running the spec command against your patched connector +docker run airbyte/source-linnworks:dev spec ``` #### Run + Then run any of the connector commands as follows: -``` + +```bash docker run --rm airbyte/source-linnworks:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev discover --config /secrets/config.json @@ -73,23 +139,30 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ## Testing + You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-linnworks test ``` ### Customizing acceptance Tests + Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: + * required for your connector to work need to go to `MAIN_REQUIREMENTS` list. * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linnworks test` 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. @@ -97,4 +170,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml index ce4b7ac5c2cf..c7343f0e5d84 100644 --- a/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml @@ -1,24 +1,38 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-linnworks:dev -tests: +acceptance_tests: spec: - - spec_path: "source_linnworks/spec.json" + tests: + - spec_path: "source_linnworks/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "processed_orders" + bypass_reason: "No seed data in our sandbox account" + - name: "processed_order_details" + bypass_reason: "No seed data in our sandbox account" + - name: "stock_items" + bypass_reason: "No seed data in our sandbox account" + # Removing incremental tests for now as we have no seed data in our sandbox account for the two streams that support it + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json index fc503ae2fd7f..bb51cc3fa496 100644 --- a/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json @@ -1,8 +1,16 @@ -{ - "processed_orders": { - "dProcessedOn": "2050-01-01T00:00:00+00:00" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "dProcessedOn": "2050-01-01T00:00:00+00:00" }, + "stream_descriptor": { "name": "processed_orders" } + } }, - "processed_order_details": { - "ProcessedDateTime": "2050-01-01T00:00:00+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "ProcessedDateTime": "2050-01-01T00:00:00+00:00" }, + "stream_descriptor": { "name": "processed_order_details" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-linnworks/main.py b/airbyte-integrations/connectors/source-linnworks/main.py index 7d0337b4c1b1..ee964c061ce0 100644 --- a/airbyte-integrations/connectors/source-linnworks/main.py +++ b/airbyte-integrations/connectors/source-linnworks/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linnworks import SourceLinnworks +from source_linnworks.run import run if __name__ == "__main__": - source = SourceLinnworks() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linnworks/metadata.yaml b/airbyte-integrations/connectors/source-linnworks/metadata.yaml index 326dbedaa270..b2bf2f742b85 100644 --- a/airbyte-integrations/connectors/source-linnworks/metadata.yaml +++ b/airbyte-integrations/connectors/source-linnworks/metadata.yaml @@ -1,24 +1,30 @@ data: + ab_internal: + ql: 100 + sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 dockerRepository: airbyte/source-linnworks + documentationUrl: https://docs.airbyte.com/integrations/sources/linnworks githubIssueLabel: source-linnworks icon: linnworks.svg license: MIT name: Linnworks - registries: + registries: # Removed from registries due to LEGACY STATE cloud: - enabled: true + enabled: false oss: - enabled: true + enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/linnworks + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-linnworks + supportLevel: community tags: - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linnworks/setup.py b/airbyte-integrations/connectors/source-linnworks/setup.py index f0a487191567..a2d048bbd31a 100644 --- a/airbyte-integrations/connectors/source-linnworks/setup.py +++ b/airbyte-integrations/connectors/source-linnworks/setup.py @@ -5,9 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] +MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy"] TEST_REQUIREMENTS = [ "pytest~=6.1", @@ -16,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-linnworks=source_linnworks.run:run", + ], + }, name="source_linnworks", description="Source implementation for Linnworks.", author="Labanoras Tech", author_email="jv@labanoras.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py new file mode 100644 index 000000000000..d08e84e3ae6c --- /dev/null +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linnworks import SourceLinnworks + + +def run(): + source = SourceLinnworks() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json index f7c206789d2d..24d1f172926a 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json @@ -26,7 +26,7 @@ "GeneralInfo": { "type": "object", "description": "General information about order", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Status": { "type": "integer", @@ -73,7 +73,7 @@ "description": "Order identifiers. [Prime | Scheduled]", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "IdentifierId": { "type": "integer", @@ -141,7 +141,7 @@ "ScheduledDelivery": { "type": "object", "description": "Scheduled delivery dates. Take priority over despatch by date", - "additionalProperties": false, + "additionalProperties": true, "properties": { "From": { "type": "string", @@ -179,7 +179,7 @@ "ShippingInfo": { "type": "object", "description": "Order shipping information", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Vendor": { "type": ["null", "string"], @@ -238,7 +238,7 @@ "CustomerInfo": { "type": "object", "description": "Order Customer information (Name, email etc)", - "additionalProperties": false, + "additionalProperties": true, "properties": { "ChannelBuyerName": { "type": "string", @@ -247,7 +247,7 @@ "Address": { "type": "object", "description": "Customer address", - "additionalProperties": false, + "additionalProperties": true, "properties": { "EmailAddress": { "type": "string", @@ -305,7 +305,7 @@ "BillingAddress": { "type": "object", "description": "Customer billing address", - "additionalProperties": false, + "additionalProperties": true, "properties": { "EmailAddress": { "type": "string", @@ -365,7 +365,7 @@ "TotalsInfo": { "type": "object", "description": "Order totals information", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOrderId": { "type": "string", @@ -426,7 +426,7 @@ "description": "Extended properties of an order", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "RowId": { "type": "string", @@ -459,7 +459,7 @@ "description": "List of order items", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "ItemId": { "type": "string", @@ -507,7 +507,7 @@ "OnPurchaseOrder": { "type": "object", "description": "Purchase order bound to this item", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkPurchaseItemId": { "type": "string", @@ -648,7 +648,7 @@ "description": "List of order item options", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOptionId": { "type": "string", @@ -698,7 +698,7 @@ "description": "List of BinRacks used for OrderItem", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Quantity": { "type": "integer", @@ -747,7 +747,7 @@ "type": "array", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "BoxId": { "type": "integer", @@ -818,7 +818,7 @@ "description": "List of order notes", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "OrderNoteId": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json index c2a7addf04b5..7f0c798b7ab1 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOrderID": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json index 115c4ccdc98e..544b8ab7312a 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Suppliers": { "type": "array", "description": "Suppliers", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "IsDefault": { "type": "boolean", @@ -82,11 +82,11 @@ "description": "Stock Levels", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Location": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "description": "Location ID", "properties": { "StockLocationId": { @@ -200,7 +200,7 @@ "description": "List of item descriptions", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -234,7 +234,7 @@ "description": "List of extended properties", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -264,7 +264,7 @@ "description": "List item titles", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -298,14 +298,14 @@ "description": "List of item prices", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Rules": { "type": "array", "description": "Pricing rule", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": ["null", "integer"], @@ -369,7 +369,7 @@ "description": "Image urls", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Source": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json index 6eb3c4976ba0..4a146974d3ab 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json @@ -1,6 +1,6 @@ { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Address1": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json index dfdde21d0c11..01671d3518d2 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "StockLocationId": { "type": "string", @@ -30,6 +30,33 @@ "IsWarehouseManaged": { "type": ["null", "boolean"], "description": "If the location is warehouse managed." + }, + "Address1": { + "type": ["null", "string"] + }, + "Address2": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "County": { + "type": ["null", "string"] + }, + "Country": { + "type": ["null", "string"] + }, + "ZipCode": { + "type": ["null", "string"] + }, + "CountInOrderUntilAcknowledgement": { + "type": ["null", "boolean"] + }, + "FulfilmentCenterDeductStockWhenProcessed": { + "type": ["null", "boolean"] + }, + "IsNotTrackable": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 733ca9cb314b..7192e2ec5826 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -17,11 +17,11 @@ class LinnworksAuthenticator(Oauth2Authenticator): def __init__( self, - token_refresh_endpoint: str, application_id: str, application_secret: str, token: str, token_expiry_date: pendulum.datetime = None, + token_refresh_endpoint: str = "https://api.linnworks.net/api/Auth/AuthorizeByApplication", access_token_name: str = "Token", expires_in_name: str = "TTL", server_name: str = "Server", @@ -36,16 +36,19 @@ def __init__( access_token_name=access_token_name, expires_in_name=expires_in_name, ) - + self.access_token_name = access_token_name self.application_id = application_id self.application_secret = application_secret + self.expires_in_name = expires_in_name self.token = token self.server_name = server_name + self.token_refresh_endpoint = token_refresh_endpoint def get_auth_header(self) -> Mapping[str, Any]: return {"Authorization": self.get_access_token()} def get_access_token(self): + if self.token_has_expired(): t0 = pendulum.now() token, expires_in, server = self.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json index 8e99bdcbc648..4ed0f2ee5d80 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json @@ -5,7 +5,7 @@ "title": "Linnworks Spec", "type": "object", "required": ["application_id", "application_secret", "token", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "application_id": { "title": "Application ID.", @@ -20,7 +20,8 @@ }, "token": { "title": "API Token", - "type": "string" + "type": "string", + "airbyte_secret": true }, "start_date": { "title": "Start Date", diff --git a/airbyte-integrations/connectors/source-lokalise/main.py b/airbyte-integrations/connectors/source-lokalise/main.py index 4d4423bbe36a..d02dfd231989 100644 --- a/airbyte-integrations/connectors/source-lokalise/main.py +++ b/airbyte-integrations/connectors/source-lokalise/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lokalise import SourceLokalise +from source_lokalise.run import run if __name__ == "__main__": - source = SourceLokalise() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lokalise/metadata.yaml b/airbyte-integrations/connectors/source-lokalise/metadata.yaml index 079f05bcb4f6..2f222da3c397 100644 --- a/airbyte-integrations/connectors/source-lokalise/metadata.yaml +++ b/airbyte-integrations/connectors/source-lokalise/metadata.yaml @@ -8,6 +8,10 @@ data: icon: lokalise.svg license: MIT name: Lokalise + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lokalise registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-lokalise/setup.py b/airbyte-integrations/connectors/source-lokalise/setup.py index 69d13c0b79f0..b2a8b5b467ec 100644 --- a/airbyte-integrations/connectors/source-lokalise/setup.py +++ b/airbyte-integrations/connectors/source-lokalise/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lokalise=source_lokalise.run:run", + ], + }, name="source_lokalise", description="Source implementation for Lokalise.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py b/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py new file mode 100644 index 000000000000..21002eb93fee --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lokalise import SourceLokalise + + +def run(): + source = SourceLokalise() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-looker/main.py b/airbyte-integrations/connectors/source-looker/main.py index dee0fd4beb4e..b6164cb0322b 100644 --- a/airbyte-integrations/connectors/source-looker/main.py +++ b/airbyte-integrations/connectors/source-looker/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_looker import SourceLooker +from source_looker.run import run if __name__ == "__main__": - source = SourceLooker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-looker/metadata.yaml b/airbyte-integrations/connectors/source-looker/metadata.yaml index 10e3a4cfe079..3d2415c0cd29 100644 --- a/airbyte-integrations/connectors/source-looker/metadata.yaml +++ b/airbyte-integrations/connectors/source-looker/metadata.yaml @@ -8,6 +8,10 @@ data: icon: looker.svg license: MIT name: Looker + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-looker registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index f38fc2a1095e..311a876d7fd9 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -20,13 +20,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-looker=source_looker.run:run", + ], + }, name="source_looker", description="Source implementation for Looker.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-looker/source_looker/run.py b/airbyte-integrations/connectors/source-looker/source_looker/run.py new file mode 100644 index 000000000000..53aaad83ea7f --- /dev/null +++ b/airbyte-integrations/connectors/source-looker/source_looker/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_looker import SourceLooker + + +def run(): + source = SourceLooker() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailchimp/README.md b/airbyte-integrations/connectors/source-mailchimp/README.md index 7f8ca7aeaa31..2e6d772187b2 100644 --- a/airbyte-integrations/connectors/source-mailchimp/README.md +++ b/airbyte-integrations/connectors/source-mailchimp/README.md @@ -1,99 +1,91 @@ -# Mailchimp Source +# Mailchimp source connector + This is the repository for the Mailchimp source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailchimp). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/mailchimp). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailchimp) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailchimp/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/mailchimp) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailchimp/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailchimp test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-mailchimp spec +poetry run source-mailchimp check --config secrets/config.json +poetry run source-mailchimp discover --config secrets/config.json +poetry run source-mailchimp read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-mailchimp build ``` -An image will be built with the tag `airbyte/source-mailchimp:dev`. +An image will be available on your host with the tag `airbyte/source-mailchimp:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-mailchimp:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-mailchimp:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailchimp:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailchimp:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-mailchimp:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailchimp:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-mailchimp test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailchimp test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailchimp.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/mailchimp.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl index 9353cdc7b744..f9a4f79caa5f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl @@ -1,12 +1,17 @@ -{"stream": "campaigns", "data": {"id": "7847cdaeff", "web_id": 13701472, "type": "regular", "create_time": "2023-11-06T20:12:07+00:00", "archive_url": "http://eepurl.com/iDnTtY", "long_archive_url": "https://mailchi.mp/5e0065d29854/invitation-to-unsubscribe", "status": "sent", "emails_sent": 1, "send_time": "2023-11-06T20:17:44+00:00", "content_type": "multichannel", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": "

      Contacts that match any of the following conditions:

      1. Tags contact is tagged Unsubscriber
      For a total of 1 emails sent.", "recipient_count": 1, "segment_opts": {"saved_segment_id": 14351532, "match": "any", "conditions": [{"condition_type": "StaticSegment", "field": "static_segment", "op": "static_is", "value": 14351532}]}}, "settings": {"subject_line": "Invitation to Unsubscribe", "title": "Invitation to unsubscribe", "from_name": "yurii", "reply_to": "integration-test+yurii@airbyte.io", "use_conversation": false, "to_name": "", "folder_id": "", "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 13, "drag_and_drop": false}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": "", "clicktale": ""}, "report_summary": {"opens": 2, "unique_opens": 1, "open_rate": 1, "clicks": 0, "subscriber_clicks": 0, "click_rate": 0, "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0}}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/campaigns", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Campaigns/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "DELETE"}, {"rel": "send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/send", "method": "POST"}, {"rel": "cancel_send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/cancel-send", "method": "POST"}, {"rel": "feedback", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/feedback", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Feedback/CollectionResponse.json"}, {"rel": "content", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/content", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Content/Response.json"}, {"rel": "send_checklist", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/send-checklist", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Checklist/Response.json"}, {"rel": "pause", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/pause", "method": "POST"}, {"rel": "resume", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/resume", "method": "POST"}, {"rel": "replicate", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/replicate", "method": "POST"}, {"rel": "create_resend", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/create-resend", "method": "POST"}]}, "emitted_at": 1699461462352} -{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/Response.json"}, {"rel": "member", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/Response.json"}], "action": "open", "timestamp": "2023-11-06T20:17:57+00:00", "ip": "74.125.212.231"}, "emitted_at": 1699457307719} +{"stream": "campaigns", "data": {"id": "324b8a398e", "web_id": 13531140, "type": "regular", "create_time": "2022-12-27T08:12:59+00:00", "archive_url": "http://eepurl.com/ig7RxP", "long_archive_url": "https://us10.campaign-archive.com/?u=caf9055242d41edd9215d1898&id=324b8a398e", "status": "save", "emails_sent": 0, "send_time": null, "content_type": "multichannel", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": null, "recipient_count": 47}, "settings": {"title": "Untitled", "use_conversation": false, "to_name": null, "folder_id": null, "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 13, "drag_and_drop": false}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": null, "clicktale": null}, "social_card": {"image_url": "https://cdn-images.mailchimp.com/monkey_rewards/grow-business-banner-2.png", "description": null, "title": null}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638872240} +{"stream": "campaigns", "data": {"id": "3cbed9a0fc", "web_id": 13531144, "type": "regular", "create_time": "2022-12-27T08:21:01+00:00", "archive_url": "http://eepurl.com/ig7SKH", "long_archive_url": "https://us10.campaign-archive.com/?u=caf9055242d41edd9215d1898&id=3cbed9a0fc", "status": "save", "emails_sent": 0, "send_time": null, "content_type": "template", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": null, "recipient_count": 47}, "settings": {"title": "Untitled", "use_conversation": false, "to_name": null, "folder_id": null, "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 145, "drag_and_drop": true}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": null, "clicktale": null}, "social_card": {"image_url": "https://cdn-images.mailchimp.com/monkey_rewards/grow-business-banner-2.png", "description": null, "title": null}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638872242} +{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "action": "open", "timestamp": "2023-11-06T20:17:57+00:00", "ip": "74.125.212.231"}, "emitted_at": 1701638876052} +{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "action": "open", "timestamp": "2023-11-07T19:49:10+00:00", "ip": "74.125.215.162"}, "emitted_at": 1701638876053} {"stream": "interests", "data": {"category_id": "a194ba131d", "list_id": "16d6ec4ffc", "id": "bbbb369575", "name": "Donating", "subscriber_count": "0", "display_order": 1}, "emitted_at": 1699963797987} {"stream": "interest_categories", "data": {"list_id": "16d6ec4ffc", "id": "1bcbe8ba9b", "title": "Product Preferences", "display_order": 0, "type": "checkboxes"}, "emitted_at": 1699963796751} -{"stream": "list_members", "data": {"id": "458f50b08c829a8ab901d3f8f88df914", "email_address": "integration-test+Thomas@airbyte.io", "unique_email_id": "42d6d67d11", "contact_id": "475a8f7f7b5087d7be924c9b331c8316", "full_name": "Thomas", "web_id": 546044608, "email_type": "html", "status": "unsubscribed", "unsubscribe_reason": "N/A (Unsubscribed by admin)", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "Thomas", "LNAME": "", "ADDRESS": "", "PHONE": "", "BIRTHDAY": ""}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": "", "timestamp_signup": "", "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2023-11-03T20:53:12+00:00", "language": "", "vip": false, "email_client": "", "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": "", "timezone": "", "region": ""}, "source": "Import", "tags_count": 0, "tags": [], "list_id": "16d6ec4ffc"}, "emitted_at": 1699302001460} -{"stream": "lists", "data": {"id": "16d6ec4ffc", "web_id": 903380, "name": "Airbyte", "contact": {"company": "Airbyte", "address1": "kyiv", "address2": "", "city": "Kiev", "state": "30", "zip": "04200", "country": "UA", "phone": ""}, "permission_reminder": "You are receiving this email because you opted in via our website.", "use_archive_bar": true, "campaign_defaults": {"from_name": "yurii", "from_email": "integration-test+yurii@airbyte.io", "subject": "", "language": "en"}, "notify_on_subscribe": "", "notify_on_unsubscribe": "", "date_created": "2022-12-27T07:56:47+00:00", "list_rating": 0, "email_type_option": false, "subscribe_url_short": "http://eepurl.com/ihg3RD", "subscribe_url_long": "https://airbyte.us10.list-manage.com/subscribe?u=caf9055242d41edd9215d1898&id=16d6ec4ffc", "beamer_address": "us10-d527bd96ba-6d1a9988db@inbound.mailchimp.com", "visibility": "prv", "double_optin": false, "has_welcome": false, "marketing_permissions": false, "modules": [], "stats": {"member_count": 47, "unsubscribe_count": 4, "cleaned_count": 0, "member_count_since_send": 0, "unsubscribe_count_since_send": 1, "cleaned_count_since_send": 0, "campaign_count": 6, "campaign_last_sent": "2022-12-27T08:37:53+00:00", "merge_field_count": 5, "avg_sub_rate": 0, "avg_unsub_rate": 1, "target_sub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117, "last_sub_date": "2022-12-27T08:34:39+00:00", "last_unsub_date": "2023-11-06T20:18:01+00:00"}, "_links": [{"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json"}, {"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/lists", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Collection.json"}, {"rel": "update", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "PATCH", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/PATCH.json"}, {"rel": "batch-sub-unsub-members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "POST", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST-Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "DELETE"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Abuse/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Abuse/Collection.json"}, {"rel": "activity", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Activity/Response.json"}, {"rel": "clients", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/clients", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Clients/Response.json"}, {"rel": "growth-history", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/growth-history", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Growth/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Growth/Collection.json"}, {"rel": "interest-categories", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/interest-categories", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/InterestCategories/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/InterestCategories/Collection.json"}, {"rel": "members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Members/Collection.json"}, {"rel": "merge-fields", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/merge-fields", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/MergeFields/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/MergeFields/Collection.json"}, {"rel": "segments", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/segments", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Segments/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Segments/Collection.json"}, {"rel": "webhooks", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/webhooks", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Webhooks/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Webhooks/Collection.json"}, {"rel": "signup-forms", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/signup-forms", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/SignupForms/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/SignupForms/Collection.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Locations/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Locations/Collection.json"}]}, "emitted_at": 1699626450570} -{"stream": "reports", "data": {"id": "a79651273b", "campaign_title": "Untitled", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Airbyte Test", "preview_text": "", "emails_sent": 50, "abuse_reports": 0, "unsubscribed": 0, "send_time": "2022-12-27T08:36:55+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 412, "unique_opens": 50, "open_rate": 1, "last_open": "2023-01-09T10:07:54+00:00"}, "clicks": {"clicks_total": 48, "unique_clicks": 47, "unique_subscriber_clicks": 33, "click_rate": 0.66, "last_click": "2022-12-27T15:28:11+00:00"}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2022-12-27T08:00:00+00:00", "emails_sent": 50, "unique_opens": 6, "recipients_clicks": 1}, {"timestamp": "2022-12-27T09:00:00+00:00", "emails_sent": 0, "unique_opens": 43, "recipients_clicks": 0}, {"timestamp": "2022-12-27T10:00:00+00:00", "emails_sent": 0, "unique_opens": 1, "recipients_clicks": 3}, {"timestamp": "2022-12-27T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 11}, {"timestamp": "2022-12-27T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 10}, {"timestamp": "2022-12-27T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 2}, {"timestamp": "2022-12-27T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T20:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Reports/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Response.json"}, {"rel": "campaign", "href": "https://us10.api.mailchimp.com/3.0/campaigns/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "sub-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sub-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Sub/Response.json"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Abuse/CollectionResponse.json"}, {"rel": "advice", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/advice", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Advice/Response.json"}, {"rel": "open-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/open-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/OpenDetails/CollectionResponse.json"}, {"rel": "click-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/click-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/ClickDetails/CollectionResponse.json"}, {"rel": "domain-performance", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/domain-performance", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/DomainPerformance/Response.json"}, {"rel": "eepurl", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/eepurl", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Eepurl/CollectionResponse.json"}, {"rel": "email-activity", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Locations/Response.json"}, {"rel": "sent-to", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sent-to", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/SentTo/CollectionResponse.json"}, {"rel": "unsubscribed", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/unsubscribed", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Unsubs/CollectionResponse.json"}]}, "emitted_at": 1699627079113} -{"stream": "segment_members", "data": {"id": "b79e1a05afb84190ec55310c5ee3f27e", "email_address": "integration-test+Lori@airbyte.io", "unique_email_id": "44f96f7b6a", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Lori", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 14351128}, "emitted_at": 1700762624968} -{"stream": "segment_members", "data": {"id": "0119f706aa9a1ec9757d852743033a2d", "email_address": "integration-test+Susan@airbyte.io", "unique_email_id": "b30576500a", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Susan", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 14351128}, "emitted_at": 1700762624967} -{"stream": "segments", "data": {"id": 13506132, "name": "Influencer", "member_count": 3, "type": "static", "created_at": "2022-12-27T08:33:35+00:00", "updated_at": "2022-12-27T08:33:35+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1699302003309} +{"stream": "list_members", "data": {"id": "87ed95f658a2efab665957871270de69", "email_address": "integration-test+yurii@airbyte.io", "unique_email_id": "38619c0ffd", "contact_id": "6ad2f9e1b25a09421fd8df87662b1634", "full_name": "yurii cherniaiev", "web_id": 546044412, "email_type": "html", "status": "subscribed", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "yurii", "LNAME": "cherniaiev", "ADDRESS": {"addr1": "Airbyte\nkyiv\nKiev 04200\nUkraine", "addr2": null, "city": null, "state": null, "zip": null, "country": "US"}, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T07:56:47+00:00", "member_rating": 2, "last_changed": "2022-12-27T07:56:47+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null, "region": null}, "source": "Admin Add", "tags_count": 1, "tags": [{"id": 14351504, "name": "Overlord"}], "list_id": "16d6ec4ffc"}, "emitted_at": 1701638878091} +{"stream": "list_members", "data": {"id": "65a02406e7dc3b786af6b94489721e46", "email_address": "integration-test+Jesse@airbyte.io", "unique_email_id": "475570cf37", "contact_id": "6639385c4bdffe110d88044782e37c12", "full_name": "Jesse", "web_id": 546044440, "email_type": "html", "status": "subscribed", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "Jesse", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:38+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:38+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null, "region": null}, "source": "Import", "tags_count": 0, "tags": [], "list_id": "16d6ec4ffc"}, "emitted_at": 1701638878092} +{"stream": "lists", "data": {"id": "16d6ec4ffc", "web_id": 903380, "name": "Airbyte", "contact": {"company": "Airbyte", "address1": "kyiv", "address2": null, "city": "Kiev", "state": "30", "zip": "04200", "country": "UA", "phone": null}, "permission_reminder": "You are receiving this email because you opted in via our website.", "use_archive_bar": true, "campaign_defaults": {"from_name": "yurii", "from_email": "integration-test+yurii@airbyte.io", "subject": null, "language": "en"}, "notify_on_subscribe": null, "notify_on_unsubscribe": null, "date_created": "2022-12-27T07:56:47+00:00", "list_rating": 0, "email_type_option": false, "subscribe_url_short": "http://eepurl.com/ihg3RD", "subscribe_url_long": "https://airbyte.us10.list-manage.com/subscribe?u=caf9055242d41edd9215d1898&id=16d6ec4ffc", "beamer_address": "us10-d527bd96ba-6d1a9988db@inbound.mailchimp.com", "visibility": "prv", "double_optin": false, "has_welcome": false, "marketing_permissions": false, "modules": [], "stats": {"member_count": 47, "unsubscribe_count": 4, "cleaned_count": 0, "member_count_since_send": 0, "unsubscribe_count_since_send": 1, "cleaned_count_since_send": 0, "campaign_count": 6, "campaign_last_sent": "2022-12-27T08:37:53+00:00", "merge_field_count": 5, "avg_sub_rate": 0, "avg_unsub_rate": 0, "target_sub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117, "last_sub_date": "2022-12-27T08:34:39+00:00", "last_unsub_date": "2023-11-06T20:18:01+00:00"}}, "emitted_at": 1701638875717} +{"stream": "reports", "data": {"id": "a79651273b", "campaign_title": "Untitled", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Airbyte Test", "preview_text": null, "emails_sent": 50, "abuse_reports": 0, "unsubscribed": 0, "send_time": "2022-12-27T08:36:55+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 412, "unique_opens": 50, "open_rate": 1, "last_open": "2023-01-09T10:07:54+00:00"}, "clicks": {"clicks_total": 48, "unique_clicks": 47, "unique_subscriber_clicks": 33, "click_rate": 0.66, "last_click": "2022-12-27T15:28:11+00:00"}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2022-12-27T08:00:00+00:00", "emails_sent": 50, "unique_opens": 6, "recipients_clicks": 1}, {"timestamp": "2022-12-27T09:00:00+00:00", "emails_sent": 0, "unique_opens": 43, "recipients_clicks": 0}, {"timestamp": "2022-12-27T10:00:00+00:00", "emails_sent": 0, "unique_opens": 1, "recipients_clicks": 3}, {"timestamp": "2022-12-27T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 11}, {"timestamp": "2022-12-27T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 10}, {"timestamp": "2022-12-27T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 2}, {"timestamp": "2022-12-27T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T20:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638878519} +{"stream": "reports", "data": {"id": "7847cdaeff", "campaign_title": "Invitation to unsubscribe", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Invitation to Unsubscribe", "preview_text": null, "emails_sent": 1, "abuse_reports": 0, "unsubscribed": 1, "send_time": "2023-11-06T20:17:44+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 2, "unique_opens": 1, "open_rate": 1, "last_open": "2023-11-07T19:49:10+00:00"}, "clicks": {"clicks_total": 0, "unique_clicks": 0, "unique_subscriber_clicks": 0, "click_rate": 0, "last_click": null}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2023-11-06T20:00:00+00:00", "emails_sent": 1, "unique_opens": 1, "recipients_clicks": 0}, {"timestamp": "2023-11-06T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-06T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-06T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T08:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T09:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T10:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638878520} +{"stream": "segment_members", "data": {"id": "1dd067951f91190b65b43305b9166bc7", "email_address": "integration-test+Michael@airbyte.io", "unique_email_id": "904643439a", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Michael", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 13506120}, "emitted_at": 1701638879995} +{"stream": "segment_members", "data": {"id": "802cb9cc84d031ca07cbf9efa3dcdc2c", "email_address": "integration-test+Carlos@airbyte.io", "unique_email_id": "41ec088075", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Carlos", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 13506120}, "emitted_at": 1701638879996} +{"stream": "segments", "data": {"id": 13506120, "name": "Customer", "member_count": 2, "type": "static", "created_at": "2022-12-27T08:12:06+00:00", "updated_at": "2022-12-27T08:12:55+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1701638883128} +{"stream": "segments", "data": {"id": 13506124, "name": "Member", "member_count": 0, "type": "static", "created_at": "2022-12-27T08:12:06+00:00", "updated_at": "2022-12-27T08:28:44+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1701638883129} {"stream": "tags", "data": {"id": 13506128, "name": "2022", "list_id": "16d6ec4ffc"}, "emitted_at": 1699963804499} -{"stream": "unsubscribes", "data": {"email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "merge_fields": {"FNAME": "Joe", "LNAME": "Barry", "ADDRESS": {"addr1": "109 Barry St", "addr2": "", "city": "Gary", "state": "IN", "zip": "46401", "country": "US"}, "PHONE": "", "BIRTHDAY": ""}, "vip": false, "timestamp": "2023-11-06T20:18:01+00:00", "reason": "Did not signup for list", "campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true}, "emitted_at": 1699302005437} +{"stream": "unsubscribes", "data": {"email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "merge_fields": {"FNAME": "Joe", "LNAME": "Barry", "ADDRESS": {"addr1": "109 Barry St", "addr2": null, "city": "Gary", "state": "IN", "zip": "46401", "country": "US"}, "PHONE": null, "BIRTHDAY": null}, "vip": false, "timestamp": "2023-11-06T20:18:01+00:00", "reason": "Did not signup for list", "campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true}, "emitted_at": 1701638884243} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/segments.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/segments.json deleted file mode 100644 index ac6a59cb5e70..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/segments.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "segment_members", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["last_changed"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "cursor_field": ["last_changed"], - "primary_key": [["id"]], - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json index 3f87ae1fcb30..26b656926fd5 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json @@ -45,15 +45,6 @@ "stream_descriptor": { "name": "reports" } } }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "16d6ec4ffc": { "last_changed": "2230-02-26T05:42:10+00:00" } - }, - "stream_descriptor": { "name": "segment_members" } - } - }, { "type": "STREAM", "stream": { diff --git a/airbyte-integrations/connectors/source-mailchimp/main.py b/airbyte-integrations/connectors/source-mailchimp/main.py index b95b566e6b8a..c61875fb7a72 100644 --- a/airbyte-integrations/connectors/source-mailchimp/main.py +++ b/airbyte-integrations/connectors/source-mailchimp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailchimp import SourceMailchimp +from source_mailchimp.run import run if __name__ == "__main__": - source = SourceMailchimp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 349c6426a418..17b3023af310 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -10,19 +10,37 @@ data: connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.10.0 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp icon: mailchimp.svg license: MIT name: Mailchimp + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailchimp registries: cloud: enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + message: + Version 1.0.0 introduces schema changes to all incremental streams. + A full schema refresh and data reset are required to upgrade to this version. + For more details, see our migration guide. + upgradeDeadline: "2024-01-10" releaseStage: generally_available + suggestedStreams: + streams: + - email_activity + - campaigns + - lists + - reports supportLevel: certified tags: - language:python diff --git a/airbyte-integrations/connectors/source-mailchimp/poetry.lock b/airbyte-integrations/connectors/source-mailchimp/poetry.lock new file mode 100644 index 000000000000..a247c0d1074e --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/poetry.lock @@ -0,0 +1,1052 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.19.0" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.19.0-py3-none-any.whl", hash = "sha256:53354b5de163aa2074312c71d8ebccb8bd1ab336cff7053abb75e84dc5637abe"}, + {file = "responses-0.19.0.tar.gz", hash = "sha256:3fc29c3117e14136b833a0a6d4e7f1217c6301bf08b6086db468e12f1e3290e2"}, +] + +[package.dependencies] +requests = ">=2.0,<3.0" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-localserver", "types-mock", "types-requests"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "77ffe28d193ca607c7abf9ea05aee9498d1a9e17ee0f93adea80bf7537e58e5d" diff --git a/airbyte-integrations/connectors/source-mailchimp/pyproject.toml b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml new file mode 100644 index 000000000000..fa7b587b6008 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.1.2" +name = "source-mailchimp" +description = "Source implementation for Mailchimp." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/mailchimp" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_mailchimp" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.8" +pytest = "==6.2.5" + +[tool.poetry.scripts] +source-mailchimp = "source_mailchimp.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.6.1" +responses = "^0.19.0" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-mailchimp/requirements.txt b/airbyte-integrations/connectors/source-mailchimp/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py deleted file mode 100644 index f2973669a61e..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "responses~=0.19.0", "requests-mock~=1.9.3"] - - -setup( - name="source_mailchimp", - description="Source implementation for Mailchimp.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=[ - "airbyte-cdk", - "pytest~=6.1", - ], - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={"tests": TEST_REQUIREMENTS}, -) diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py new file mode 100644 index 000000000000..15226fdfeebd --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailchimp import SourceMailchimp + + +def run(): + source = SourceMailchimp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json index 234d6e6965a5..27e691cf22d9 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json @@ -8,10 +8,14 @@ "type": ["null", "string"] }, "create_time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte-type": "timestamp_with_timezone" }, "start_time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "status": { "type": ["null", "string"] @@ -42,7 +46,10 @@ }, "conditions": { "type": ["null", "array"], - "items": {} + "items": { + "type": ["null", "object"], + "additionalProperties": true + } } } }, @@ -180,29 +187,6 @@ "type": ["null", "number"] } } - }, - "_links": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "rel": { - "type": ["null", "string"] - }, - "href": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "targetSchema": { - "type": ["null", "string"] - }, - "schema": { - "type": ["null", "string"] - } - } - } } } } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json index 453e53807a22..8d058b78e9e2 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json @@ -16,7 +16,7 @@ "readOnly": true }, "parent_campaign_id": { - "type": "string", + "type": ["null", "string"], "title": "Parent Campaign ID", "description": "If this campaign is the child of another campaign, this identifies the parent campaign. For Example, for RSS or Automation children.", "readOnly": true @@ -28,16 +28,18 @@ "type": "string", "title": "Create Time", "description": "The date and time the campaign was created in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "archive_url": { - "type": "string", + "type": ["null", "string"], "title": "Archive URL", "description": "The link to the campaign's archive version in ISO 8601 format.", "readOnly": true }, "long_archive_url": { - "type": "string", + "type": ["null", "string"], "title": "Long Archive URL", "description": "The original link to the campaign's archive version.", "readOnly": true @@ -52,13 +54,15 @@ "readOnly": true }, "send_time": { - "type": "string", + "type": ["null", "string"], "title": "Send Time", "description": "The date and time a campaign was sent.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "content_type": { - "type": "string", + "type": ["null", "string"], "title": "Content Type", "description": "How the campaign's content is put together.", "enum": ["template", "html", "url", "multichannel"] @@ -81,7 +85,7 @@ "description": "List settings for the campaign.", "properties": { "list_id": { - "type": "string", + "type": ["null", "string"], "title": "List ID", "description": "The unique list id." }, @@ -92,13 +96,13 @@ "readOnly": true }, "list_name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list.", "readOnly": true }, "segment_text": { - "type": "string", + "type": ["null", "string"], "title": "Segment Text", "description": "A description of the [segment](https://mailchimp.com/help/create-and-send-to-a-segment/) used for the campaign. Formatted as a string marked up with HTML.", "readOnly": true @@ -120,27 +124,27 @@ "description": "The settings for your campaign, including subject, from name, reply-to address, and more.", "properties": { "subject_line": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Subject Line", "description": "The subject line for the campaign." }, "preview_text": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Preview Text", "description": "The preview text for the campaign." }, "title": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Title", "description": "The title of the campaign." }, "from_name": { - "type": "string", + "type": ["null", "string"], "title": "From Name", "description": "The 'from' name on the campaign (not an email address)." }, "reply_to": { - "type": "string", + "type": ["null", "string"], "title": "Reply To Address", "description": "The reply-to email address for the campaign." }, @@ -150,12 +154,12 @@ "description": "Use Mailchimp Conversation feature to manage out-of-office replies." }, "to_name": { - "type": "string", + "type": ["null", "string"], "title": "To Name", "description": "The campaign's custom 'To' name. Typically the first name [merge field](https://mailchimp.com/help/getting-started-with-merge-tags/)." }, "folder_id": { - "type": "string", + "type": ["null", "string"], "title": "Folder ID", "description": "If the campaign is listed in a folder, the id for that folder." }, @@ -184,7 +188,7 @@ "title": "Auto Post to Facebook", "description": "An array of [Facebook](https://mailchimp.com/help/connect-or-disconnect-the-facebook-integration/) page ids to auto-post to.", "items": { - "type": "string" + "type": ["null", "string"] } }, "fb_comments": { @@ -218,19 +222,19 @@ "description": "The settings specific to A/B test campaigns.", "properties": { "winning_combination_id": { - "type": "string", + "type": ["null", "string"], "title": "Winning Combination ID", "description": "ID for the winning combination.", "readOnly": true }, "winning_campaign_id": { - "type": "string", + "type": ["null", "string"], "title": "Winning Campaign ID", "description": "ID of the campaign that was sent to the remaining recipients based on the winning combination.", "readOnly": true }, "winner_criteria": { - "type": "string", + "type": ["null", "string"], "title": "Winning Criteria", "description": "The combination that performs the best. This may be determined automatically by click rate, open rate, or total revenue -- or you may choose manually based on the reporting data you find the most valuable. For Multivariate Campaigns testing send_time, winner_criteria is ignored. For Multivariate Campaigns with 'manual' as the winner_criteria, the winner must be chosen in the Mailchimp web application.", "enum": ["opens", "clicks", "manual", "total_revenue"] @@ -250,7 +254,7 @@ "title": "Subject Lines", "description": "The possible subject lines to test. If no subject lines are provided, settings.subject_line will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "send_times": { @@ -258,7 +262,9 @@ "title": "Send Times", "description": "The possible send times to test. The times provided should be in the format YYYY-MM-DD HH:MM:SS. If send_times are provided to test, the test_size will be set to 100% and winner_criteria will be ignored.", "items": { - "type": "string" + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } }, "from_names": { @@ -266,7 +272,7 @@ "title": "From Names", "description": "The possible from names. The number of from_names provided must match the number of reply_to_addresses. If no from_names are provided, settings.from_name will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "reply_to_addresses": { @@ -274,7 +280,7 @@ "title": "Reply To Addresses", "description": "The possible reply-to addresses. The number of reply_to_addresses provided must match the number of from_names. If no reply_to_addresses are provided, settings.reply_to will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "contents": { @@ -282,7 +288,7 @@ "title": "Content Descriptions", "description": "Descriptions of possible email contents. To set campaign contents, make a PUT request to /campaigns/{campaign_id}/content with the field 'variate_contents'.", "items": { - "type": "string" + "type": ["null", "string"] }, "readOnly": true }, @@ -295,7 +301,7 @@ "type": "object", "properties": { "id": { - "type": "string", + "type": ["null", "string"], "title": "ID", "description": "Unique ID for the combination." }, @@ -365,12 +371,12 @@ "description": "Whether to enable [eCommerce360](https://mailchimp.com/help/connect-your-online-store-to-mailchimp/) tracking." }, "google_analytics": { - "type": "string", + "type": ["null", "string"], "title": "Google Analytics Tracking", "description": "The custom slug for [Google Analytics](https://mailchimp.com/help/integrate-google-analytics-with-mailchimp/) tracking (max of 50 bytes)." }, "clicktale": { - "type": "string", + "type": ["null", "string"], "title": "ClickTale Analytics Tracking", "description": "The custom slug for [ClickTale](https://mailchimp.com/help/additional-tracking-options-for-campaigns/) tracking (max of 50 bytes)." }, @@ -411,13 +417,13 @@ "description": "[RSS](https://mailchimp.com/help/share-your-blog-posts-with-mailchimp/) options for a campaign.", "properties": { "feed_url": { - "type": "string", + "type": ["null", "string"], "title": "Feed URL", "format": "uri", "description": "The URL for the RSS feed." }, "frequency": { - "type": "string", + "type": ["null", "string"], "title": "Frequency", "description": "The frequency of the RSS Campaign.", "enum": ["daily", "weekly", "monthly"] @@ -477,7 +483,7 @@ } }, "weekly_send_day": { - "type": "string", + "type": ["null", "string"], "enum": [ "sunday", "monday", @@ -500,10 +506,12 @@ } }, "last_sent": { - "type": "string", + "type": ["null", "string"], "title": "Last Sent", "description": "The date the campaign was last sent.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "constrain_rss_img": { "type": "boolean", @@ -519,19 +527,19 @@ "readOnly": true, "properties": { "split_test": { - "type": "string", + "type": ["null", "string"], "title": "Split Test", "description": "The type of AB split to run.", "enum": ["subject", "from_name", "schedule"] }, "pick_winner": { - "type": "string", + "type": ["null", "string"], "title": "Pick Winner", "description": "How we should evaluate a winner. Based on 'opens', 'clicks', or 'manual'.", "enum": ["opens", "clicks", "manual"] }, "wait_units": { - "type": "string", + "type": ["null", "string"], "title": "Wait Time", "description": "How unit of time for measuring the winner ('hours' or 'days'). This cannot be changed after a campaign is sent.", "enum": ["hours", "days"] @@ -549,47 +557,51 @@ "description": "The size of the split groups. Campaigns split based on 'schedule' are forced to have a 50/50 split. Valid split integers are between 1-50." }, "from_name_a": { - "type": "string", + "type": ["null", "string"], "title": "From Name Group A", "description": "For campaigns split on 'From Name', the name for Group A." }, "from_name_b": { - "type": "string", + "type": ["null", "string"], "title": "From Name Group B", "description": "For campaigns split on 'From Name', the name for Group B." }, "reply_email_a": { - "type": "string", + "type": ["null", "string"], "title": "Reply Email Group A", "description": "For campaigns split on 'From Name', the reply-to address for Group A." }, "reply_email_b": { - "type": "string", + "type": ["null", "string"], "title": "Reply Email Group B", "description": "For campaigns split on 'From Name', the reply-to address for Group B." }, "subject_a": { - "type": "string", + "type": ["null", "string"], "title": "Subject Line Group A", "description": "For campaigns split on 'Subject Line', the subject line for Group A." }, "subject_b": { - "type": "string", + "type": ["null", "string"], "title": "Subject Line Group B", "description": "For campaigns split on 'Subject Line', the subject line for Group B." }, "send_time_a": { - "type": "string", + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time Group A", "description": "The send time for Group A." }, "send_time_b": { - "type": "string", + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time Group B", "description": "The send time for Group B." }, "send_time_winner": { - "type": "string", + "type": ["null", "string"], "title": "Send Time Winner", "description": "The send time for the winning version." } @@ -601,17 +613,17 @@ "description": "The preview for the campaign, rendered by social networks like Facebook and Twitter. [Learn more](https://mailchimp.com/help/enable-and-customize-social-cards/).", "properties": { "image_url": { - "type": "string", + "type": ["null", "string"], "title": "Image URL", "description": "The url for the header image for the card." }, "description": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Description", "description": "A short summary of the campaign to display." }, "title": { - "type": "string", + "type": ["null", "string"], "title": "Title", "description": "The title for the card. Typically the subject line of the campaign." } @@ -703,7 +715,7 @@ "readOnly": true }, "status": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Delivery Status", "description": "The current state of a campaign delivery.", "enum": ["delivering", "delivered", "canceling", "canceled"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json index 4c4f978fc423..b416956c5427 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json @@ -44,7 +44,8 @@ "type": ["string", "null"], "title": "Action date and time", "description": "The date and time recorded for the action in ISO 8601 format.", - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "url": { "type": ["string", "null"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json index ae20971a55e4..50376c80b74a 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json @@ -70,19 +70,25 @@ "type": ["null", "string"] }, "timestamp_signup": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "ip_opt": { "type": ["null", "string"] }, "timestamp_opt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "member_rating": { "type": ["null", "integer"] }, "last_changed": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "language": { "type": ["null", "string"] @@ -141,7 +147,8 @@ }, "created_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "created_by": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json index f0a21ce1a299..01cd5b3e1881 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json @@ -16,7 +16,7 @@ "readOnly": true }, "name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list." }, @@ -26,49 +26,49 @@ "description": "[Contact information displayed in campaign footers](https://mailchimp.com/help/about-campaign-footers/) to comply with international spam laws.", "properties": { "company": { - "type": "string", + "type": ["null", "string"], "title": "Company Name", "description": "The company name for the list." }, "address1": { - "type": "string", + "type": ["null", "string"], "title": "Address", "description": "The street address for the list contact." }, "address2": { - "type": "string", + "type": ["null", "string"], "title": "Address", "description": "The street address for the list contact." }, "city": { - "type": "string", + "type": ["null", "string"], "title": "City", "description": "The city for the list contact." }, "state": { - "type": "string", + "type": ["null", "string"], "title": "State", "description": "The state for the list contact." }, "zip": { - "type": "string", + "type": ["null", "string"], "title": "Postal Code", "description": "The postal or zip code for the list contact." }, "country": { - "type": "string", + "type": ["null", "string"], "title": "Country Code", "description": "A two-character ISO3166 country code. Defaults to US if invalid." }, "phone": { - "type": "string", + "type": ["null", "string"], "title": "Phone Number", "description": "The phone number for the list contact." } } }, "permission_reminder": { - "type": "string", + "type": ["null", "string"], "title": "Permission Reminder", "description": "The [permission reminder](https://mailchimp.com/help/edit-the-permission-reminder/) for the list." }, @@ -84,35 +84,35 @@ "description": "[Default values for campaigns](https://mailchimp.com/help/edit-your-emails-subject-preview-text-from-name-or-from-email-address/) created for this list.", "properties": { "from_name": { - "type": "string", + "type": ["null", "string"], "title": "Sender's Name", "description": "The default from name for campaigns sent to this list." }, "from_email": { - "type": "string", + "type": ["null", "string"], "title": "Sender's Email Address", "description": "The default from email for campaigns sent to this list." }, "subject": { - "type": "string", + "type": ["null", "string"], "title": "Subject", "description": "The default subject line for campaigns sent to this list." }, "language": { - "type": "string", + "type": ["null", "string"], "title": "Language", "description": "The default language for this lists's forms." } } }, "notify_on_subscribe": { - "type": "string", + "type": ["null", "string"], "title": "Notify on Subscribe", "description": "The email address to send [subscribe notifications](https://mailchimp.com/help/change-subscribe-and-unsubscribe-notifications/) to.", "default": false }, "notify_on_unsubscribe": { - "type": "string", + "type": ["null", "string"], "title": "Notify on Unsubscribe", "description": "The email address to send [unsubscribe notifications](https://mailchimp.com/help/change-subscribe-and-unsubscribe-notifications/) to.", "default": false @@ -122,6 +122,7 @@ "title": "Creation Date", "description": "The date and time that this list was created in ISO 8601 format.", "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "readOnly": true }, "list_rating": { @@ -136,25 +137,25 @@ "description": "Whether the list supports [multiple formats for emails](https://mailchimp.com/help/change-list-name-and-defaults/). When set to `true`, subscribers can choose whether they want to receive HTML or plain-text emails. When set to `false`, subscribers will receive HTML emails, with a plain-text alternative backup." }, "subscribe_url_short": { - "type": "string", + "type": ["null", "string"], "title": "Subscribe URL Short", "description": "Our [EepURL shortened](https://mailchimp.com/help/share-your-signup-form/) version of this list's subscribe form.", "readOnly": true }, "subscribe_url_long": { - "type": "string", + "type": ["null", "string"], "title": "Subscribe URL Long", "description": "The full version of this list's subscribe form (host will vary).", "readOnly": true }, "beamer_address": { - "type": "string", + "type": ["null", "string"], "title": "Beamer Address", "description": "The list's [Email Beamer](https://mailchimp.com/help/use-email-beamer-to-create-a-campaign/) address.", "readOnly": true }, "visibility": { - "type": "string", + "type": ["null", "string"], "title": "Visibility", "enum": ["pub", "prv"], "description": "Whether this list is [public or private](https://mailchimp.com/help/about-list-publicity/)." @@ -183,7 +184,7 @@ "title": "Modules", "description": "Any list-specific modules installed for this list.", "items": { - "type": "string" + "type": ["null", "string"] }, "readOnly": true }, @@ -242,10 +243,12 @@ "readOnly": true }, "campaign_last_sent": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Last Sent", "description": "The date and time the last campaign was sent to this list in ISO 8601 format. This is updated when a campaign is sent to 10 or more recipients.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "merge_field_count": { "type": "integer", @@ -284,16 +287,20 @@ "readOnly": true }, "last_sub_date": { - "type": "string", + "type": ["null", "string"], "title": "Date of Last List Subscribe", "description": "The date and time of the last time someone subscribed to this list in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "last_unsub_date": { - "type": "string", + "type": ["null", "string"], "title": "Date of Last List Unsubscribe", "description": "The date and time of the last time someone unsubscribed from this list in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } } } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json index 34e513022879..940b0a83202c 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json @@ -9,13 +9,13 @@ "description": "A string that uniquely identifies this campaign." }, "campaign_title": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Title", "description": "The title of the campaign.", "readOnly": true }, "type": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Type", "description": "The type of campaign (regular, plain-text, ab_split, rss, automation, variate, or auto)." }, @@ -32,19 +32,19 @@ "readOnly": true }, "list_name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list.", "readOnly": true }, "subject_line": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Subject Line", "description": "The subject line for the campaign.", "readOnly": true }, "preview_text": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Preview Text", "description": "The preview text for the campaign." }, @@ -65,15 +65,17 @@ "readOnly": true }, "send_time": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time", "description": "The date and time a campaign was sent in ISO 8601 format.", "readOnly": true }, "rss_last_send": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "RSS Last Send", "description": "For RSS campaigns, the date and time of the last send in ISO 8601 format.", "readOnly": true @@ -140,6 +142,7 @@ "last_open": { "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Open", "description": "The date and time of the last recorded open in ISO 8601 format." } @@ -173,6 +176,7 @@ "last_click": { "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Click", "description": "The date and time of the last recorded click for the campaign in ISO 8601 format." } @@ -206,7 +210,7 @@ "description": "The average campaign statistics for your industry.", "properties": { "type": { - "type": "string", + "type": ["null", "string"], "title": "Industry Type", "description": "The type of business industry associated with your account. For example: retail, education, etc." }, @@ -319,9 +323,11 @@ "description": "Opens for Campaign A." }, "last_open": { - "type": "string", + "type": ["null", "string"], "title": "Last Open", - "description": "The last open for Campaign A." + "description": "The last open for Campaign A.", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "unique_opens": { "type": "integer", @@ -371,9 +377,11 @@ "description": "Opens for Campaign B." }, "last_open": { - "type": "string", + "type": ["null", "string"], "title": "Last Open", - "description": "The last open for Campaign B." + "description": "The last open for Campaign B.", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "unique_opens": { "type": "integer", @@ -402,8 +410,9 @@ "description": "The number of opens." }, "last_open": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Open", "description": "The date and time of the last open in ISO 8601 format." }, @@ -418,8 +427,9 @@ "description": "The number of clicks." }, "last_click": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Click", "description": "The date and time of the last click in ISO 8601 format." }, @@ -444,8 +454,9 @@ "type": "object", "properties": { "timestamp": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Timestamp", "description": "The date and time for the series in ISO 8601 format." }, @@ -473,13 +484,13 @@ "description": "The url and password for the [VIP report](https://mailchimp.com/help/share-a-campaign-report/).", "properties": { "share_url": { - "type": "string", + "type": ["null", "string"], "title": "Report URL", "description": "The URL for the VIP report.", "readOnly": true }, "share_password": { - "type": "string", + "type": ["null", "string"], "title": "Report Password", "description": "If password protected, the password for the VIP report.", "readOnly": true @@ -510,7 +521,7 @@ "readOnly": true }, "currency_code": { - "type": "string", + "type": ["null", "string"], "title": "Three letter currency code for this user", "readOnly": true, "example": "USD" @@ -535,7 +546,7 @@ "readOnly": true }, "status": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Delivery Status", "description": "The current state of a campaign delivery.", "enum": ["delivering", "delivered", "canceling", "canceled"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json index 3c760dfdeb43..8766876fd2b7 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json @@ -42,20 +42,24 @@ }, "timestamp_signup": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "ip_opt": { "type": ["null", "string"] }, "timestamp_opt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "member_rating": { "type": ["null", "integer"] }, "last_changed": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "language": { "type": ["null", "string"] @@ -97,7 +101,8 @@ }, "created_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "created_by": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json index b5f5c5e051e6..8840817de2e9 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json @@ -17,11 +17,13 @@ }, "created_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "updated_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "options": { "type": ["null", "object"], @@ -33,6 +35,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "condition_type": { "type": ["null", "string"] @@ -42,9 +45,6 @@ }, "op": { "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json index 73aa5dc4015f..ead264a0c180 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json @@ -17,7 +17,9 @@ "type": ["null", "boolean"] }, "timestamp": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "reason": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py index de12e5b8e2f5..0edf00993e5f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py @@ -4,13 +4,16 @@ import base64 +import re from typing import Any, List, Mapping, Tuple +import pendulum import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from pendulum.parsing.exceptions import ParserError from requests.auth import AuthBase from .streams import ( @@ -78,7 +81,30 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: class SourceMailchimp(AbstractSource): + def _validate_start_date(self, config: Mapping[str, Any]): + start_date = config.get("start_date") + + if start_date: + pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + if not pattern.match(start_date): # Compare against the pattern descriptor. + return "Please check the format of the start date against the pattern descriptor." + + try: # Handle invalid dates. + parsed_start_date = pendulum.parse(start_date) + except ParserError: + return "The provided start date is not a valid date. Please check the date you input and try again." + + if parsed_start_date > pendulum.now("UTC"): # Handle future start date. + return "The start date cannot be greater than the current date." + + return None + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + # First, check for a valid start date if it is provided + start_date_validation_error = self._validate_start_date(config) + if start_date_validation_error: + return False, start_date_validation_error + try: authenticator = MailChimpAuthenticator().get_auth(config) response = requests.get( @@ -102,21 +128,22 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = MailChimpAuthenticator().get_auth(config) campaign_id = config.get("campaign_id") + start_date = config.get("start_date") - lists = Lists(authenticator=authenticator) + lists = Lists(authenticator=authenticator, start_date=start_date) interest_categories = InterestCategories(authenticator=authenticator, parent=lists) return [ - Automations(authenticator=authenticator), - Campaigns(authenticator=authenticator), - EmailActivity(authenticator=authenticator, campaign_id=campaign_id), + Automations(authenticator=authenticator, start_date=start_date), + Campaigns(authenticator=authenticator, start_date=start_date), + EmailActivity(authenticator=authenticator, start_date=start_date, campaign_id=campaign_id), interest_categories, Interests(authenticator=authenticator, parent=interest_categories), lists, - ListMembers(authenticator=authenticator), - Reports(authenticator=authenticator), - SegmentMembers(authenticator=authenticator), - Segments(authenticator=authenticator), + ListMembers(authenticator=authenticator, start_date=start_date), + Reports(authenticator=authenticator, start_date=start_date), + SegmentMembers(authenticator=authenticator, start_date=start_date), + Segments(authenticator=authenticator, start_date=start_date), Tags(authenticator=authenticator, parent=lists), - Unsubscribes(authenticator=authenticator, campaign_id=campaign_id), + Unsubscribes(authenticator=authenticator, start_date=start_date, campaign_id=campaign_id), ] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json index c18777fcd36b..f88649faa153 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json @@ -61,6 +61,15 @@ } ] }, + "start_date": { + "title": "Incremental Sync Start Date", + "description": "The date from which you want to start syncing data for Incremental streams. Only records that have been created or modified since this date will be synced. If left blank, all data will by synced.", + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", + "examples": ["2020-01-01T00:00:00.000Z"] + }, "campaign_id": { "type": "string", "title": "ID of a campaign to sync email activities", diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 8206c8f77124..158eaf1e8b47 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import StreamData @@ -45,7 +46,8 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {"count": self.page_size} + # The ._links field is returned by most Mailchimp endpoints and contains non-relevant schema metadata. + params = {"count": self.page_size, "exclude_fields": f"{self.data_field}._links"} # Handle pagination by inserting the next page's token in the request parameters if next_page_token: @@ -80,6 +82,10 @@ def read_records( class IncrementalMailChimpStream(MailChimpStream, ABC): state_checkpoint_interval = math.inf + def __init__(self, **kwargs): + self.start_date = kwargs.pop("start_date", None) + super().__init__(**kwargs) + @property @abstractmethod def cursor_field(self) -> str: @@ -97,6 +103,23 @@ def filter_field(self): def sort_field(self): return self.cursor_field + def filter_empty_fields(self, element: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Many Mailchimp endpoints return empty strings instead of null values. + This causes validation errors on datetime columns, so for safety, we need to check for empty strings and set their value to None/null. + This method recursively traverses each element in a record and replaces any "" values with None, based on three conditions: + + 1. If the element is a dictionary, apply the method recursively to each value in the dictionary. + 2. If the element is a list, apply the method recursively to each item in the list. + 3. If the element is a string, check if it is an empty string. If so, replace it with None. + """ + + if isinstance(element, dict): + element = {k: self.filter_empty_fields(v) if v != "" else None for k, v in element.items()} + elif isinstance(element, list): + element = [self.filter_empty_fields(v) for v in element] + return element + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object @@ -111,11 +134,37 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: slice_ = {} stream_state = stream_state or {} - cursor_value = stream_state.get(self.cursor_field) + cursor_value = self.get_filter_date(self.start_date, stream_state.get(self.cursor_field)) if cursor_value: slice_[self.filter_field] = cursor_value yield slice_ + @staticmethod + def get_filter_date(start_date: str, state_date: str) -> str: + """ + Calculate the filter date to pass in the request parameters by comparing the start_date + with the value of state obtained from the stream_slice. + If only one value exists, use it by default. Otherwise, return None. + If no filter_date is provided, the API will fetch all available records. + """ + + start_date_parsed = pendulum.parse(start_date).to_iso8601_string() if start_date else None + state_date_parsed = pendulum.parse(state_date).to_iso8601_string() if state_date else None + + # Return the max of the two dates if both are present. Otherwise return whichever is present, or None. + if start_date_parsed or state_date_parsed: + return max(filter(None, [start_date_parsed, state_date_parsed]), default=None) + + def filter_old_records(self, records: Iterable, filter_date) -> Iterable: + """ + Filters out records with older cursor_values than the filter_date. + This can be used to enforce the filter for incremental streams that do not support sorting/filtering via query params. + """ + for record in records: + record_cursor_value = record.get(self.cursor_field) + if not filter_date or record_cursor_value >= filter_date: + yield record + def request_params(self, stream_state=None, stream_slice=None, **kwargs): stream_state = stream_state or {} stream_slice = stream_slice or {} @@ -124,6 +173,11 @@ def request_params(self, stream_state=None, stream_slice=None, **kwargs): params.update(default_params) return params + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response = super().parse_response(response, **kwargs) + for record in response: + yield self.filter_empty_fields(record) + class MailChimpListSubStream(IncrementalMailChimpStream): """ @@ -133,8 +187,12 @@ class MailChimpListSubStream(IncrementalMailChimpStream): def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: stream_state = stream_state or {} parent = Lists(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) - for slice in parent: - yield {"list_id": slice["id"]} + for parent_record in parent: + slice = {"list_id": parent_record["id"]} + cursor_value = self.get_filter_date(self.start_date, stream_state.get(parent_record["id"], {}).get(self.cursor_field)) + if cursor_value: + slice[self.filter_field] = cursor_value + yield slice def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: list_id = stream_slice.get("list_id") @@ -143,9 +201,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: def request_params(self, stream_state=None, stream_slice=None, **kwargs) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - # Exclude the _links field, as it is not user-relevant data - params["exclude_fields"] = f"{self.data_field}._links" - # Get the current state value for this list_id, if it exists # Then, use the value in state to filter the request current_slice = stream_slice.get("list_id") @@ -218,7 +273,8 @@ def stream_slices( campaigns = Campaigns(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) for campaign in campaigns: slice_ = {"campaign_id": campaign["id"]} - cursor_value = stream_state.get(campaign["id"], {}).get(self.cursor_field) + state_value = stream_state.get(campaign["id"], {}).get(self.cursor_field) + cursor_value = self.get_filter_date(self.start_date, state_value) if cursor_value: slice_[self.filter_field] = cursor_value yield slice_ @@ -275,13 +331,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: list_id = stream_slice.get("parent").get("id") return f"lists/{list_id}/interest-categories" - def request_params(self, **kwargs): - - # Exclude the _links field, as it is not user-relevant data - params = super().request_params(**kwargs) - params["exclude_fields"] = "categories._links" - return params - class Interests(MailChimpStream, HttpSubStream): """ @@ -299,13 +348,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: category_id = stream_slice.get("parent").get("id") return f"lists/{list_id}/interest-categories/{category_id}/interests" - def request_params(self, **kwargs): - - # Exclude the _links field, as it is not user-relevant data - params = super().request_params(**kwargs) - params["exclude_fields"] = "interests._links" - return params - class ListMembers(MailChimpListSubStream): """ @@ -321,29 +363,9 @@ class Reports(IncrementalMailChimpStream): cursor_field = "send_time" data_field = "reports" - @staticmethod - def remove_empty_datetime_fields(record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - In some cases, the 'clicks.last_click' and 'opens.last_open' fields are returned as an empty string, - which causes validation errors on the `date-time` format. - To avoid this, we remove the fields if they are empty. - """ - clicks = record.get("clicks", {}) - opens = record.get("opens", {}) - if not clicks.get("last_click"): - clicks.pop("last_click", None) - if not opens.get("last_open"): - opens.pop("last_open", None) - return record - def path(self, **kwargs) -> str: return "reports" - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response = super().parse_response(response, **kwargs) - for record in response: - yield self.remove_empty_datetime_fields(record) - class SegmentMembers(MailChimpListSubStream): """ @@ -354,24 +376,6 @@ class SegmentMembers(MailChimpListSubStream): cursor_field = "last_changed" data_field = "members" - def nullify_empty_string_fields(self, element: Mapping[str, Any]) -> Mapping[str, Any]: - """ - SegmentMember records may contain multiple fields that are returned as empty strings, which causes validation issues for fields with declared "datetime" formats. - Since all fields are nullable, replacing any string value of "" with None is a safe way to handle these edge cases. - - :param element: A SegmentMember record, dictionary or list - """ - - if isinstance(element, dict): - # If the element is a dictionary, apply the method recursively to each value, - # replacing the empty string value with None. - element = {k: self.nullify_empty_string_fields(v) if v != "" else None for k, v in element.items()} - elif isinstance(element, list): - # If the element is a list, apply the method recursively to each item in the list. - element = [self.nullify_empty_string_fields(v) for v in element] - - return element - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: """ Each slice consists of a list_id and segment_id pair @@ -391,18 +395,20 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice, **kwargs) -> Iterable[Mapping]: """ - SegmentMembers endpoint does not support sorting, so we need to filter out records that are older than the current state + The SegmentMembers endpoint does not support sorting or filtering, + so we need to apply our own filtering logic before reading. + The foreign key "segment_id" is also added to each record before being read. """ response = super().parse_response(response, **kwargs) - for record in response: + # Calculate the filter date to compare all records against in this slice + slice_cursor_value = stream_state.get(str(stream_slice.get("segment_id")), {}).get(self.cursor_field) + filter_date = self.get_filter_date(self.start_date, slice_cursor_value) + + for record in self.filter_old_records(response, filter_date): # Add the segment_id foreign_key to each record record["segment_id"] = stream_slice.get("segment_id") - - current_cursor_value = stream_state.get(str(record.get("segment_id")), {}).get(self.cursor_field) - record_cursor_value = record.get(self.cursor_field) - if current_cursor_value is None or record_cursor_value >= current_cursor_value: - yield self.nullify_empty_string_fields(record) + yield record def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = current_stream_state or {} @@ -485,22 +491,17 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: campaign_id = stream_slice.get("campaign_id") return f"reports/{campaign_id}/unsubscribed" - def request_params(self, stream_state=None, stream_slice=None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - # Exclude the _links field, as it is not user-relevant data - params["exclude_fields"] = "unsubscribes._links" - return params - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice, **kwargs) -> Iterable[Mapping]: + """ + The Unsubscribes endpoint does not support sorting or filtering, + so we need to apply our own filtering logic before reading. + """ response = super().parse_response(response, **kwargs) - # Unsubscribes endpoint does not support sorting, so we need to filter out records that are older than the current state - for record in response: - current_cursor_value = stream_state.get(record.get("campaign_id"), {}).get(self.cursor_field) - record_cursor_value = record.get(self.cursor_field) - if current_cursor_value is None or record_cursor_value >= current_cursor_value: - yield record + slice_cursor_value = stream_state.get(stream_slice.get("campaign_id", {}), {}).get(self.cursor_field) + filter_date = self.get_filter_date(self.start_date, slice_cursor_value) + yield from self.filter_old_records(response, filter_date) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = current_stream_state or {} diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py index c2bdd6af7712..5305f0dadab4 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py @@ -14,7 +14,7 @@ def data_center_fixture(): @fixture(name="config") def config_fixture(data_center): - return {"apikey": f"API_KEY-{data_center}"} + return {"apikey": f"API_KEY-{data_center}", "start_date": "2022-01-01T00:00:00.000Z"} @fixture(name="access_token") diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py index 92be6ee481b5..b1ccfcddac6a 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py @@ -88,6 +88,29 @@ def test_wrong_config(wrong_config): MailChimpAuthenticator().get_auth(wrong_config) +@pytest.mark.parametrize( + "config, expected_return", + [ + ({}, None), + ({"start_date": "2021-01-01T00:00:00.000Z"}, None), + ({"start_date": "2021-99-99T79:89:99.123Z"}, "The provided start date is not a valid date. Please check the date you input and try again."), + ({"start_date": "2021-01-01T00:00:00.000"}, "Please check the format of the start date against the pattern descriptor."), + ({"start_date": "2025-01-25T00:00:00.000Z"}, "The start date cannot be greater than the current date."), + ], + ids=[ + "No start date", + "Valid start date", + "Invalid start date", + "Invalid format", + "Future start date", + ] +) +def test_validate_start_date(config, expected_return): + source = SourceMailchimp() + result = source._validate_start_date(config) + assert result == expected_return + + def test_streams_count(config): streams = SourceMailchimp().streams(config) assert len(streams) == 12 diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py index 47bc7028f5f8..b441fe26f7b3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py @@ -76,12 +76,12 @@ def test_next_page_token(auth): ( Lists, {"stream_slice": None, "stream_state": None, "next_page_token": None}, - {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created"}, + {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "exclude_fields": "lists._links"}, ), ( Lists, {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 1000}}, - {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000}, + {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000, "exclude_fields": "lists._links"}, ), ( InterestCategories, @@ -170,7 +170,7 @@ def test_stream_parse_json_error(auth, caplog): # Test case 2: state and next_page_token ( ListMembers, - {"list_id": "123"}, + {"list_id": "123", "since_last_changed": "2023-10-15T00:00:00Z"}, {"123": {"last_changed": "2023-10-15T00:00:00Z"}}, {"offset": 1000}, { @@ -299,34 +299,48 @@ def test_segment_members_parse_response(auth, stream_state, records, expected): @pytest.mark.parametrize( - "record, expected_record", + "stream, record, expected_record", [ ( + SegmentMembers, {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": ""}, {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": None} ), ( + SegmentMembers, {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": "", "ADDRESS": "", "PHONE": ""}}, {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": None, "ADDRESS": None, "PHONE": None}} ), ( - {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": "Bobson", "ADDRESS": "101 Bob Ln", "PHONE": "111-111-1111"}}, - {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": "Bobson", "ADDRESS": "101 Bob Ln", "PHONE": "111-111-1111"}} + Campaigns, + {"id": "1", "web_id": 2, "email_type": "html", "create_time": "2022-01-01T00:00:00.000Z", "send_time": ""}, + {"id": "1", "web_id": 2, "email_type": "html", "create_time": "2022-01-01T00:00:00.000Z", "send_time": None} + ), + ( + Reports, + {"id": "1", "type": "rss", "clicks": {"clicks_total": 1, "last_click": "2022-01-01T00:00:00Z"}, "opens": {"opens_total": 0, "last_open": ""}}, + {"id": "1", "type": "rss", "clicks": {"clicks_total": 1, "last_click": "2022-01-01T00:00:00Z"}, "opens": {"opens_total": 0, "last_open": None}} + ), + ( + Lists, + {"id": "1", "name": "Santa's List", "stats": {"last_sub_date": "2022-01-01T00:00:00Z", "last_unsub_date": ""}}, + {"id": "1", "name": "Santa's List", "stats": {"last_sub_date": "2022-01-01T00:00:00Z", "last_unsub_date": None}} ) ], ids=[ - "Replace empty string with None", - "Replace empty strings with None in nested fields", - "Leave non-empty string fields unchanged" + "segment_members: opt_timestamp nullified", + "segment_members: nested merge_fields nullified", + "campaigns: send_time nullified", + "reports: nested opens.last_open nullified", + "lists: stats.last_unsub_date nullified" ] ) -def test_segment_members_nullify_empty_string_fields(auth, record, expected_record): +def test_filter_empty_fields(auth, stream, record, expected_record): """ - Tests that empty string values in SegmentMembers stream are converted to None + Tests that empty string values are converted to None. """ - stream = SegmentMembers(authenticator=auth) - - assert stream.nullify_empty_string_fields(record) == expected_record + stream = stream(authenticator=auth) + assert stream.filter_empty_fields(record) == expected_record def test_unsubscribes_stream_slices(requests_mock, unsubscribes_stream, campaigns_stream, mock_campaigns_response): @@ -391,7 +405,8 @@ def test_parse_response(stream_state, expected_records, unsubscribes_stream): {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, ] } - records = list(unsubscribes_stream.parse_response(response=mock_response, stream_state=stream_state)) + stream_slice = {"campaign_id": "campaign_1"} + records = list(unsubscribes_stream.parse_response(response=mock_response, stream_slice=stream_slice, stream_state=stream_state)) assert records == expected_records @@ -595,38 +610,87 @@ def test_path(auth, stream, stream_slice, expected_endpoint): @pytest.mark.parametrize( - "record, expected_return", + "start_date, state_date, expected_return_value", [ ( - {"clicks": {"last_click": ""}, "opens": {"last_open": ""}}, - {"clicks": {}, "opens": {}}, + "2021-01-01T00:00:00.000Z", + "2020-01-01T00:00:00+00:00", + "2021-01-01T00:00:00Z" ), ( - {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": ""}}, - {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {}}, + "2021-01-01T00:00:00.000Z", + "2023-10-05T00:00:00+00:00", + "2023-10-05T00:00:00+00:00" ), - ( - {"clicks": {"last_click": ""}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, - {"clicks": {}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, - + ( + None, + "2022-01-01T00:00:00+00:00", + "2022-01-01T00:00:00+00:00" ), ( - {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, - {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, + "2020-01-01T00:00:00.000Z", + None, + "2020-01-01T00:00:00Z" ), + ( + None, + None, + None + ) + ] +) +def test_get_filter_date(auth, start_date, state_date, expected_return_value): + """ + Tests that the get_filter_date method returns the correct date string + """ + stream = Campaigns(authenticator=auth, start_date=start_date) + result = stream.get_filter_date(start_date, state_date) + assert result == expected_return_value, f"Expected: {expected_return_value}, Actual: {result}" + + +@pytest.mark.parametrize( + "stream_class, records, filter_date, expected_return_value", + [ + ( + Unsubscribes, + [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-04T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-03T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-01T00:00:00Z"}, + ], + "2022-01-02T12:00:00+00:00", + [ + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-04T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ( + SegmentMembers, + [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-04T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 3, "segment_id": "segment_1", "last_changed": "2021-01-03T00:00:00Z"}, + {"id": 4, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"}, + ], + None, + [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-04T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 3, "segment_id": "segment_1", "last_changed": "2021-01-03T00:00:00Z"}, + {"id": 4, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"}, + ], + ) ], ids=[ - "last_click and last_open empty", - "last_click empty", - "last_open empty", - "last_click and last_open not empty" + "Unsubscribes: filter_date is set, records filtered", + "SegmentMembers: filter_date is None, all records returned" ] ) -def test_reports_remove_empty_datetime_fields(auth, record, expected_return): +def test_filter_old_records(auth, stream_class, records, filter_date, expected_return_value): """ - Tests that the Reports stream removes the 'clicks' and 'opens' fields from the response - when they are empty strings + Tests the logic for filtering old records in streams that do not support query_param filtering. """ - stream = Reports(authenticator=auth) - assert stream.remove_empty_datetime_fields(record) == expected_return, f"Expected: {expected_return}, Actual: {stream.remove_empty_datetime_fields(record)}" - + stream = stream_class(authenticator=auth) + filtered_records = list(stream.filter_old_records(records, filter_date)) + assert filtered_records == expected_return_value diff --git a/airbyte-integrations/connectors/source-mailerlite/main.py b/airbyte-integrations/connectors/source-mailerlite/main.py index 9c906a15e012..1f2347a95e63 100644 --- a/airbyte-integrations/connectors/source-mailerlite/main.py +++ b/airbyte-integrations/connectors/source-mailerlite/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailerlite import SourceMailerlite +from source_mailerlite.run import run if __name__ == "__main__": - source = SourceMailerlite() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml index fae3ac67fb61..b89c174e8d0f 100644 --- a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailerlite.svg license: MIT name: MailerLite + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailerlite registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailerlite/setup.py b/airbyte-integrations/connectors/source-mailerlite/setup.py index 5b49cb1a5d31..71fc89f1f56a 100644 --- a/airbyte-integrations/connectors/source-mailerlite/setup.py +++ b/airbyte-integrations/connectors/source-mailerlite/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailerlite=source_mailerlite.run:run", + ], + }, name="source_mailerlite", description="Source implementation for Mailerlite.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py new file mode 100644 index 000000000000..c1c7548459e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailerlite import SourceMailerlite + + +def run(): + source = SourceMailerlite() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailersend/main.py b/airbyte-integrations/connectors/source-mailersend/main.py index 8b72a409be46..9afae9104c15 100644 --- a/airbyte-integrations/connectors/source-mailersend/main.py +++ b/airbyte-integrations/connectors/source-mailersend/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailersend import SourceMailersend +from source_mailersend.run import run if __name__ == "__main__": - source = SourceMailersend() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailersend/metadata.yaml b/airbyte-integrations/connectors/source-mailersend/metadata.yaml index 436e6a8514ed..97799066552c 100644 --- a/airbyte-integrations/connectors/source-mailersend/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailersend/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailersend.svg license: MIT name: MailerSend + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailersend registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailersend/setup.py b/airbyte-integrations/connectors/source-mailersend/setup.py index 8fc9c7d2bdbc..33d180e17b8f 100644 --- a/airbyte-integrations/connectors/source-mailersend/setup.py +++ b/airbyte-integrations/connectors/source-mailersend/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailersend=source_mailersend.run:run", + ], + }, name="source_mailersend", description="Source implementation for Mailersend.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py b/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py new file mode 100644 index 000000000000..ddc9feff24b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailersend import SourceMailersend + + +def run(): + source = SourceMailersend() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailgun/main.py b/airbyte-integrations/connectors/source-mailgun/main.py index bffb131e2745..ee1ad2379837 100644 --- a/airbyte-integrations/connectors/source-mailgun/main.py +++ b/airbyte-integrations/connectors/source-mailgun/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailgun import SourceMailgun +from source_mailgun.run import run if __name__ == "__main__": - source = SourceMailgun() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailgun/metadata.yaml b/airbyte-integrations/connectors/source-mailgun/metadata.yaml index 7d3e7d30ee15..669d11d1ef41 100644 --- a/airbyte-integrations/connectors/source-mailgun/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailgun/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.mailgun.net/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailgun registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-mailgun/setup.py b/airbyte-integrations/connectors/source-mailgun/setup.py index f7661245f2fd..616448f500d5 100644 --- a/airbyte-integrations/connectors/source-mailgun/setup.py +++ b/airbyte-integrations/connectors/source-mailgun/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailgun=source_mailgun.run:run", + ], + }, name="source_mailgun", description="Source implementation for Mailgun.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py b/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py new file mode 100644 index 000000000000..6cda7fec6e71 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailgun import SourceMailgun + + +def run(): + source = SourceMailgun() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile index db4a76403c77..a05924e3dc82 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile +++ b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile @@ -34,5 +34,5 @@ COPY source_mailjet_mail ./source_mailjet_mail ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-mailjet-mail diff --git a/airbyte-integrations/connectors/source-mailjet-mail/main.py b/airbyte-integrations/connectors/source-mailjet-mail/main.py index 46dc79c4b5f7..5d8c4b9df24e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/main.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailjet_mail import SourceMailjetMail +from source_mailjet_mail.run import run if __name__ == "__main__": - source = SourceMailjetMail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml index 9a5b49f50b99..ce0bd1f6e3b2 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: 56582331-5de2-476b-b913-5798de77bbdf - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-mailjet-mail githubIssueLabel: source-mailjet-mail icon: mailjetmail.svg license: MIT name: Mailjet Mail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailjet-mail registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailjet-mail/setup.py b/airbyte-integrations/connectors/source-mailjet-mail/setup.py index 5f6daf3cdc2b..ea60824d3a2c 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailjet-mail=source_mailjet_mail.run:run", + ], + }, name="source_mailjet_mail", description="Source implementation for Mailjet Mail.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml index 222189652d69..dfad8159a782 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml @@ -83,7 +83,7 @@ definitions: $parameters: name: "message" primary_key: "ID" - path: "/message" + path: "/message?ShowSubject=true" listrecipient_stream: $ref: "#/definitions/base_stream" retriever: diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py new file mode 100644 index 000000000000..87dcd7ecc198 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_mail import SourceMailjetMail + + +def run(): + source = SourceMailjetMail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/main.py b/airbyte-integrations/connectors/source-mailjet-sms/main.py index 7dfd0868672e..52420c275761 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/main.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailjet_sms import SourceMailjetSms +from source_mailjet_sms.run import run if __name__ == "__main__": - source = SourceMailjetSms() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml index a68dbed823e2..307381bb7f6c 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailjetsms.svg license: MIT name: Mailjet SMS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailjet-sms registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-mailjet-sms/setup.py b/airbyte-integrations/connectors/source-mailjet-sms/setup.py index 7edfd5e35e8e..6b4be21374cc 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailjet-sms=source_mailjet_sms.run:run", + ], + }, name="source_mailjet_sms", description="Source implementation for Mailjet Sms.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py new file mode 100644 index 000000000000..a8568fdb2f68 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_sms import SourceMailjetSms + + +def run(): + source = SourceMailjetSms() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-marketo/README.md b/airbyte-integrations/connectors/source-marketo/README.md index 50676f5b93e4..078e93d4a425 100644 --- a/airbyte-integrations/connectors/source-marketo/README.md +++ b/airbyte-integrations/connectors/source-marketo/README.md @@ -1,118 +1,55 @@ -# Marketo Source +# Marketo source connector + This is the repository for the Marketo source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/marketo). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/marketo). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/marketo) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_marketo/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/marketo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_marketo/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source marketo test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-marketo spec +poetry run source-marketo check --config secrets/config.json +poetry run source-marketo discover --config secrets/config.json +poetry run source-marketo read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-marketo build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-marketo:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-marketo:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-marketo:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-marketo:dev . -# Running the spec command against your patched connector -docker run airbyte/source-marketo:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-marketo:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-marketo:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-marketo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-marketo test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-marketo test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/marketo.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/marketo.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-marketo/main.py b/airbyte-integrations/connectors/source-marketo/main.py index 127c4d2c05ad..4b7b8e8d1708 100644 --- a/airbyte-integrations/connectors/source-marketo/main.py +++ b/airbyte-integrations/connectors/source-marketo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_marketo import SourceMarketo +from source_marketo.run import run if __name__ == "__main__": - source = SourceMarketo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-marketo/metadata.yaml b/airbyte-integrations/connectors/source-marketo/metadata.yaml index d607c7488003..4b3115eb7f2c 100644 --- a/airbyte-integrations/connectors/source-marketo/metadata.yaml +++ b/airbyte-integrations/connectors/source-marketo/metadata.yaml @@ -1,22 +1,26 @@ data: ab_internal: - ql: 400 + ql: 200 sl: 200 allowedHosts: hosts: - "*.mktorest.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 - dockerImageTag: 1.2.2 + dockerImageTag: 1.2.6 dockerRepository: airbyte/source-marketo documentationUrl: https://docs.airbyte.com/integrations/sources/marketo githubIssueLabel: source-marketo icon: marketo.svg license: ELv2 name: Marketo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-marketo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-marketo/poetry.lock b/airbyte-integrations/connectors/source-marketo/poetry.lock new file mode 100644 index 000000000000..d478bc710394 --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/poetry.lock @@ -0,0 +1,1061 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "faker" +version = "23.1.0" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Faker-23.1.0-py3-none-any.whl", hash = "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2"}, + {file = "Faker-23.1.0.tar.gz", hash = "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-faker" +version = "2.0.0" +description = "Faker integration with the pytest framework." +optional = false +python-versions = "*" +files = [ + {file = "pytest-faker-2.0.0.tar.gz", hash = "sha256:6b37bb89d94f96552bfa51f8e8b89d32addded8ddb58a331488299ef0137d9b6"}, +] + +[package.dependencies] +Faker = ">=0.7.3" + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "9345aade7d55768682023c361cdcf0a5fd1c24460ac815588361db515dbbcb29" diff --git a/airbyte-integrations/connectors/source-marketo/pyproject.toml b/airbyte-integrations/connectors/source-marketo/pyproject.toml new file mode 100644 index 000000000000..c530ce9cb4ac --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.2.6" +name = "source-marketo" +description = "Source implementation for Marketo." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/marketo" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_marketo" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.8" + +[tool.poetry.scripts] +source-marketo = "source_marketo.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +requests-mock = "^1.11.0" +pytest-faker = "==2.0.0" +pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-marketo/requirements.txt b/airbyte-integrations/connectors/source-marketo/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-marketo/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py deleted file mode 100644 index 1588bd2fc2a5..000000000000 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-faker==2.0.0", - "pytest-mock~=3.6.1", - "requests-mock", -] - -setup( - name="source_marketo", - description="Source implementation for Marketo.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/run.py b/airbyte-integrations/connectors/source-marketo/source_marketo/run.py new file mode 100644 index 000000000000..0831c3167f5f --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_marketo import SourceMarketo + + +def run(): + source = SourceMarketo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index fffc07af0974..62d4ded15196 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -5,6 +5,7 @@ import csv import datetime import json +import re from abc import ABC from time import sleep from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -17,6 +18,8 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from .utils import STRING_TYPES, clean_string, format_value, to_datetime_str @@ -231,7 +234,10 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp schema = self.get_json_schema()["properties"] response.encoding = "utf-8" - reader = csv.DictReader(response.iter_lines(chunk_size=1024, decode_unicode=True)) + response_lines = response.iter_lines(chunk_size=1024, decode_unicode=True) + filtered_response_lines = self.filter_null_bytes(response_lines) + reader = self.csv_rows(filtered_response_lines) + for record in reader: new_record = {**record} attributes = json.loads(new_record.pop("attributes", "{}")) @@ -257,6 +263,23 @@ def read_records( self.sleep_till_export_completed(stream_slice) return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + def filter_null_bytes(self, response_lines: Iterable[str]) -> Iterable[str]: + for line in response_lines: + res = line.replace("\x00", "") + if len(res) < len(line): + self.logger.warning("Filter 'null' bytes from string, size reduced %d -> %d chars", len(line), len(res)) + yield res + + @staticmethod + def csv_rows(lines: Iterable[str]) -> Iterable[Mapping]: + reader = csv.reader(lines) + headers = None + for row in reader: + if headers is None: + headers = row + else: + yield dict(zip(headers, row)) + class MarketoExportCreate(MarketoStream): """ @@ -280,8 +303,12 @@ def path(self, **kwargs) -> str: def should_retry(self, response: requests.Response) -> bool: if response.status_code == 429 or 500 <= response.status_code < 600: return True - record = next(self.parse_response(response, {}), {}) - status, export_id = record.get("status", "").lower(), record.get("exportId") + if errors := response.json().get("errors"): + if errors[0].get("code") == "1029" and re.match("Export daily quota \d+MB exceeded", errors[0].get("message")): + message = "Daily limit for job extractions has been reached (resets daily at 12:00AM CST)." + raise AirbyteTracedException(internal_message=response.text, message=message, failure_type=FailureType.config_error) + result = response.json().get("result")[0] + status, export_id = result.get("status", "").lower(), result.get("exportId") if status != "created" or not export_id: self.logger.warning(f"Failed to create export job! Status is {status}!") return True diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index b28e89130264..806f39da100d 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -6,12 +6,24 @@ import os import tracemalloc from functools import partial -from unittest.mock import ANY, Mock, patch +from unittest.mock import ANY, MagicMock, Mock, patch import pendulum import pytest +import requests from airbyte_cdk.models.airbyte_protocol import SyncMode -from source_marketo.source import Activities, Campaigns, Leads, MarketoStream, Programs, SourceMarketo +from airbyte_cdk.utils import AirbyteTracedException +from source_marketo.source import ( + Activities, + Campaigns, + IncrementalMarketoStream, + Leads, + MarketoExportCreate, + MarketoStream, + Programs, + Segmentations, + SourceMarketo, +) def test_create_export_job(mocker, send_email_stream, caplog): @@ -26,6 +38,28 @@ def test_create_export_job(mocker, send_email_stream, caplog): assert "Failed to create export job! Status is failed!" in caplog.records[-1].message +def test_should_retry_quota_exceeded(config, requests_mock): + create_job_url = "https://602-euo-598.mktorest.com/rest/v1/leads/export/create.json?batchSize=300" + response_json = { + "requestId": "d2ca#18c0b9833bf", + "success": False, + "errors": [ + { + "code": "1029", + "message": "Export daily quota 500MB exceeded." + } + ] + } + requests_mock.register_uri("GET", create_job_url, status_code=200, json=response_json) + + response = requests.get(create_job_url) + with pytest.raises(AirbyteTracedException) as e: + MarketoExportCreate(config).should_retry(response) + + assert e.value.message == "Daily limit for job extractions has been reached (resets daily at 12:00AM CST)." + + + @pytest.mark.parametrize( "activity, expected_schema", ( @@ -284,10 +318,40 @@ def test_check_connection(config, requests_mock, status_code, response, is_conne ("2020-08-01", "%Y-%m-%dT%H:%M:%SZ%z", "2020-08-01"), ), ) -def test_normalize_datetime(config, input, format, expected_result): +def test_programs_normalize_datetime(config, input, format, expected_result): stream = Programs(config) assert stream.normalize_datetime(input, format) == expected_result +def test_programs_next_page_token(config): + mock_json = MagicMock() + mock_json.return_value = {"result": [{"test": 'testValue'}]} + mocked_response = MagicMock() + mocked_response.json = mock_json + stream = Programs(config) + result = stream.next_page_token(mocked_response) + assert result == {"offset": 201} + +@pytest.mark.parametrize("input, stream_state, expected_result",[( + {"result": [{"id": "1", "createdAt": "2020-07-01T00:00:00Z+0000", "updatedAt": "2020-07-01T00:00:00Z+0000"}]}, + {"updatedAt": "2020-06-01T00:00:00Z"}, + [{"id": "1", "createdAt": "2020-07-01T00:00:00Z", "updatedAt": "2020-07-01T00:00:00Z"}], + )], +) +def test_programs_parse_response(mocker, config, input, stream_state, expected_result): + response = requests.Response() + mocker.patch.object(response, "json", return_value=input) + stream = Programs(config) + result = stream.parse_response(response, stream_state) + assert list(result) == expected_result + +def test_segmentations_next_page_token(config): + mock_json = MagicMock() + mock_json.return_value = {"result": [{"test": 'testValue'}]} + mocked_response = MagicMock() + mocked_response.json = mock_json + stream = Segmentations(config) + result = stream.next_page_token(mocked_response) + assert result == {"offset": 201} today = pendulum.now() yesterday = pendulum.now().subtract(days=1).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -314,3 +378,56 @@ def test_get_updated_state(config, latest_record, current_state, expected_state) if expected_state == "start_date": expected_state = {"updatedAt": config["start_date"]} assert stream.get_updated_state(latest_record, current_state) == expected_state + + +def test_filter_null_bytes(config): + stream = Leads(config) + + test_lines = [ + "Hello\x00World\n", + "Name,Email\n", + "John\x00Doe,john.doe@example.com\n" + ] + expected_lines = [ + "HelloWorld\n", + "Name,Email\n", + "JohnDoe,john.doe@example.com\n" + ] + filtered_lines = stream.filter_null_bytes(test_lines) + for expected_line, filtered_line in zip(expected_lines, filtered_lines): + assert expected_line == filtered_line + + +def test_csv_rows(config): + stream = Leads(config) + + test_lines = [ + "Name,Email\n", + "John Doe,john.doe@example.com\n", + "Jane Doe,jane.doe@example.com\n" + ] + expected_records = [ + {"Name": "John Doe", "Email": "john.doe@example.com"}, + {"Name": "Jane Doe", "Email": "jane.doe@example.com"} + ] + records = stream.csv_rows(test_lines) + for expected_record, record in zip(expected_records, records): + assert expected_record == record + +def test_availablity_strategy(config): + stream = Leads(config) + assert stream.availability_strategy == None + +def test_path(config): + stream = MarketoStream(config) + assert stream.path() == "rest/v1/marketo_stream.json" + +def test_get_state(config): + stream = IncrementalMarketoStream(config) + assert stream.state == {} + +def test_set_tate(config): + stream = IncrementalMarketoStream(config) + expected_state = {"id": 1} + stream.state = expected_state + assert stream._state == expected_state diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py index 946885db1554..df3638db9614 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py @@ -3,8 +3,10 @@ # +from datetime import datetime + import pytest -from source_marketo.utils import clean_string, format_value +from source_marketo.utils import clean_string, format_value, to_datetime_str test_data = [ (1, {"type": "integer"}, int), @@ -15,11 +17,12 @@ ("1.5", {"type": "integer"}, int), ("15", {"type": "integer"}, int), ("true", {"type": "boolean"}, bool), + ("test_custom", {"type": "custom_type"}, str), ] @pytest.mark.parametrize("value,schema,expected_output_type", test_data) -def test_fromat_value(value, schema, expected_output_type): +def test_format_value(value, schema, expected_output_type): test = format_value(value, schema) assert isinstance(test, expected_output_type) @@ -55,3 +58,9 @@ def test_clean_string(value, expected): test = clean_string(value) assert test == expected + +def test_to_datetime_str(): + input = datetime(2023, 1, 1) + expected = "2023-01-01T00:00:00Z" + + assert to_datetime_str(input) == expected diff --git a/airbyte-integrations/connectors/source-merge/main.py b/airbyte-integrations/connectors/source-merge/main.py index 53420fa4b3c1..694c83302b0b 100644 --- a/airbyte-integrations/connectors/source-merge/main.py +++ b/airbyte-integrations/connectors/source-merge/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_merge import SourceMerge +from source_merge.run import run if __name__ == "__main__": - source = SourceMerge() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-merge/metadata.yaml b/airbyte-integrations/connectors/source-merge/metadata.yaml index 13c6df4feb03..8a311f0a516d 100644 --- a/airbyte-integrations/connectors/source-merge/metadata.yaml +++ b/airbyte-integrations/connectors/source-merge/metadata.yaml @@ -8,6 +8,10 @@ data: icon: merge.svg license: MIT name: Merge + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-merge registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-merge/setup.py b/airbyte-integrations/connectors/source-merge/setup.py index aa45d3bdc5e2..b403aa227f4c 100644 --- a/airbyte-integrations/connectors/source-merge/setup.py +++ b/airbyte-integrations/connectors/source-merge/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-merge=source_merge.run:run", + ], + }, name="source_merge", description="Source implementation for Merge.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-merge/source_merge/run.py b/airbyte-integrations/connectors/source-merge/source_merge/run.py new file mode 100644 index 000000000000..63187bee4753 --- /dev/null +++ b/airbyte-integrations/connectors/source-merge/source_merge/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_merge import SourceMerge + + +def run(): + source = SourceMerge() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-metabase/main.py b/airbyte-integrations/connectors/source-metabase/main.py index aeaaab111542..1e571d2ddb48 100644 --- a/airbyte-integrations/connectors/source-metabase/main.py +++ b/airbyte-integrations/connectors/source-metabase/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_metabase import SourceMetabase +from source_metabase.run import run if __name__ == "__main__": - source = SourceMetabase() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-metabase/metadata.yaml b/airbyte-integrations/connectors/source-metabase/metadata.yaml index c6289bf09e4f..2ee8fe2744b6 100644 --- a/airbyte-integrations/connectors/source-metabase/metadata.yaml +++ b/airbyte-integrations/connectors/source-metabase/metadata.yaml @@ -15,6 +15,10 @@ data: icon: metabase.svg license: MIT name: Metabase + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-metabase registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-metabase/setup.py b/airbyte-integrations/connectors/source-metabase/setup.py index 228d443e50e2..e6772737da24 100644 --- a/airbyte-integrations/connectors/source-metabase/setup.py +++ b/airbyte-integrations/connectors/source-metabase/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-metabase=source_metabase.run:run", + ], + }, name="source_metabase", description="Source implementation for Metabase.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/run.py b/airbyte-integrations/connectors/source-metabase/source_metabase/run.py new file mode 100644 index 000000000000..90df278478af --- /dev/null +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_metabase import SourceMetabase + + +def run(): + source = SourceMetabase() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py index 7cf530c14f54..88b4cf3808e8 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_microsoft_dataverse import SourceMicrosoftDataverse +from source_microsoft_dataverse.run import run if __name__ == "__main__": - source = SourceMicrosoftDataverse() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml index b91c74f4767e..30fbf7e9da33 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml @@ -8,6 +8,10 @@ data: icon: microsoftdataverse.svg license: MIT name: Microsoft Dataverse + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-microsoft-dataverse registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py index e86306c30138..57920ae5cd4a 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-microsoft-dataverse=source_microsoft_dataverse.run:run", + ], + }, name="source_microsoft_dataverse", description="Source implementation for Microsoft Dataverse.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py new file mode 100644 index 000000000000..b5e6982f7292 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_microsoft_dataverse import SourceMicrosoftDataverse + + +def run(): + source = SourceMicrosoftDataverse() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/README.md b/airbyte-integrations/connectors/source-microsoft-onedrive/README.md new file mode 100644 index 000000000000..ee6043e14b38 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/README.md @@ -0,0 +1,166 @@ +# Microsoft Onedrive Source + +This is the repository for the Microsoft Onedrive source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/microsoft-onedrive). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/microsoft-onedrive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_microsoft_onedrive/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source microsoft-onedrive test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-microsoft-onedrive build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-microsoft-onedrive:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-microsoft-onedrive:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-microsoft-onedrive:dev . +# Running the spec command against your patched connector +docker run airbyte/source-microsoft-onedrive:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-microsoft-onedrive:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-onedrive:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-onedrive:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-microsoft-onedrive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-microsoft-onedrive test +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-microsoft-onedrive test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/microsoft-onedrive.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml new file mode 100644 index 000000000000..3966afb65b00 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml @@ -0,0 +1,31 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-microsoft-onedrive:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_microsoft_onedrive/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_trace_message_on_failure: false + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg b/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg new file mode 100644 index 000000000000..eaf2ddb94622 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg @@ -0,0 +1 @@ +OneDrive_64x \ No newline at end of file diff --git a/octavia-cli/unit_tests/test__import/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/__init__.py similarity index 100% rename from octavia-cli/unit_tests/test__import/__init__.py rename to airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/__init__.py diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..1c86c5e27bee --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json @@ -0,0 +1,30 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_csv" + }, + "stream_state": { + "_ab_source_file_last_modified": "2023-12-23T06:49:25.000000Z_Test_folder_2/TestFileOneDrive.csv", + "history": { + "Test_folder_2/TestFileOneDrive.csv": "2023-12-23T06:49:25.000000Z" + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_unstructured" + }, + "stream_state": { + "_ab_source_file_last_modified": "2023-12-23T06:49:25.000000Z_simple_pdf_file.pdf", + "history": { + "simple_pdf_file.pdf": "2023-12-23T06:49:25.000000Z" + } + } + } + } +] diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..1bda267ec914 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json @@ -0,0 +1,26 @@ +{ + "streams": [ + { + "stream": { + "name": "test_csv", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_unstructured", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json new file mode 100644 index 000000000000..68af1008c0ac --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json @@ -0,0 +1,21 @@ +{ + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "drive_name": "drive_name", + "folder_path": "folder_path", + "streams": [ + { + "name": "test_stream", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json new file mode 100644 index 000000000000..24ee82201f5a --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json @@ -0,0 +1,455 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/microsoft-onedrive", + "connectionSpecification": { + "title": "Microsoft OneDrive Source Spec", + "description": "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source Specification.\nThis class combines the authentication details with additional configuration for the OneDrive API.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "default": ["**"], + "order": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "type": "string" + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_file_types": { + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "default": true, + "always_show": true, + "type": "boolean" + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "credentials": { + "title": "Authentication", + "description": "Credentials for connecting to the One Drive API", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "Authenticate via Microsoft (OAuth)", + "description": "OAuthCredentials class to hold authentication details for Microsoft OAuth authentication.\nThis class uses pydantic for data validation and settings management.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Client", + "const": "Client", + "enum": ["Client"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft OneDrive user", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "Refresh Token of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "client_id", + "client_secret", + "refresh_token" + ] + }, + { + "title": "Service Key Authentication", + "description": "ServiceCredentials class for service key authentication.\nThis class is structured similarly to OAuthCredentials but for a different authentication method.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Service", + "const": "Service", + "enum": ["Service"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft OneDrive user", + "airbyte_secret": true, + "type": "string" + }, + "user_principal_name": { + "title": "User Principal Name", + "description": "Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "user_principal_name", + "client_id", + "client_secret" + ] + } + ] + }, + "drive_name": { + "title": "Drive Name", + "description": "Name of the Microsoft OneDrive drive where the file(s) exist.", + "default": "OneDrive", + "order": 2, + "type": "string" + }, + "folder_path": { + "title": "Folder Path", + "description": "Path to folder of the Microsoft OneDrive drive where the file(s) exist.", + "order": 3, + "type": "string" + } + }, + "required": ["streams", "credentials", "folder_path"] + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py new file mode 100644 index 000000000000..c9280337d108 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_microsoft_onedrive.run import run + +if __name__ == "__main__": + run() diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml new file mode 100644 index 000000000000..f8833c9d4d99 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -0,0 +1,34 @@ +data: + ab_internal: + ql: 200 + sl: 100 + allowedHosts: + hosts: + - graph.microsoft.com + - login.microsoftonline.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-microsoft-onedrive + registries: + oss: + enabled: true + cloud: + enabled: false # We need to either implement OAuth for cloud or remove OAuth from the config for cloud + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 01d1c685-fd4a-4837-8f4c-93fe5a0d2188 + dockerImageTag: 0.1.6 + dockerRepository: airbyte/source-microsoft-onedrive + githubIssueLabel: source-microsoft-onedrive + icon: microsoft-onedrive.svg + license: MIT + name: Microsoft OneDrive + supportLevel: community + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-onedrive + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chargebee/requirements.txt b/airbyte-integrations/connectors/source-microsoft-onedrive/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-chargebee/requirements.txt rename to airbyte-integrations/connectors/source-microsoft-onedrive/requirements.txt diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py new file mode 100644 index 000000000000..526e39f5c813 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]>=0.61.0", + "msal~=1.25.0", + "Office365-REST-Python-Client~=2.5.2", + "smart-open~=6.4.0", +] + +TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", + "pytest~=6.1", + "requests-mock~=1.11.0", +] + +setup( + entry_points={ + "console_scripts": [ + "source-microsoft-onedrive=source_microsoft_onedrive.run:run", + ], + }, + name="source_microsoft_onedrive", + description="Source implementation for Microsoft OneDrive.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py new file mode 100644 index 000000000000..c8e3b2178fbb --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMicrosoftOneDrive + +__all__ = ["SourceMicrosoftOneDrive"] diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py new file mode 100644 index 000000000000..f083b8c70abc --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_microsoft_onedrive import SourceMicrosoftOneDrive + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + source = SourceMicrosoftOneDrive( + SourceMicrosoftOneDrive.read_catalog(catalog_path) if catalog_path else None, + SourceMicrosoftOneDrive.read_config(config_path) if config_path else None, + SourceMicrosoftOneDrive.read_state(state_path) if state_path else None, + ) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py new file mode 100644 index 000000000000..f317a0990e84 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import Any, Mapping, Optional + +from airbyte_cdk.models import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.source import TState +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec +from source_microsoft_onedrive.stream_reader import SourceMicrosoftOneDriveStreamReader + + +class SourceMicrosoftOneDrive(FileBasedSource): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState]): + super().__init__( + stream_reader=SourceMicrosoftOneDriveStreamReader(), + spec_class=SourceMicrosoftOneDriveSpec, + catalog=catalog, + config=config, + state=state, + cursor_cls=DefaultFileBasedCursor, + ) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + """ + Returns the specification describing what fields can be configured by a user when setting up a file-based source. + """ + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=self.spec_class.schema(), + advanced_auth=AdvancedAuth( + auth_flow_type="oauth2.0", + predicate_key=["credentials", "auth_type"], + predicate_value="Client", + oauth_config_specification=OAuthConfigSpecification( + complete_oauth_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}}, + }, + complete_oauth_server_input_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + complete_oauth_server_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + oauth_user_input_from_connector_config_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"tenant_id": {"type": "string", "path_in_connector_config": ["credentials", "tenant_id"]}}, + }, + ), + ), + ) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py new file mode 100644 index 000000000000..180993a685c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict, Literal, Optional, Union + +import dpath.util +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from pydantic import BaseModel, Field + + +class OAuthCredentials(BaseModel): + """ + OAuthCredentials class to hold authentication details for Microsoft OAuth authentication. + This class uses pydantic for data validation and settings management. + """ + + class Config: + title = "Authenticate via Microsoft (OAuth)" + + # Fields for the OAuth authentication, including tenant_id, client_id, client_secret, and refresh_token + auth_type: Literal["Client"] = Field("Client", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft OneDrive user", airbyte_secret=True) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + refresh_token: str = Field( + title="Refresh Token", + description="Refresh Token of your Microsoft developer application", + airbyte_secret=True, + ) + + +class ServiceCredentials(BaseModel): + """ + ServiceCredentials class for service key authentication. + This class is structured similarly to OAuthCredentials but for a different authentication method. + """ + + class Config: + title = "Service Key Authentication" + + # Fields for the Service authentication, similar to OAuthCredentials + auth_type: Literal["Service"] = Field("Service", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft OneDrive user", airbyte_secret=True) + user_principal_name: str = Field( + title="User Principal Name", + description="Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + airbyte_secret=True, + ) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + + +class SourceMicrosoftOneDriveSpec(AbstractFileBasedSpec, BaseModel): + """ + SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source Specification. + This class combines the authentication details with additional configuration for the OneDrive API. + """ + + class Config: + title = "Microsoft OneDrive Source Spec" + + # Union type for credentials, allowing for either OAuth or Service Key authentication + credentials: Union[OAuthCredentials, ServiceCredentials] = Field( + title="Authentication", + description="Credentials for connecting to the One Drive API", + discriminator="auth_type", + type="object", + order=0, + ) + + drive_name: Optional[str] = Field( + title="Drive Name", description="Name of the Microsoft OneDrive drive where the file(s) exist.", default="OneDrive", order=2 + ) + folder_path: str = Field( + title="Folder Path", description="Path to folder of the Microsoft OneDrive drive where the file(s) exist.", order=3 + ) + + @classmethod + def documentation_url(cls) -> str: + """Provides the URL to the documentation for this specific source.""" + return "https://docs.airbyte.com/integrations/sources/one-drive" + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the schema mapping for configuration fields. + It also cleans up the schema by removing legacy settings and discriminators. + """ + schema = super().schema(*args, **kwargs) + + # Remove legacy settings related to streams + dpath.util.delete(schema, "properties/streams/items/properties/legacy_prefix") + dpath.util.delete(schema, "properties/streams/items/properties/format/oneOf/*/properties/inference_type") + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py new file mode 100644 index 000000000000..1fbd5d665bb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from functools import lru_cache +from io import IOBase +from typing import Iterable, List, Optional + +import smart_open +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from msal import ConfidentialClientApplication +from msal.exceptions import MsalServiceError +from office365.graph_client import GraphClient +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec + + +class MicrosoftOneDriveRemoteFile(RemoteFile): + download_url: str + + +class SourceMicrosoftOneDriveClient: + """ + Client to interact with Microsoft OneDrive. + """ + + def __init__(self, config: SourceMicrosoftOneDriveSpec): + self.config = config + self._client = None + + @property + @lru_cache(maxsize=None) + def msal_app(self): + """Returns an MSAL app instance for authentication.""" + return ConfidentialClientApplication( + self.config.credentials.client_id, + authority=f"https://login.microsoftonline.com/{self.config.credentials.tenant_id}", + client_credential=self.config.credentials.client_secret, + ) + + @property + def client(self): + """Initializes and returns a GraphClient instance.""" + if not self.config: + raise ValueError("Configuration is missing; cannot create the Office365 graph client.") + if not self._client: + self._client = GraphClient(self._get_access_token) + return self._client + + def _get_access_token(self): + """Retrieves an access token for OneDrive access.""" + scope = ["https://graph.microsoft.com/.default"] + refresh_token = self.config.credentials.refresh_token if hasattr(self.config.credentials, "refresh_token") else None + + if refresh_token: + result = self.msal_app.acquire_token_by_refresh_token(refresh_token, scopes=scope) + else: + result = self.msal_app.acquire_token_for_client(scopes=scope) + + if "access_token" not in result: + error_description = result.get("error_description", "No error description provided.") + raise MsalServiceError(error=result.get("error"), error_description=error_description) + + return result + + +class SourceMicrosoftOneDriveStreamReader(AbstractFileBasedStreamReader): + """ + A stream reader for Microsoft OneDrive. Handles file enumeration and reading from OneDrive. + """ + + ROOT_PATH = [".", "/"] + + def __init__(self): + super().__init__() + + @property + def config(self) -> SourceMicrosoftOneDriveSpec: + return self._config + + @property + def one_drive_client(self) -> SourceMicrosoftOneDriveSpec: + return SourceMicrosoftOneDriveClient(self._config).client + + @config.setter + def config(self, value: SourceMicrosoftOneDriveSpec): + """ + The FileBasedSource reads and parses configuration from a file, then sets this configuration in its StreamReader. While it only + uses keys from its abstract configuration, concrete StreamReader implementations may need additional keys for third-party + authentication. Therefore, subclasses of AbstractFileBasedStreamReader should verify that the value in their config setter + matches the expected config type for their StreamReader. + """ + assert isinstance(value, SourceMicrosoftOneDriveSpec) + self._config = value + + def list_directories_and_files(self, root_folder, path=None): + """Enumerates folders and files starting from a root folder.""" + drive_items = root_folder.children.get().execute_query() + found_items = [] + for item in drive_items: + item_path = path + "/" + item.name if path else item.name + if item.is_file: + found_items.append((item, item_path)) + else: + found_items.extend(self.list_directories_and_files(item, item_path)) + return found_items + + def get_files_by_drive_name(self, drives, drive_name, folder_path): + """Yields files from the specified drive.""" + path_levels = [level for level in folder_path.split("/") if level] + folder_path = "/".join(path_levels) + + for drive in drives: + is_onedrive = drive.drive_type in ["personal", "business"] + if drive.name == drive_name and is_onedrive: + folder = drive.root if folder_path in self.ROOT_PATH else drive.root.get_by_path(folder_path).get().execute_query() + yield from self.list_directories_and_files(folder) + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Retrieve all files matching the specified glob patterns in OneDrive. + """ + drives = self.one_drive_client.drives.get().execute_query() + + if self.config.credentials.auth_type == "Client": + my_drive = self.one_drive_client.me.drive.get().execute_query() + else: + my_drive = ( + self.one_drive_client.users.get_by_principal_name(self.config.credentials.user_principal_name).drive.get().execute_query() + ) + + drives.add_child(my_drive) + + files = self.get_files_by_drive_name(drives, self.config.drive_name, self.config.folder_path) + + try: + first_file, path = next(files) + + yield from self.filter_files_by_globs_and_start_date( + [ + MicrosoftOneDriveRemoteFile( + uri=path, + download_url=first_file.properties["@microsoft.graph.downloadUrl"], + last_modified=first_file.properties["lastModifiedDateTime"], + ) + ], + globs, + ) + + except StopIteration as e: + raise AirbyteTracedException( + internal_message=str(e), + message=f"Drive '{self.config.drive_name}' is empty or does not exist.", + failure_type=FailureType.config_error, + exception=e, + ) + + yield from self.filter_files_by_globs_and_start_date( + [ + MicrosoftOneDriveRemoteFile( + uri=path, + download_url=file.properties["@microsoft.graph.downloadUrl"], + last_modified=file.properties["lastModifiedDateTime"], + ) + for file, path in files + ], + globs, + ) + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + try: + return smart_open.open(file.download_url, mode=mode.value, encoding=encoding) + except Exception as e: + logger.exception(f"Error opening file {file.uri}: {e}") diff --git a/octavia-cli/unit_tests/test_get/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/__init__.py similarity index 100% rename from octavia-cli/unit_tests/test_get/__init__.py rename to airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/__init__.py diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py new file mode 100644 index 000000000000..f610ad67a646 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock, patch + +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec +from source_microsoft_onedrive.stream_reader import FileReadMode, SourceMicrosoftOneDriveClient, SourceMicrosoftOneDriveStreamReader + + +def create_mock_drive_item(is_file, name): + """Helper function to create a mock drive item.""" + mock_item = Mock() + mock_item.is_file = is_file + mock_item.name = name + return mock_item + + +@patch("smart_open.open") +def test_open_file(mock_smart_open): + """Test the open_file method in SourceMicrosoftOneDriveStreamReader.""" + mock_file = Mock(download_url="http://example.com/file.txt") + mock_logger = Mock() + + stream_reader = SourceMicrosoftOneDriveStreamReader() + stream_reader._config = Mock() # Assuming _config is required + + with stream_reader.open_file(mock_file, FileReadMode.READ, "utf-8", mock_logger) as result: + pass + + mock_smart_open.assert_called_once_with(mock_file.download_url, mode='r', encoding='utf-8') + assert result is not None + + +def test_microsoft_onedrive_client_initialization(requests_mock): + """Test the initialization of SourceMicrosoftOneDriveClient.""" + config = { + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "drive_name": "drive_name", + "folder_path": "folder_path", + "streams": [{"name": "test_stream", "globs": ["*.csv"], "validation_policy": "Emit Record", "format": {"filetype": "csv"}}] + } + + authority_url = 'https://login.microsoftonline.com/tenant_id/v2.0/.well-known/openid-configuration' + mock_response = {'authorization_endpoint': 'https://login.microsoftonline.com/tenant_id/oauth2/v2.0/authorize', 'token_endpoint': 'https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token'} + requests_mock.get(authority_url, json=mock_response, status_code=200) + + client = SourceMicrosoftOneDriveClient(SourceMicrosoftOneDriveSpec(**config)) + + assert client.config == SourceMicrosoftOneDriveSpec(**config) + assert client.msal_app is not None + + +@patch("source_microsoft_onedrive.stream_reader.SourceMicrosoftOneDriveStreamReader.list_directories_and_files") +def test_list_directories_and_files(mock_list_directories_and_files): + """Test the list_directories_and_files method in SourceMicrosoftOneDriveStreamReader.""" + mock_root_folder = create_mock_drive_item(False, "root") + mock_child_file = create_mock_drive_item(True, "file1.txt") + mock_child_folder = create_mock_drive_item(False, "folder1") + mock_child_folder.children.get().execute_query.return_value = [mock_child_file] + mock_root_folder.children.get().execute_query.return_value = [mock_child_folder, mock_child_file] + + mock_list_directories_and_files.return_value = [mock_child_folder, mock_child_file] + + stream_reader = SourceMicrosoftOneDriveStreamReader() + result = stream_reader.list_directories_and_files(mock_root_folder) + + assert len(result) == 2 + assert result[0].name == "folder1" + assert result[1].name == "file1.txt" + + +@patch("source_microsoft_onedrive.stream_reader.SourceMicrosoftOneDriveStreamReader.list_directories_and_files") +def test_get_files_by_drive_name(mock_list_directories_and_files): + # Helper function usage + mock_drive = Mock() + mock_drive.name = "testDrive" + mock_drive.drive_type = "business" + mock_drive.root.get_by_path.return_value.get().execute_query.return_value = create_mock_drive_item(is_file=False, name="root") + + # Mock files + mock_file = create_mock_drive_item(is_file=True, name="testFile.txt") + mock_list_directories_and_files.return_value = [mock_file] + + # Create stream reader instance + stream_reader = SourceMicrosoftOneDriveStreamReader() + stream_reader._config = Mock() + + # Call the method + files = list(stream_reader.get_files_by_drive_name([mock_drive], "testDrive", "/test/path")) + + # Assertions + assert len(files) == 1 + assert files[0].name == "testFile.txt" diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/.coveragerc b/airbyte-integrations/connectors/source-microsoft-sharepoint/.coveragerc new file mode 100644 index 000000000000..6ab3ced7f624 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + source_microsoft_sharepoint/spec.py + source_microsoft_sharepoint/source.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/README.md b/airbyte-integrations/connectors/source-microsoft-sharepoint/README.md new file mode 100644 index 000000000000..fa4d35e78654 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/README.md @@ -0,0 +1,166 @@ +# Microsoft SharePoint Source + +This is the repository for the Microsoft SharePoint source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/microsoft-sharepoint). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/microsoft-sharepoint) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_microsoft_sharepoint/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source microsoft-sharepoint test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-microsoft-sharepoint build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-microsoft-sharepoint:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-microsoft-sharepoint:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-microsoft-sharepoint:dev . +# Running the spec command against your patched connector +docker run airbyte/source-microsoft-sharepoint:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-microsoft-sharepoint:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-sharepoint:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-sharepoint:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-microsoft-sharepoint:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-microsoft-sharepoint test +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-microsoft-sharepoint test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/microsoft-sharepoint.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/acceptance-test-config.yml b/airbyte-integrations/connectors/source-microsoft-sharepoint/acceptance-test-config.yml new file mode 100644 index 000000000000..01a7a8bfbf9d --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/acceptance-test-config.yml @@ -0,0 +1,35 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-microsoft-sharepoint:dev +test_strictness_level: high +acceptance_tests: + spec: + tests: + - spec_path: "source_microsoft_sharepoint/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + timeout_seconds: 90 # the check is slow because list of files is fetched for every of 5 streams + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + expect_trace_message_on_failure: false + expect_records: + path: integration_tests/expected_records.jsonl + exact_order: true + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/icon.svg b/airbyte-integrations/connectors/source-microsoft-sharepoint/icon.svg new file mode 100644 index 000000000000..e8daae912216 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/icon.svg @@ -0,0 +1 @@ +OfficeCore10_32x_24x_20x_16x_01-22-2019 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/__init__.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..ae378266f263 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/abnormal_state.json @@ -0,0 +1,74 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_csv" + }, + "stream_state": { + "history": { + "Test_folder/TestFile.csv": "2130-11-17T13:52:35.000000Z", + "Test_folder/Test_folder_2/TestFileSharePoint.csv": "2130-12-15T17:34:08.000000Z", + "Test_folder/simple_test.csv": "2130-01-16T12:45:20.000000Z" + }, + "_ab_source_file_last_modified": "2130-01-16T12:45:20.000000Z_Test_folder/simple_test.csv" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_unstructured" + }, + "stream_state": { + "_ab_source_file_last_modified": "2130-12-23T16:47:21.000000Z_Test_folder/Test_foler_2_1/simple_pdf_file.pdf", + "history": { + "Test_folder/Test_foler_2_1/simple_pdf_file.pdf": "2130-12-23T16:47:21.000000Z" + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_csv_gzip" + }, + "stream_state": { + "history": { + "Test_folder/simple_test.csv.gz": "2130-01-18T11:24:38.000000Z" + }, + "_ab_source_file_last_modified": "2130-01-18T11:24:38.000000Z_Test_folder/simple_test.csv.gz" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_parquet" + }, + "stream_state": { + "history": { + "Test_folder/test_parquet/simple_test.parquet": "2130-01-17T11:47:39.000000Z" + }, + "_ab_source_file_last_modified": "2130-01-17T11:47:39.000000Z_Test_folder/test_parquet/simple_test.parquet" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_avro" + }, + "stream_state": { + "history": { + "Test_folder/test_avro/test_sample.avro": "2130-01-17T11:46:55.000000Z" + }, + "_ab_source_file_last_modified": "2130-01-17T11:46:55.000000Z_Test_folder/test_avro/test_sample.avro" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py new file mode 100644 index 000000000000..82823254d266 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..09e4d159a2db --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/configured_catalog.json @@ -0,0 +1,59 @@ +{ + "streams": [ + { + "stream": { + "name": "test_csv_gzip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_avro", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_parquet", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_csv", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_unstructured", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..b6d5ccdbd7c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/expected_records.jsonl @@ -0,0 +1,13 @@ +{"stream": "test_avro", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2024-01-17T11:46:55.000000Z", "_ab_source_file_url": "Test_folder/test_avro/test_sample.avro"}, "emitted_at": 1706099187061} +{"stream": "test_avro", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2024-01-17T11:46:55.000000Z", "_ab_source_file_url": "Test_folder/test_avro/test_sample.avro"}, "emitted_at": 1706099187063} +{"stream": "test_avro", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2024-01-17T11:46:55.000000Z", "_ab_source_file_url": "Test_folder/test_avro/test_sample.avro"}, "emitted_at": 1706099187063} +{"stream": "test_csv", "data": {"\ufeff\"{\"\"type\"\": \"\"RECORD\"\"\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"200818142097572\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"200818142097572\"\"\";\" \"\"impressions\"\": \"\"85\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - PROD - [Cyclone Dust Collector] - Competitors - Video\"\"}}\";\" \"\"emitted_at\"\": 1691500775298}}\";;;": "{\"type\": \"RECORD\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"160532120248153\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"160532120248153\"\"\";\" \"\"impressions\"\": \"\"4695\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - ATV - Remarketing - [Cyclone Dust Collector] - 7\";14;30;\" 60 Days\"\"}}\";\" \"\"emitted_at\"\": 1691500775316}}\"", "_ab_source_file_last_modified": "2023-11-17T13:52:35.000000Z", "_ab_source_file_url": "Test_folder/TestFile.csv"}, "emitted_at": 1706099256586} +{"stream": "test_csv", "data": {"\ufeff\"{\"\"type\"\": \"\"RECORD\"\"\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"200818142097572\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"200818142097572\"\"\";\" \"\"impressions\"\": \"\"85\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - PROD - [Cyclone Dust Collector] - Competitors - Video\"\"}}\";\" \"\"emitted_at\"\": 1691500775298}}\";;;": "{\"type\": \"RECORD\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"196774177063693\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"196774177063693\"\"\";\" \"\"impressions\"\": \"\"0\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - ATV - Remarketing - [Cyclone Dust Collector] - 30 Days - Video\"\"}}\";\" \"\"emitted_at\"\": 1691500775316}}\";;;", "_ab_source_file_last_modified": "2023-11-17T13:52:35.000000Z", "_ab_source_file_url": "Test_folder/TestFile.csv"}, "emitted_at": 1706099256588} +{"stream": "test_csv", "data": {"\ufeff\"{\"\"type\"\": \"\"RECORD\"\"\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"200818142097572\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"200818142097572\"\"\";\" \"\"impressions\"\": \"\"85\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - PROD - [Cyclone Dust Collector] - Competitors - Video\"\"}}\";\" \"\"emitted_at\"\": 1691500775298}}\";;;": "{\"type\": \"RECORD\";\" \"\"record\"\": {\"\"stream\"\": \"\"sponsored_display_report_stream\"\"\";\" \"\"data\"\": {\"\"profileId\"\": 1560175474359565\";\" \"\"recordType\"\": \"\"campaigns\"\"\";\" \"\"reportDate\"\": \"\"20230701\"\"\";\" \"\"recordId\"\": \"\"57815899765268\"\"\";\" \"\"metric\"\": {\"\"campaignId\"\": \"\"57815899765268\"\"\";\" \"\"impressions\"\": \"\"0\"\"\";\" \"\"campaignName\"\": \"\"**LP - SD - ATV - Similar To - [Cyclone Dust Collector] - 30 Days - Video\"\"}}\";\" \"\"emitted_at\"\": 1691500775317}}\";;;", "_ab_source_file_last_modified": "2023-11-17T13:52:35.000000Z", "_ab_source_file_url": "Test_folder/TestFile.csv"}, "emitted_at": 1706099256588} +{"stream": "test_csv_gzip", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2024-01-18T11:24:38.000000Z", "_ab_source_file_url": "Test_folder/simple_test.csv.gz"}, "emitted_at": 1706099153362} +{"stream": "test_csv_gzip", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2024-01-18T11:24:38.000000Z", "_ab_source_file_url": "Test_folder/simple_test.csv.gz"}, "emitted_at": 1706099153363} +{"stream": "test_csv_gzip", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2024-01-18T11:24:38.000000Z", "_ab_source_file_url": "Test_folder/simple_test.csv.gz"}, "emitted_at": 1706099153363} +{"stream": "test_parquet", "data": {"number": 1.0, "name": "foo", "flag": true, "delta": -1.0, "_ab_source_file_last_modified": "2024-01-17T11:47:39.000000Z", "_ab_source_file_url": "Test_folder/test_parquet/simple_test.parquet"}, "emitted_at": 1706099224674} +{"stream": "test_parquet", "data": {"number": 2.0, "name": null, "flag": false, "delta": 2.5, "_ab_source_file_last_modified": "2024-01-17T11:47:39.000000Z", "_ab_source_file_url": "Test_folder/test_parquet/simple_test.parquet"}, "emitted_at": 1706099224675} +{"stream": "test_parquet", "data": {"number": 3.0, "name": "bar", "flag": null, "delta": 0.1, "_ab_source_file_last_modified": "2024-01-17T11:47:39.000000Z", "_ab_source_file_url": "Test_folder/test_parquet/simple_test.parquet"}, "emitted_at": 1706099224675} +{"stream": "test_unstructured", "data": {"content": "# A Simple PDF File\n\nThis is a small demonstration .pdf file -\n\njust for use in the Virtual Mechanics tutorials. More text. And more text. And more text. And more text. And more text.\n\nAnd more text. And more text. And more text. And more text. And more text. And more text. Boring, zzzzz. And more text. And more text. And more text. And more text. And more text. And more text. And more text. And more text. And more text.\n\nAnd more text. And more text. And more text. And more text. And more text. And more text. And more text. Even more. Continued on page 2 ...\n\n# Simple PDF File 2\n\n...continued from page 1. Yet more text. And more text. And more text. And more text. And more text. And more text. And more text. And more text. Oh, how boring typing this stuff. But not as boring as watching paint dry. And more text. And more text. And more text. And more text. Boring. More, a little more text. The end, and just as well.", "document_key": "Test_folder/Test_foler_2_1/simple_pdf_file.pdf", "_ab_source_file_parse_error": null, "_ab_source_file_last_modified": "2023-12-15T16:47:21.000000Z", "_ab_source_file_url": "Test_folder/Test_foler_2_1/simple_pdf_file.pdf"}, "emitted_at": 1706099290771} diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/invalid_config.json new file mode 100644 index 000000000000..c0c922494f6c --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/invalid_config.json @@ -0,0 +1,20 @@ +{ + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "folder_path": "folder_path", + "streams": [ + { + "name": "test_stream", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/sample_config.json new file mode 100644 index 000000000000..6cab50fdd084 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/sample_config.json @@ -0,0 +1,86 @@ +{ + "credentials": { + "auth_type": "Client", + "client_id": "cleint_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "folder_path": ".", + "streams": [ + { + "name": "test_parquet", + "globs": ["**/*.parquet"], + "validation_policy": "Emit Record", + "format": { + "filetype": "parquet" + } + }, + { + "name": "test_avro", + "globs": ["**/*.avro"], + "validation_policy": "Emit Record", + "format": { + "filetype": "avro" + } + }, + { + "name": "test_csv_gzip", + "file_type": "csv", + "globs": ["**/simple_test.csv.gz"], + "legacy_prefix": "", + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "delimiter": ",", + "quote_char": "\"", + "double_quote": true, + "null_values": [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null" + ], + "true_values": ["1", "True", "TRUE", "true"], + "false_values": ["0", "False", "FALSE", "false"], + "inference_type": "Primitive Types Only", + "strings_can_be_null": false, + "encoding": "utf8", + "header_definition": { + "header_definition_type": "From CSV" + } + } + }, + { + "name": "test_csv", + "globs": ["**/*.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + }, + { + "name": "test_unstructured", + "globs": ["**/*.pdf"], + "format": { + "filetype": "unstructured" + }, + "schemaless": false, + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3 + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/spec.json b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/spec.json new file mode 100644 index 000000000000..6018f9bfbdd4 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/spec.json @@ -0,0 +1,483 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/one-drive", + "connectionSpecification": { + "title": "Microsoft SharePoint Source Spec", + "description": "SourceMicrosoftSharePointSpec class for Microsoft SharePoint Source Specification.\nThis class combines the authentication details with additional configuration for the SharePoint API.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "default": ["**"], + "order": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "airbyte_hidden": true, + "type": "string" + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_files": { + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "default": true, + "always_show": true, + "type": "boolean" + }, + "strategy": { + "title": "Parsing Strategy", + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf", + "default": "auto", + "always_show": true, + "order": 0, + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "type": "string" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "credentials": { + "title": "Authentication", + "description": "Credentials for connecting to the One Drive API", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "Authenticate via Microsoft (OAuth)", + "description": "OAuthCredentials class to hold authentication details for Microsoft OAuth authentication.\nThis class uses pydantic for data validation and settings management.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Client", + "const": "Client", + "enum": ["Client"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft SharePoint user", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "Refresh Token of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "client_id", + "client_secret", + "refresh_token" + ] + }, + { + "title": "Service Key Authentication", + "description": "ServiceCredentials class for service key authentication.\nThis class is structured similarly to OAuthCredentials but for a different authentication method.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Service", + "const": "Service", + "enum": ["Service"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft SharePoint user", + "airbyte_secret": true, + "type": "string" + }, + "user_principal_name": { + "title": "User Principal Name", + "description": "Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "user_principal_name", + "client_id", + "client_secret" + ] + } + ] + }, + "folder_path": { + "title": "Folder Path", + "description": "Path to folder of the Microsoft SharePoint drive where the file(s) exist.", + "order": 3, + "type": "string" + } + }, + "required": ["streams", "credentials", "folder_path"] + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py new file mode 100644 index 000000000000..d9b4eee715dc --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_microsoft_sharepoint import SourceMicrosoftSharePoint + +if __name__ == "__main__": + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + source = SourceMicrosoftSharePoint(catalog_path) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-sharepoint/metadata.yaml new file mode 100644 index 000000000000..fe6b0d8b8763 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/metadata.yaml @@ -0,0 +1,31 @@ +data: + ab_internal: + ql: 300 + sl: 200 + allowedHosts: + hosts: + - graph.microsoft.com + - login.microsoftonline.com + registries: + oss: + enabled: true + cloud: + enabled: true + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 59353119-f0f2-4e5a-a8ba-15d887bc34f6 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-microsoft-sharepoint + githubIssueLabel: source-microsoft-sharepoint + icon: microsoft-sharepoint.svg + license: MIT + name: Microsoft SharePoint + supportLevel: community + releaseStage: alpha + releaseDate: 2024-02-02 + documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-sharepoint + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-github/requirements.txt b/airbyte-integrations/connectors/source-microsoft-sharepoint/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-github/requirements.txt rename to airbyte-integrations/connectors/source-microsoft-sharepoint/requirements.txt diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/setup.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/setup.py new file mode 100644 index 000000000000..ce83f37b5940 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/setup.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]==0.59.2", + "msal~=1.25.0", + "Office365-REST-Python-Client~=2.5.2", + "smart-open~=6.4.0", +] + +TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", + "pytest~=6.1", + "requests-mock~=1.11.0", +] + +setup( + name="source_microsoft_sharepoint", + description="Source implementation for Microsoft SharePoint.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/__init__.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/__init__.py new file mode 100644 index 000000000000..3e882028ad8d --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMicrosoftSharePoint + +__all__ = ["SourceMicrosoftSharePoint"] diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/source.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/source.py new file mode 100644 index 000000000000..5515c22e2398 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/source.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any + +from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec +from source_microsoft_sharepoint.stream_reader import SourceMicrosoftSharePointStreamReader + + +class SourceMicrosoftSharePoint(FileBasedSource): + def __init__(self, catalog_path: str): + super().__init__( + stream_reader=SourceMicrosoftSharePointStreamReader(), + spec_class=SourceMicrosoftSharePointSpec, + catalog_path=catalog_path, + cursor_cls=DefaultFileBasedCursor, + ) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + """ + Returns the specification describing what fields can be configured by a user when setting up a file-based source. + """ + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=self.spec_class.schema(), + advanced_auth=AdvancedAuth( + auth_flow_type="oauth2.0", + predicate_key=["credentials", "auth_type"], + predicate_value="Client", + oauth_config_specification=OAuthConfigSpecification( + complete_oauth_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}}, + }, + complete_oauth_server_input_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + complete_oauth_server_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + oauth_user_input_from_connector_config_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"tenant_id": {"type": "string", "path_in_connector_config": ["credentials", "tenant_id"]}}, + }, + ), + ), + ) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py new file mode 100644 index 000000000000..45434d78b155 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict, Literal, Optional, Union + +import dpath.util +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from pydantic import BaseModel, Field + + +class OAuthCredentials(BaseModel): + """ + OAuthCredentials class to hold authentication details for Microsoft OAuth authentication. + This class uses pydantic for data validation and settings management. + """ + + class Config: + title = "Authenticate via Microsoft (OAuth)" + + # Fields for the OAuth authentication, including tenant_id, client_id, client_secret, and refresh_token + auth_type: Literal["Client"] = Field("Client", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft SharePoint user", airbyte_secret=True) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + refresh_token: str = Field( + title="Refresh Token", + description="Refresh Token of your Microsoft developer application", + airbyte_secret=True, + ) + + +class ServiceCredentials(BaseModel): + """ + ServiceCredentials class for service key authentication. + This class is structured similarly to OAuthCredentials but for a different authentication method. + """ + + class Config: + title = "Service Key Authentication" + + # Fields for the Service authentication, similar to OAuthCredentials + auth_type: Literal["Service"] = Field("Service", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft SharePoint user", airbyte_secret=True) + user_principal_name: str = Field( + title="User Principal Name", + description="Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + airbyte_secret=True, + ) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + + +class SourceMicrosoftSharePointSpec(AbstractFileBasedSpec, BaseModel): + """ + SourceMicrosoftSharePointSpec class for Microsoft SharePoint Source Specification. + This class combines the authentication details with additional configuration for the SharePoint API. + """ + + class Config: + title = "Microsoft SharePoint Source Spec" + + # Union type for credentials, allowing for either OAuth or Service Key authentication + credentials: Union[OAuthCredentials, ServiceCredentials] = Field( + title="Authentication", + description="Credentials for connecting to the One Drive API", + discriminator="auth_type", + type="object", + order=0, + ) + + folder_path: str = Field( + title="Folder Path", description="Path to folder of the Microsoft SharePoint drive where the file(s) exist.", order=3 + ) + + @classmethod + def documentation_url(cls) -> str: + """Provides the URL to the documentation for this specific source.""" + return "https://docs.airbyte.com/integrations/sources/microsoft-sharepoint" + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the schema mapping for configuration fields. + It also cleans up the schema by removing legacy settings and discriminators. + """ + schema = super().schema(*args, **kwargs) + + # Remove legacy settings related to streams + dpath.util.delete(schema, "properties/streams/items/properties/legacy_prefix") + dpath.util.delete(schema, "properties/streams/items/properties/format/oneOf/*/properties/inference_type") + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py new file mode 100644 index 000000000000..97804668cd30 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from functools import lru_cache +from io import IOBase +from typing import Iterable, List, Optional + +import smart_open +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from msal import ConfidentialClientApplication +from msal.exceptions import MsalServiceError +from office365.graph_client import GraphClient +from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec + +from .utils import MicrosoftSharePointRemoteFile, execute_query_with_retry, filter_http_urls + + +class SourceMicrosoftSharePointClient: + """ + Client to interact with Microsoft SharePoint. + """ + + def __init__(self, config: SourceMicrosoftSharePointSpec): + self.config = config + self._client = None + self._msal_app = ConfidentialClientApplication( + self.config.credentials.client_id, + authority=f"https://login.microsoftonline.com/{self.config.credentials.tenant_id}", + client_credential=self.config.credentials.client_secret, + ) + + @property + def client(self): + """Initializes and returns a GraphClient instance.""" + if not self.config: + raise ValueError("Configuration is missing; cannot create the Office365 graph client.") + if not self._client: + self._client = GraphClient(self._get_access_token) + return self._client + + def _get_access_token(self): + """Retrieves an access token for SharePoint access.""" + scope = ["https://graph.microsoft.com/.default"] + refresh_token = self.config.credentials.refresh_token if hasattr(self.config.credentials, "refresh_token") else None + + if refresh_token: + result = self._msal_app.acquire_token_by_refresh_token(refresh_token, scopes=scope) + else: + result = self._msal_app.acquire_token_for_client(scopes=scope) + + if "access_token" not in result: + error_description = result.get("error_description", "No error description provided.") + message = f"Failed to acquire access token. Error: {result.get('error')}. Error description: {error_description}." + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + return result + + +class SourceMicrosoftSharePointStreamReader(AbstractFileBasedStreamReader): + """ + A stream reader for Microsoft SharePoint. Handles file enumeration and reading from SharePoint. + """ + + ROOT_PATH = [".", "/"] + + def __init__(self): + super().__init__() + self._one_drive_client = None + + @property + def config(self) -> SourceMicrosoftSharePointSpec: + return self._config + + @property + def one_drive_client(self) -> SourceMicrosoftSharePointSpec: + if self._one_drive_client is None: + self._one_drive_client = SourceMicrosoftSharePointClient(self._config).client + return self._one_drive_client + + @config.setter + def config(self, value: SourceMicrosoftSharePointSpec): + """ + The FileBasedSource reads and parses configuration from a file, then sets this configuration in its StreamReader. While it only + uses keys from its abstract configuration, concrete StreamReader implementations may need additional keys for third-party + authentication. Therefore, subclasses of AbstractFileBasedStreamReader should verify that the value in their config setter + matches the expected config type for their StreamReader. + """ + assert isinstance(value, SourceMicrosoftSharePointSpec) + self._config = value + + def _list_directories_and_files(self, root_folder, path=None): + """Enumerates folders and files starting from a root folder.""" + drive_items = execute_query_with_retry(root_folder.children.get()) + found_items = [] + for item in drive_items: + item_path = path + "/" + item.name if path else item.name + if item.is_file: + found_items.append((item, item_path)) + else: + found_items.extend(self._list_directories_and_files(item, item_path)) + return found_items + + def _get_files_by_drive_name(self, drives, folder_path): + """Yields files from the specified drive.""" + path_levels = [level for level in folder_path.split("/") if level] + folder_path = "/".join(path_levels) + + for drive in drives: + is_sharepoint = drive.drive_type == "documentLibrary" + if is_sharepoint: + folder = ( + drive.root if folder_path in self.ROOT_PATH else execute_query_with_retry(drive.root.get_by_path(folder_path).get()) + ) + yield from self._list_directories_and_files(folder) + + @property + @lru_cache(maxsize=None) + def drives(self): + """ + Retrieves and caches SharePoint drives, including the user's drive based on authentication type. + """ + drives = execute_query_with_retry(self.one_drive_client.drives.get()) + + if self.config.credentials.auth_type == "Client": + my_drive = execute_query_with_retry(self.one_drive_client.me.drive.get()) + else: + my_drive = execute_query_with_retry( + self.one_drive_client.users.get_by_principal_name(self.config.credentials.user_principal_name).drive.get() + ) + + drives.add_child(my_drive) + + return drives + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Retrieve all files matching the specified glob patterns in SharePoint. + """ + files = self._get_files_by_drive_name(self.drives, self.config.folder_path) + + files_generator = filter_http_urls( + self.filter_files_by_globs_and_start_date( + [ + MicrosoftSharePointRemoteFile( + uri=path, + download_url=file.properties["@microsoft.graph.downloadUrl"], + last_modified=file.properties["lastModifiedDateTime"], + ) + for file, path in files + ], + globs, + ), + logger, + ) + + items_processed = False + for file in files_generator: + items_processed = True + yield file + + if not items_processed: + raise AirbyteTracedException( + message=f"Drive is empty or does not exist.", + failure_type=FailureType.config_error, + ) + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + # choose correct compression mode because the url is random and doesn't end with filename extension + file_extension = file.uri.split(".")[-1] + if file_extension in ["gz", "bz2"]: + compression = "." + file_extension + else: + compression = "disable" + + try: + return smart_open.open(file.download_url, mode=mode.value, compression=compression, encoding=encoding) + except Exception as e: + logger.exception(f"Error opening file {file.uri}: {e}") diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py new file mode 100644 index 000000000000..c77cd9badbde --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import time +from datetime import datetime, timedelta +from http import HTTPStatus + +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +class MicrosoftSharePointRemoteFile(RemoteFile): + download_url: str + + +def filter_http_urls(files, logger): + for file in files: + if file.download_url.startswith("http") and not file.download_url.startswith("https"): # ignore-https-check + logger.error(f"Cannot open file {file.uri}. The URL returned by SharePoint is not secure.") + else: + yield file + + +def execute_query_with_retry(obj, max_retries=5, initial_retry_after=5, max_retry_after=300, max_total_wait_time=600): + """ + Executes a query with retry logic on encountering specific HTTP errors. + + This function attempts to execute `obj.execute_query()` method, applying exponential backoff + retry logic for HTTP status codes 429 (Too Many Requests) and 503 (Service Unavailable). It + respects the 'Retry-After' header from the response, if present. + + Parameters: + obj (object): The object that has the `execute_query` method to be executed. + max_retries (int): Maximum number of retry attempts. Defaults to 5. + initial_retry_after (int): Initial waiting time (in seconds) before the first retry. Defaults to 5 seconds. + max_retry_after (int): Maximum waiting time (in seconds) between retries. Defaults to 300 seconds. + max_total_wait_time (int): Maximum total waiting time (in seconds) for all retries. Defaults to 600 seconds. + + Raises: + AirbyteTracedException: If the maximum total wait time or the maximum number of retries is exceeded. + + Returns: + The result of `obj.execute_query()` if successful within the retry constraints. + """ + retries = 0 + start_time = datetime.now() + retry_after = initial_retry_after + + while retries < max_retries: + try: + return obj.execute_query() + except Exception as ex: + if hasattr(ex, "response") and ex.response.status_code in (HTTPStatus.TOO_MANY_REQUESTS, HTTPStatus.SERVICE_UNAVAILABLE): + current_time = datetime.now() + elapsed_time = (current_time - start_time).total_seconds() + + retry_after_header = ex.response.headers.get("Retry-After", None) + if retry_after_header: + retry_after = int(retry_after_header) + + if elapsed_time + retry_after > max_total_wait_time: + message = ( + f"Maximum total wait time of {max_total_wait_time} seconds exceeded for execute_query. " + f"The latest response status code is {ex.response.status_code}." + ) + if retry_after_header: + message += f" Retry-After header: {retry_after_header}" + raise AirbyteTracedException(message, message, failure_type=FailureType.system_error) + + time.sleep(retry_after) + retries += 1 + retry_after = min(retry_after * 2, max_retry_after) # Double the wait time for next retry, up to a max limit + else: + # Re-raise exceptions that are not related to rate limits or service availability + raise AirbyteTracedException.from_exception(ex, message="Caught unexpected exception") + + message = f"Maximum number of retries of {max_retries} exceeded for execute_query." + raise AirbyteTracedException(message, message, failure_type=FailureType.system_error) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/__init__.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py new file mode 100644 index 000000000000..4a0d5d7116e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py @@ -0,0 +1,254 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import datetime +from unittest.mock import Mock, patch + +import pytest +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from msal.exceptions import MsalServiceError +from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec +from source_microsoft_sharepoint.stream_reader import ( + FileReadMode, + MicrosoftSharePointRemoteFile, + SourceMicrosoftSharePointClient, + SourceMicrosoftSharePointStreamReader, +) +from wcmatch.glob import GLOBSTAR, globmatch + + +def create_mock_drive_item(is_file, name, children=None): + """Helper function to create a mock drive item.""" + mock_item = Mock() + mock_item.is_file = is_file + mock_item.name = name + mock_item.children.get.return_value.execute_query = Mock(return_value=children or []) + return mock_item + + +@pytest.fixture +def setup_reader_class(): + reader = SourceMicrosoftSharePointStreamReader() # Instantiate your class here + config = Mock(spec=SourceMicrosoftSharePointSpec) + config.start_date = None + config.credentials = Mock() + config.folder_path = "." + config.credentials.auth_type = "Client" + reader.config = config # Set up the necessary configuration + + # Mock the client creation + with patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointClient") as mock_client_class: + mock_client = mock_client_class.return_value + mock_client.client = Mock() # Mock the client attribute of SourceMicrosoftSharePointClient + yield reader + + +@pytest.fixture +def mock_drive_files(): + # Mock files returned by SharePoint + return [ + Mock( + properties={ + "@microsoft.graph.downloadUrl": "https://example.com/file1.csv", + "lastModifiedDateTime": datetime.datetime(2021, 1, 1), + } + ), + Mock( + properties={ + "@microsoft.graph.downloadUrl": "https://example.com/file2.txt", + "lastModifiedDateTime": datetime.datetime(2021, 1, 1), + } + ), + ] + + +@pytest.fixture +def setup_client_class(): + config = Mock(spec=SourceMicrosoftSharePointSpec) + config.credentials = Mock() + config.folder_path = "." + config.credentials.auth_type = "Client" + + with patch("source_microsoft_sharepoint.stream_reader.ConfidentialClientApplication") as mock_client_class: + mock_msal_app_instance = Mock() + mock_client_class.return_value = mock_msal_app_instance + + client_class = SourceMicrosoftSharePointClient(config) + + yield client_class + + +@pytest.mark.parametrize( + "has_refresh_token, token_response, expected_result, raises_exception", + [ + (False, {"access_token": "test_access_token"}, {"access_token": "test_access_token"}, False), + (True, {"access_token": "test_access_token"}, {"access_token": "test_access_token"}, False), + (False, {"error": "test_error", "error_description": "test_error_description"}, None, True), + ], +) +def test_get_access_token(setup_client_class, has_refresh_token, token_response, expected_result, raises_exception): + instance = setup_client_class + if has_refresh_token: + instance.config.credentials.refresh_token = "test_refresh_token" + instance._msal_app.acquire_token_by_refresh_token.return_value = token_response + else: + instance.config.credentials.refresh_token = None + instance._msal_app.acquire_token_for_client.return_value = token_response + + if raises_exception: + with pytest.raises(AirbyteTracedException) as exception: + instance._get_access_token() + assert exception.value.message == f"Failed to acquire access token. Error: test_error. Error description: test_error_description." + else: + assert instance._get_access_token() == expected_result + + if has_refresh_token: + instance._msal_app.acquire_token_by_refresh_token.assert_called_once_with( + "test_refresh_token", scopes=["https://graph.microsoft.com/.default"] + ) + else: + instance._msal_app.acquire_token_for_client.assert_called_once_with(scopes=["https://graph.microsoft.com/.default"]) + + +@patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") +@patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.filter_files_by_globs_and_start_date") +def test_get_matching_files(mock_filter_files, mock_execute_query, setup_reader_class, mock_drive_files): + instance = setup_reader_class + instance._get_files_by_drive_name = Mock(return_value=iter([(mock_drive_files[0], "file1.csv"), (mock_drive_files[1], "file2.txt")])) + + # Set up mocks + mock_drive = Mock() + mock_drive.get.return_value = mock_drive + mock_execute_query.return_value = mock_drive + mock_filter_files.side_effect = lambda files, globs: (f for f in files if any(globmatch(f.uri, g, flags=GLOBSTAR) for g in globs)) + + # Define test parameters + globs = ["*.csv"] + prefix = None + logger = Mock() + + # Call the method + files = list(instance.get_matching_files(globs, prefix, logger)) + + # Assertions + assert len(files) == 1 + assert isinstance(files[0], MicrosoftSharePointRemoteFile) + assert files[0].uri == "file1.csv" + assert "https://example.com/file1.csv" in files[0].download_url + + +def test_get_matching_files_empty_drive(setup_reader_class): + instance = setup_reader_class + instance._get_files_by_drive_name = Mock(return_value=iter([])) + + # Define test parameters + globs = ["*.csv"] + prefix = None + logger = Mock() + + # Expecting an exception when drive is empty + with pytest.raises(AirbyteTracedException): + list(instance.get_matching_files(globs, prefix, logger)) + + +@pytest.mark.parametrize( + "file_extension, expected_compression", + [ + (".txt.gz", ".gz"), + (".txt.bz2", ".bz2"), + ("txt", "disable"), + ], +) +@patch("smart_open.open") +def test_open_file(mock_smart_open, file_extension, expected_compression): + """Test the open_file method in SourceMicrosoftSharePointStreamReader.""" + mock_file = Mock(download_url=f"https://example.com/file.{file_extension}", uri=f"file.{file_extension}") + mock_logger = Mock() + + stream_reader = SourceMicrosoftSharePointStreamReader() + stream_reader._config = Mock() # Assuming _config is required + + with stream_reader.open_file(mock_file, FileReadMode.READ, "utf-8", mock_logger) as result: + pass + + mock_smart_open.assert_called_once_with(mock_file.download_url, mode="r", encoding="utf-8", compression=expected_compression) + assert result is not None + + +def test_microsoft_sharepoint_client_initialization(requests_mock): + """Test the initialization of SourceMicrosoftSharePointClient.""" + config = { + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, + "drive_name": "drive_name", + "folder_path": "folder_path", + "streams": [{"name": "test_stream", "globs": ["*.csv"], "validation_policy": "Emit Record", "format": {"filetype": "csv"}}], + } + + authority_url = "https://login.microsoftonline.com/tenant_id/v2.0/.well-known/openid-configuration" + mock_response = { + "authorization_endpoint": "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/authorize", + "token_endpoint": "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + } + requests_mock.get(authority_url, json=mock_response, status_code=200) + + client = SourceMicrosoftSharePointClient(SourceMicrosoftSharePointSpec(**config)) + + assert client.config == SourceMicrosoftSharePointSpec(**config) + assert client._msal_app is not None + + +def test_list_directories_and_files(): + """Test the list_directories_and_files method in SourceMicrosoftSharePointStreamReader.""" + # Create a mock structure of folders and files + mock_child_file1 = create_mock_drive_item(True, "file1.txt") + mock_child_file2 = create_mock_drive_item(True, "file2.txt") + mock_child_folder = create_mock_drive_item(False, "folder1", children=[mock_child_file1]) + mock_root_folder = create_mock_drive_item(False, "root", children=[mock_child_folder, mock_child_file2]) + + stream_reader = SourceMicrosoftSharePointStreamReader() + + result = stream_reader._list_directories_and_files(mock_root_folder) + + assert len(result) == 2 + assert result[0][1] == "folder1/file1.txt" + assert result[1][1] == "file2.txt" + + +@pytest.mark.parametrize( + "drive_type, files_number", + [ + ("documentLibrary", 1), + ("business", 0), + ], +) +@patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader._list_directories_and_files") +def test_get_files_by_drive_name(mock_list_directories_and_files, drive_type, files_number): + # Helper function usage + mock_drive = Mock() + mock_drive.name = "testDrive" + mock_drive.drive_type = drive_type + mock_drive.root.get_by_path.return_value.get().execute_query_with_incremental_retry.return_value = create_mock_drive_item( + is_file=False, name="root" + ) + + # Mock files + mock_file = create_mock_drive_item(is_file=True, name="testFile.txt") + mock_list_directories_and_files.return_value = [mock_file] + + # Create stream reader instance + stream_reader = SourceMicrosoftSharePointStreamReader() + stream_reader._config = Mock() + + # Call the method + files = list(stream_reader._get_files_by_drive_name([mock_drive], "/test/path")) + + # Assertions + assert len(files) == files_number + if files_number: + assert files[0].name == "testFile.txt" diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py new file mode 100644 index 000000000000..8a8d5110f56e --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py @@ -0,0 +1,90 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta +from http import HTTPStatus +from unittest.mock import Mock, patch + +import pytest +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from source_microsoft_sharepoint.utils import execute_query_with_retry, filter_http_urls + + +class MockResponse: + def __init__(self, status_code, headers=None): + self.status_code = status_code + self.headers = headers or {} + + +class MockException(Exception): + def __init__(self, status_code, headers=None): + self.response = MockResponse(status_code, headers) + + +@pytest.mark.parametrize( + "status_code, retry_after_header, expected_retries, error_message", + [ + ( + HTTPStatus.TOO_MANY_REQUESTS, + None, + 4, + "Maximum total wait time of 10 seconds exceeded for execute_query. The latest response status code is 429.", + ), # No 'Retry-After' header, should retry max times + ( + HTTPStatus.SERVICE_UNAVAILABLE, + "4", + 4, + "Maximum total wait time of 10 seconds exceeded for execute_query. The latest response status code is 503. Retry-After header: 4", + ), # With 'Retry-After' header, limited retries due to time constraint + ( + HTTPStatus.SERVICE_UNAVAILABLE, + "1", + 5, + "Maximum number of retries of 5 exceeded for execute_query.", + ), # With 'Retry-After' header, max number of retries + (HTTPStatus.FORBIDDEN, "1", 1, "Caught unexpected exception"), # unexpected exception + ], +) +def test_execute_query_with_retry(status_code, retry_after_header, expected_retries, error_message): + obj = Mock() + obj.execute_query = Mock(side_effect=MockException(status_code, {"Retry-After": retry_after_header})) + + with patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, patch( + "source_microsoft_sharepoint.utils.datetime" + ) as mock_datetime: + start_time = datetime(2021, 1, 1, 0, 0, 0) + if retry_after_header: + mock_datetime.now.side_effect = [start_time] * 2 + [ + start_time + timedelta(seconds=int(retry_after_header) * i) for i in range(5) + ] + else: + mock_datetime.now.side_effect = [start_time] * 2 + [start_time + timedelta(seconds=2**i) for i in range(5)] + + with pytest.raises(AirbyteTracedException) as exception: + execute_query_with_retry(obj, max_retries=5, initial_retry_after=1, max_retry_after=10, max_total_wait_time=10) + assert exception.value.message == error_message + assert obj.execute_query.call_count == expected_retries + + +def test_execute_query_success_before_max_retries(): + obj = Mock() + obj.execute_query = Mock(side_effect=[MockException(HTTPStatus.TOO_MANY_REQUESTS), "success"]) + + result = execute_query_with_retry(obj, max_retries=5, initial_retry_after=1, max_retry_after=10, max_total_wait_time=10) + + assert obj.execute_query.call_count == 2 + assert result == "success" + + +def test_filter_http_urls(): + files = [ + Mock(download_url="https://example.com/file1.txt"), + Mock(download_url="https://example.com/file2.txt"), + Mock(uri="file3.txt", download_url="http://example.com/file3.txt"), + ] + + mock_logger = Mock() + filtered_files = filter_http_urls(files, mock_logger) + filtered_files = list(filtered_files) + + assert len(filtered_files) == 2 + mock_logger.error.assert_called_once_with("Cannot open file file3.txt. The URL returned by SharePoint is not secure.") diff --git a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile index 3cdb20113e74..4b206258d0b3 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile +++ b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile @@ -34,5 +34,5 @@ COPY source_microsoft_teams ./source_microsoft_teams ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-microsoft-teams diff --git a/airbyte-integrations/connectors/source-microsoft-teams/main.py b/airbyte-integrations/connectors/source-microsoft-teams/main.py index c8bc03c5deea..213b13e85ee5 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/main.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_microsoft_teams import SourceMicrosoftTeams +from source_microsoft_teams.run import run if __name__ == "__main__": - source = SourceMicrosoftTeams() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml index cf90d7d46549..f9307d6d500c 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml @@ -2,17 +2,29 @@ data: connectorSubtype: api connectorType: source definitionId: eaf50f04-21dd-4620-913b-2a83f5635227 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-microsoft-teams githubIssueLabel: source-microsoft-teams icon: microsoft-teams.svg license: MIT name: Microsoft teams + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-microsoft-teams registries: cloud: enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + message: + Version 1.0.0 introduces breaking schema changes to all streams. + A full schema refresh is required to upgrade to this version. + For more details, see our migration guide. + upgradeDeadline: "2024-01-24" releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-teams tags: diff --git a/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json index ca4a5fc9076c..a395f1f9b46a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json @@ -5,55 +5,7 @@ "name": "users", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -63,159 +15,7 @@ "name": "groups", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "deleted_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "classification": { - "type": ["null", "string"] - }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "creation_options": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "description": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "expiration_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "group_types": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "is_assignable_to_role": { - "type": ["null", "boolean"] - }, - "mail": { - "type": ["null", "string"] - }, - "mail_enabled": { - "type": ["null", "boolean"] - }, - "mail_nickname": { - "type": ["null", "string"] - }, - "membership_rule": { - "type": ["null", "string"] - }, - "membership_rule_processing_state": { - "type": ["null", "string"] - }, - "onPremises_domain_name": { - "type": ["null", "string"] - }, - "on_premises_last_sync_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "on_premises_net_bios_name": { - "type": ["null", "string"] - }, - "on_premises_sam_account_name": { - "type": ["null", "string"] - }, - "on_premises_security_identifier": { - "type": ["null", "string"] - }, - "on_premises_sync_enabled": { - "type": ["null", "boolean"] - }, - "preferred_data_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "proxy_addresses": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "renewed_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "resource_behavior_options": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "resource_provisioning_options": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "security_enabled": { - "type": ["null", "boolean"] - }, - "security_edentifier": { - "type": ["null", "string"] - }, - "theme": { - "type": ["null", "string"] - }, - "visibility": { - "type": ["null", "string"] - }, - "on_premises_provisioning_errors": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -225,55 +25,7 @@ "name": "group_members", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -283,58 +35,7 @@ "name": "group_owners", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -344,27 +45,7 @@ "name": "channels", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -374,40 +55,7 @@ "name": "channel_members", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "user_id": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -417,72 +65,7 @@ "name": "channel_tabs", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "sort_order_index": { - "type": ["null", "string"] - }, - "teams_app": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "distribution_method": { - "type": ["null", "string"] - } - } - }, - "configuration": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "entity_id": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "remove_url": { - "type": ["null", "string"] - }, - "website_url": { - "type": ["null", "string"] - }, - "wiki_tab_id": { - "type": ["null", "integer"] - }, - "wiki_default_tab": { - "type": ["null", "boolean"] - }, - "has_content": { - "type": ["null", "boolean"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -492,72 +75,7 @@ "name": "conversations", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "sort_order_index": { - "type": ["null", "string"] - }, - "teams_app": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "distribution_method": { - "type": ["null", "string"] - } - } - }, - "configuration": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "entity_id": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "remove_url": { - "type": ["null", "string"] - }, - "website_url": { - "type": ["null", "string"] - }, - "wiki_tab_id": { - "type": ["null", "integer"] - }, - "wiki_default_tab": { - "type": ["null", "boolean"] - }, - "has_content": { - "type": ["null", "boolean"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -567,40 +85,7 @@ "name": "conversation_threads", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "conversation_id": { - "type": ["null", "string"] - }, - "topic": { - "type": ["null", "string"] - }, - "has_attachments": { - "type": ["null", "boolean"] - }, - "last_delivered_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "unique_senders": { - "type": ["null", "string"] - }, - "preview": { - "type": ["null", "string"] - }, - "is_locked": { - "type": ["null", "boolean"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -610,100 +95,7 @@ "name": "conversation_posts", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "thread_id": { - "type": ["null", "string"] - }, - "conversation_id": { - "type": ["null", "string"] - }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_modified_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "change_key": { - "type": ["null", "string"] - }, - "categories": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "received_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "has_attachments": { - "type": ["null", "boolean"] - }, - "body": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "content_type": { - "type": ["null", "string"] - }, - "content": { - "type": ["null", "string"] - } - } - }, - "from": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "emailAddress": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - } - } - } - } - }, - "sender": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "emailAddress": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - } - } - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -713,84 +105,7 @@ "name": "team_drives", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "last_modified_date_time": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "drive_type": { - "type": ["null", "string"] - }, - "created_by": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "display_name": { - "type": ["null", "string"] - } - } - } - } - }, - "owner": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "group": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "email": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - } - } - } - } - }, - "quota": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "deleted": { - "type": ["null", "integer"] - }, - "remaining": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "string"] - }, - "total": { - "type": ["null", "number"] - }, - "used": { - "type": ["null", "integer"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -800,48 +115,7 @@ "name": "team_device_usage_report", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "report_refresh_date": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - }, - "last_activity_date": { - "type": ["null", "string"] - }, - "is_deleted": { - "type": ["null", "string"] - }, - "deleted_date": { - "type": ["null", "string"] - }, - "used_web": { - "type": ["null", "string"] - }, - "used_windows_phone": { - "type": ["null", "string"] - }, - "used_i_os": { - "type": ["null", "string"] - }, - "used_mac": { - "type": ["null", "string"] - }, - "used_android_phone": { - "type": ["null", "string"] - }, - "used_windows": { - "type": ["null", "string"] - }, - "report_period": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index 1867013845c2..07b984cad43d 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", "requests", "msal==1.7.0", "backoff", @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-microsoft-teams=source_microsoft_teams.run:run", + ], + }, name="source_microsoft_teams", description="Source implementation for Microsoft Teams.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py index 8a3d4893fa77..c16459025f98 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py @@ -243,19 +243,24 @@ def get_team_device_usage_report(self): csv_response.readline() with io.TextIOWrapper(csv_response, encoding="utf-8-sig") as text_file: field_names = [ - "report_refresh_date", - "user_principal_name", - "last_activity_date", - "is_deleted", - "deleted_date", - "used_web", - "used_windows_phone", - "used_i_os", - "used_mac", - "used_android_phone", - "used_windows", - "report_period", + "reportRefreshDate", + "userId", + "userPrincipalName", + "lastActivityDate", + "isDeleted", + "deletedDate", + "usedWeb", + "usedWindowsPhone", + "usedIOs", + "usedMac", + "usedAndroidPhone", + "usedWindows", + "usedChromeOS", + "usedLinux", + "isLisenced", + "reportPeriod", ] + reader = csv.DictReader(text_file, fieldnames=field_names) for row in reader: yield [ diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py new file mode 100644 index 000000000000..f22e12386ac7 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_microsoft_teams import SourceMicrosoftTeams + + +def run(): + source = SourceMicrosoftTeams() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json index 536c1efc8116..b99d57b03940 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json @@ -6,21 +6,14 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "description": { "type": ["null", "string"] @@ -28,7 +21,7 @@ "email": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json index 0f72d22d63c5..3c236063c1c6 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json @@ -6,30 +6,34 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "@odata.type": { + "type": ["null", "string"] + }, + "displayName": { "type": ["null", "string"] }, "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "user_id": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "userId": { "type": ["null", "string"] }, "email": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] + }, + "tenantId": { + "type": ["null", "string"] + }, + "visibleHistoryStartDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json index 9e58d5f02fbe..ac7cbb06d002 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json @@ -6,26 +6,34 @@ "id": { "type": ["null", "string"] }, - "reply_to_id": { + "replyToId": { "type": ["null", "string"] }, "etag": { "type": ["null", "string"] }, - "message_type": { + "messageType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "last_modified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "deleted_date_time": { + "lastEditedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "deletedDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "subject": { "type": ["null", "string"] @@ -33,7 +41,7 @@ "summary": { "type": ["null", "string"] }, - "chat_id": { + "chatId": { "type": ["null", "string"] }, "importance": { @@ -42,10 +50,10 @@ "locale": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "policy_violation": { + "policyViolation": { "type": ["null", "string"] }, "from": { @@ -68,10 +76,13 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { + "type": ["null", "string"] + }, + "tenantId": { "type": ["null", "string"] } } @@ -82,7 +93,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -90,156 +101,136 @@ } } }, - "channel_identity": { + "channelIdentity": { "type": ["null", "object"], "additionalProperties": true, "properties": { "teamId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] } } }, "attachments": { - "anyOf": [ - { - "type": "array", - "items": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "content_type": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "content": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "thumbnail_url": { - "type": ["null", "string"] - } - } + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "contentType": { + "type": ["null", "string"] + }, + "contentUrl": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "thumbnailUrl": { + "type": ["null", "string"] } - }, - { - "type": "null" } - ] + } }, "mentions": { - "anyOf": [ - { - "type": "array", - "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "mentionText": { + "type": ["null", "string"] + }, + "mentioned": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { - "type": ["null", "integer"] + "application": { + "type": ["null", "string"] }, - "mention_text": { + "device": { "type": ["null", "string"] }, - "mentioned": { + "conversation": { + "type": ["null", "string"] + }, + "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "application": { + "id": { "type": ["null", "string"] }, - "device": { + "displayName": { "type": ["null", "string"] }, - "conversation": { + "userIdentityType": { "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "user_identity_type": { - "type": ["null", "string"] - } - } } } } } } - }, - { - "type": "null" } - ] + } }, "reactions": { - "anyOf": [ - { - "type": "array", - "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "reactionType": { + "type": ["null", "string"] + }, + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "reaction_type": { + "application": { + "type": ["null", "string"] + }, + "device": { "type": ["null", "string"] }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" + "conversation": { + "type": ["null", "string"] }, "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "application": { + "id": { "type": ["null", "string"] }, - "device": { + "displayName": { "type": ["null", "string"] }, - "conversation": { + "userIdentityType": { "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "user_identity_type": { - "type": ["null", "string"] - } - } } } } } } - }, - { - "type": "null" } - ] + } } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json index 3b72078dfbb9..ced93bf8121b 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json @@ -6,26 +6,29 @@ "id": { "type": ["null", "string"] }, - "reply_to_id": { + "replyToId": { "type": ["null", "string"] }, "etag": { "type": ["null", "string"] }, - "message_type": { + "messageType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "lastModified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "deleted_date_time": { + "deletedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "subject": { "type": ["null", "string"] @@ -33,7 +36,7 @@ "summary": { "type": ["null", "string"] }, - "chat_id": { + "chatId": { "type": ["null", "string"] }, "importance": { @@ -42,10 +45,10 @@ "locale": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "policy_violation": { + "policyViolation": { "type": ["null", "string"] }, "from": { @@ -68,10 +71,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { "type": ["null", "string"] } } @@ -82,7 +85,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -90,14 +93,14 @@ } } }, - "channel_identity": { + "channelIdentity": { "type": ["null", "object"], "additionalProperties": true, "properties": { "teamId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] } } @@ -113,10 +116,10 @@ "id": { "type": ["null", "string"] }, - "content_type": { + "contentType": { "type": ["null", "string"] }, - "content_url": { + "contentUrl": { "type": ["null", "string"] }, "content": { @@ -125,7 +128,7 @@ "name": { "type": ["null", "string"] }, - "thumbnail_url": { + "thumbnailUrl": { "type": ["null", "string"] } } @@ -147,7 +150,7 @@ "id": { "type": ["null", "integer"] }, - "mention_text": { + "mentionText": { "type": ["null", "string"] }, "mentioned": { @@ -170,10 +173,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "userIdentity_type": { + "userIdentityType": { "type": ["null", "string"] } } @@ -196,10 +199,10 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "reaction_type": { + "reactionType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], "format": "date-time" }, @@ -223,10 +226,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json index c66b4bf72179..ed0867b1e4c7 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json @@ -6,32 +6,32 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "sort_order_index": { + "sortOrderIndex": { "type": ["null", "string"] }, - "teams_app": { + "teamsApp": { "type": ["null", "object"], "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "distribution_method": { + "distributionMethod": { "type": ["null", "string"] } } @@ -40,25 +40,25 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "entity_id": { + "entityId": { "type": ["null", "string"] }, - "content_url": { + "contentUrl": { "type": ["null", "string"] }, - "remove_url": { + "removeUrl": { "type": ["null", "string"] }, - "website_url": { + "websiteUrl": { "type": ["null", "string"] }, - "wiki_tab_id": { + "wikiTabId": { "type": ["null", "integer"] }, - "wiki_default_tab": { + "wikiDefaultTab": { "type": ["null", "boolean"] }, - "has_content": { + "hasContent": { "type": ["null", "boolean"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json index 156390fc505e..999eae607c23 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json @@ -6,7 +6,12 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "createdDateTime": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "displayName": { "type": ["null", "string"] }, "description": { @@ -15,7 +20,16 @@ "email": { "type": ["null", "string"] }, - "web_url": { + "isFavoriteByDefault": { + "type": ["null", "boolean"] + }, + "membershipType": { + "type": ["null", "string"] + }, + "tenantId": { + "type": ["null", "string"] + }, + "webUrl": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json index 24bf5cd4268e..4389b581962a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json @@ -6,48 +6,47 @@ "id": { "type": ["null", "string"] }, - "thread_id": { + "threadId": { "type": ["null", "string"] }, - "conversation_id": { + "conversationId": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "last_modified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "change_key": { + "@odata.etag": { + "type": ["null", "string"] + }, + "changeKey": { "type": ["null", "string"] }, "categories": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "received_date_time": { + "receivedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, "body": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -61,7 +60,6 @@ "properties": { "emailAddress": { "type": ["null", "object"], - "additionalProperties": false, "properties": { "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json index 35078c5f4d89..54c27157062e 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json @@ -6,29 +6,33 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "conversation_id": { + "conversationId": { "type": ["null", "string"] }, "topic": { "type": ["null", "string"] }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, - "last_delivered_date_time": { + "lastDeliveredDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "unique_senders": { - "type": ["null", "string"] + "uniqueSenders": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "preview": { "type": ["null", "string"] }, - "is_locked": { + "isLocked": { "type": ["null", "boolean"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json index 58a88cffff10..e9045284dd5c 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json @@ -6,31 +6,25 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, "topic": { "type": ["null", "string"] }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, - "last_delivered_date_time": { + "lastDeliveredDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "unique_senders": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "uniqueSenders": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "preview": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json index ef6a3bb26028..2bf02fd72977 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json @@ -6,44 +6,40 @@ "id": { "type": ["null", "string"] }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "@odata.type": { + "type": ["null", "string"] + }, + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json index dc222c183354..aa1b8915682d 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json @@ -6,47 +6,43 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "@odata.type": { + "type": ["null", "string"] + }, + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json index a348b7743603..2876585b6fad 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json @@ -6,149 +6,144 @@ "id": { "type": ["null", "string"] }, - "deleted_date_time": { + "deletedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "classification": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "creation_options": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "creationOptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "description": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "expiration_date_time": { + "expirationDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "group_types": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "groupTypes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "is_assignable_to_role": { + "isAssignableToRole": { "type": ["null", "boolean"] }, "mail": { "type": ["null", "string"] }, - "mail_enabled": { + "mailEnabled": { "type": ["null", "boolean"] }, - "mail_nickname": { + "mailNickname": { "type": ["null", "string"] }, - "membership_rule": { + "membershipRule": { "type": ["null", "string"] }, - "membership_rule_processing_state": { + "membershipRuleProcessingState": { "type": ["null", "string"] }, - "onPremises_domain_name": { + "onPremisesDomainName": { "type": ["null", "string"] }, - "on_premises_last_sync_date_time": { + "onPremisesLastSyncDateTime": { "type": ["null", "string"], "format": "date-time" }, - "on_premises_net_bios_name": { + "onPremisesNetBiosName": { "type": ["null", "string"] }, - "on_premises_sam_account_name": { + "onPremisesSamAccountName": { "type": ["null", "string"] }, - "on_premises_security_identifier": { + "onPremisesSecurityIdentifier": { "type": ["null", "string"] }, - "on_premises_sync_enabled": { + "onPremisesSyncEnabled": { "type": ["null", "boolean"] }, - "preferred_data_location": { + "preferredDataLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, - "proxy_addresses": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "proxyAddresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "renewed_date_time": { + "renewedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "resource_behavior_options": { + "resourceBehaviorOptions": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, - "resource_provisioning_options": { + "resourceProvisioningOptions": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, - "security_enabled": { + "securityEnabled": { "type": ["null", "boolean"] }, - "security_edentifier": { + "securityIdentifier": { "type": ["null", "string"] }, + "serviceProvisioningErrors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "isResolved": { + "type": ["null", "boolean"] + }, + "serviceInstance": { + "type": ["null", "string"] + } + } + } + }, "theme": { "type": ["null", "string"] }, "visibility": { "type": ["null", "string"] }, - "on_premises_provisioning_errors": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "onPremisesProvisioningErrors": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json index 40066ae4fc4d..8ae6a571f5d3 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json @@ -3,40 +3,53 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "report_refresh_date": { + "reportRefreshDate": { + "type": ["null", "string"], + "format": "date" + }, + "userId": { + "type": ["null", "string"] + }, + "userPrincipalName": { + "type": ["null", "string"] + }, + "lastActivityDate": { + "type": ["null", "string"] + }, + "isDeleted": { "type": ["null", "string"] }, - "user_principal_name": { + "deletedDate": { "type": ["null", "string"] }, - "last_activity_date": { + "usedWeb": { "type": ["null", "string"] }, - "is_deleted": { + "usedWindowsPhone": { "type": ["null", "string"] }, - "deleted_date": { + "usedIOs": { "type": ["null", "string"] }, - "used_web": { + "usedMac": { "type": ["null", "string"] }, - "used_windows_phone": { + "usedAndroidPhone": { "type": ["null", "string"] }, - "used_i_os": { + "usedWindows": { "type": ["null", "string"] }, - "used_mac": { + "usedChromeOS": { "type": ["null", "string"] }, - "used_android_phone": { + "usedLinux": { "type": ["null", "string"] }, - "used_windows": { + "isLisenced": { "type": ["null", "string"] }, - "report_period": { + "reportPeriod": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json index 0b39515c620e..fbb40f7dcf97 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json @@ -6,19 +6,41 @@ "id": { "type": ["null", "string"] }, - "last_modified_date_time": { + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "description": { "type": ["null", "string"] }, + "lastModifiedBy": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "displayName": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } + }, + "lastModifiedDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, "name": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "drive_type": { + "driveType": { "type": ["null", "string"] }, - "created_by": { + "createdBy": { "type": ["null", "object"], "additionalProperties": true, "properties": { @@ -26,7 +48,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "display_name": { + "displayName": { "type": ["null", "string"] } } @@ -47,7 +69,7 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json index b5853c0d396e..e02d86a53106 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json @@ -3,44 +3,37 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] }, "id": { diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json index ab4af0e7d1ed..39de5a8b8a96 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json @@ -27,7 +27,6 @@ "client_secret", "refresh_token" ], - "additionalProperties": false, "properties": { "auth_type": { "type": "string", @@ -39,7 +38,8 @@ "tenant_id": { "title": "Directory (tenant) ID", "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" + "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL", + "airbyte_secret": true }, "client_id": { "title": "Client ID", @@ -64,7 +64,6 @@ "type": "object", "title": "Authenticate via Microsoft", "required": ["tenant_id", "client_id", "client_secret"], - "additionalProperties": false, "properties": { "auth_type": { "type": "string", @@ -76,7 +75,8 @@ "tenant_id": { "title": "Directory (tenant) ID", "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" + "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL", + "airbyte_secret": true }, "client_id": { "title": "Client ID", diff --git a/airbyte-integrations/connectors/source-mixpanel/README.md b/airbyte-integrations/connectors/source-mixpanel/README.md index 0867d06cb7c0..14431eab38ba 100644 --- a/airbyte-integrations/connectors/source-mixpanel/README.md +++ b/airbyte-integrations/connectors/source-mixpanel/README.md @@ -1,118 +1,55 @@ -# Mixpanel Source +# Mixpanel source connector + This is the repository for the Mixpanel source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mixpanel). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/mixpanel). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python3 -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mixpanel) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mixpanel/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/mixpanel) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mixpanel/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mixpanel test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-mixpanel spec +poetry run source-mixpanel check --config secrets/config.json +poetry run source-mixpanel discover --config secrets/config.json +poetry run source-mixpanel read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-mixpanel build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-mixpanel:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-mixpanel:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-mixpanel:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-mixpanel:dev . -# Running the spec command against your patched connector -docker run airbyte/source-mixpanel:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-mixpanel:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mixpanel:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mixpanel:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-mixpanel test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mixpanel test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mixpanel.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/mixpanel.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mixpanel/main.py b/airbyte-integrations/connectors/source-mixpanel/main.py index 5c1449d8dcf4..df8cb33fc826 100644 --- a/airbyte-integrations/connectors/source-mixpanel/main.py +++ b/airbyte-integrations/connectors/source-mixpanel/main.py @@ -2,14 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mixpanel import SourceMixpanel -from source_mixpanel.config_migrations import MigrateProjectId +from source_mixpanel.run import run if __name__ == "__main__": - source = SourceMixpanel() - MigrateProjectId.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index bceeb8d9eb00..c8d1c570c5dc 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -11,13 +11,17 @@ data: connectorSubtype: api connectorType: source definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a - dockerImageTag: 2.0.0 + dockerImageTag: 2.1.0 dockerRepository: airbyte/source-mixpanel documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel githubIssueLabel: source-mixpanel icon: mixpanel.svg license: MIT name: Mixpanel + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mixpanel registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-mixpanel/poetry.lock b/airbyte-integrations/connectors/source-mixpanel/poetry.lock new file mode 100644 index 000000000000..7941e3a9b3df --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "b9f1312ff855d2ea6c2f6c7a329923044ad6cd1b88c9c3de3b49736510b45be6" diff --git a/airbyte-integrations/connectors/source-mixpanel/pyproject.toml b/airbyte-integrations/connectors/source-mixpanel/pyproject.toml new file mode 100644 index 000000000000..a958e0159aa4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.0.2" +name = "source-mixpanel" +description = "Source implementation for Mixpanel." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/mixpanel" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_mixpanel" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.8" + +[tool.poetry.scripts] +source-mixpanel = "source_mixpanel.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-mixpanel/requirements.txt b/airbyte-integrations/connectors/source-mixpanel/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-mixpanel/setup.py b/airbyte-integrations/connectors/source-mixpanel/setup.py deleted file mode 100644 index 4a9918b16c91..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] - -setup( - name="source_mixpanel", - description="Source implementation for Mixpanel.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py new file mode 100644 index 000000000000..1d512c472c84 --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mixpanel import SourceMixpanel +from source_mixpanel.config_migrations import MigrateProjectId + + +def run(): + source = SourceMixpanel() + MigrateProjectId.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/funnels.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/funnels.json index a0488a87df68..1cb068a86c7a 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/funnels.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/funnels.json @@ -25,6 +25,12 @@ "count": { "type": ["null", "integer"] }, + "custom_event": { + "type": ["null", "boolean"] + }, + "custom_event_id": { + "type": ["null", "integer"] + }, "avg_time": { "type": ["null", "number"], "multipleOf": 1e-20 diff --git a/airbyte-integrations/connectors/source-monday/Dockerfile b/airbyte-integrations/connectors/source-monday/Dockerfile deleted file mode 100644 index e3e950b0cfef..000000000000 --- a/airbyte-integrations/connectors/source-monday/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_monday ./source_monday - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.1.3 -LABEL io.airbyte.name=airbyte/source-monday diff --git a/airbyte-integrations/connectors/source-monday/README.md b/airbyte-integrations/connectors/source-monday/README.md index 88b2490a0940..39551615dec6 100644 --- a/airbyte-integrations/connectors/source-monday/README.md +++ b/airbyte-integrations/connectors/source-monday/README.md @@ -1,69 +1,55 @@ -# Monday Source +# Monday source connector + This is the repository for the Monday source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/monday). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/monday). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/monday) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_monday/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/monday) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_monday/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source monday test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-monday spec +poetry run source-monday check --config secrets/config.json +poetry run source-monday discover --config secrets/config.json +poetry run source-monday read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-monday build ``` -An image will be built with the tag `airbyte/source-monday:dev`. +An image will be available on your host with the tag `airbyte/source-monday:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-monday:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-monday:dev spec @@ -72,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-monday:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-monday:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-monday test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-monday test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/monday.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/monday.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml index 5f312963ecd6..01fc98c6f304 100644 --- a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml @@ -22,13 +22,13 @@ acceptance_tests: # `boards`, `items`, `updates` streams schemas were modified. PR: https://github.com/airbytehq/airbyte/pull/27410 # Changes applies to all configs backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" - config_path: "secrets/config_api_token.json" backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" - config_path: "secrets/config_oauth.json" backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" basic_read: tests: - config_path: "secrets/config_api_token.json" @@ -39,7 +39,7 @@ acceptance_tests: extra_records: yes empty_streams: - name: teams - bypass_reason: "unable to populate" + bypass_reason: "The stream has no test data and tested with integration tests" ignored_fields: items: - name: assets/*/public_url @@ -55,7 +55,7 @@ acceptance_tests: extra_records: yes empty_streams: - name: teams - bypass_reason: "unable to populate" + bypass_reason: "The stream has no test data and tested with integration tests" ignored_fields: items: - name: assets/*/public_url diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl index d7077351a0ca..21febb1c0f36 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl @@ -1,17 +1,8 @@ -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2019-03-01T17:24:57.321Z\"}", "description": null, "id": "status", "text": "Working on it", "title": "Status", "type": "color", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-03-01T17:24:57.321Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "open", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038090]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211945", "name": "Item 1", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-15T16:19:37Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "updated_at_int": 1686845977}, "emitted_at": 1690884054247} -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "description": null, "id": "status", "text": "Done", "title": "Status", "type": "color", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-03-01T17:28:23.178Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "closed", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038091]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211964", "name": "Item 2", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:59:36Z", "updates": [], "updated_at_int": 1686664776}, "emitted_at": 1690884054254} -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":null,\"color\":\"#c4c4c4\",\"changed_at\":\"2019-03-01T17:25:02.248Z\"}", "description": null, "id": "status", "text": null, "title": "Status", "type": "color", "value": "{\"index\":5,\"post_id\":null,\"changed_at\":\"2019-03-01T17:25:02.248Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-13", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-13\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:26.291Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "", "title": "Tags", "type": "tag", "value": null}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211977", "name": "Item 3", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:58:26Z", "updates": [], "updated_at_int": 1686664706}, "emitted_at": 1690884054258} -{"stream": "boards", "data": {"board_kind": "public", "type": "board", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Person", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "tags", "settings_str": "{\"hide_footer\":false}", "title": "Tags", "type": "tag", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}, {"archived": false, "color": "#a25ddc", "deleted": false, "id": "group_title", "position": "98304", "title": "Group Title"}, {"archived": false, "color": "#808080", "deleted": false, "id": "new_group", "position": "163840.0", "title": "New Group unit board"}], "id": "4635211873", "name": "New Board", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-20T12:12:46Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "views": [], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1687263166}, "emitted_at": 1696447529789} -{"stream": "boards", "data": {"board_kind": "public", "type": "document", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "files", "settings_str": "{\"hide_footer\":false}", "title": "Files", "type": "file", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}], "id": "4634950289", "name": "test doc", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-13T13:28:31Z", "updates": [], "views": [{"id": "103920755", "name": "Table", "settings_str": "{}", "type": "FeatureBoardView", "view_specific_data_str": "{}"}], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1686662911}, "emitted_at": 1696447529793} -{"stream": "boards", "data": {"board_kind": "public", "type": "board", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "description": null, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "status1", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Manager", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "budget_owner_approval4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "color", "width": 185}, {"archived": false, "description": null, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "budget_owner_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "procurement_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "finance_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "description": null, "id": "legal", "settings_str": "{}", "title": "Security", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "legal_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "pulse-updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": false, "color": "#FF642E", "deleted": false, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": false, "color": "#037f4c", "deleted": false, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "id": "3555407826", "name": "Procurement process", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-09-22T09:35:45Z", "updates": [], "views": [], "workspace": null, "updated_at_int": 1695375345}, "emitted_at": 1696447529797} -{"stream": "tags", "data": {"color": "#00c875", "id": 19038090, "name": "open"}, "emitted_at": 1690884065804} -{"stream": "tags", "data": {"color": "#fdab3d", "id": 19038091, "name": "closed"}, "emitted_at": 1690884065806} -{"stream": "updates", "data": {"assets": [{"created_at": "2023-06-15T16:19:31Z", "file_extension": ".jpg", "file_size": 116107, "id": "919077184", "name": "black_cat.jpg", "original_geometry": "473x600", "public_url": "https://files-monday-com.s3.amazonaws.com/14202902/resources/919077184/black_cat.jpg?response-content-disposition=attachment&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA4MPVJMFXGWGLJTLY%2F20230801%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230801T100107Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=5d2d3ca95375589e620f89630d58ff0f7417f1ddd8968ceb57af854657718564", "uploaded_by": {"id": 36694549}, "url": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/black_cat.jpg", "url_thumbnail": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/thumb_small-black_cat.jpg"}], "body": "", "created_at": "2023-06-15T16:19:36Z", "creator_id": "36694549", "id": "2223820299", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:19:36Z"}, "emitted_at": 1690884067025} -{"stream": "updates", "data": {"assets": [], "body": "



      ", "created_at": "2023-06-15T16:18:50Z", "creator_id": "36694549", "id": "2223818363", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:18:50Z"}, "emitted_at": 1690884067027} -{"stream": "updates", "data": {"assets": [], "body": "

      \ufeffTest

      ", "created_at": "2022-11-21T14:41:21Z", "creator_id": "36694549", "id": "1825302913", "item_id": "3555437747", "replies": [{"id": "1825303266", "creator_id": "36694549", "created_at": "2022-11-21T14:41:29Z", "text_body": "Test test", "updated_at": "2022-11-21T14:41:29Z", "body": "

      \ufeffTest test

      "}, {"id": "2223806079", "creator_id": "36694549", "created_at": "2023-06-15T16:14:13Z", "text_body": "", "updated_at": "2023-06-15T16:14:13Z", "body": "



      "}], "text_body": "Test", "updated_at": "2023-06-15T16:14:13Z"}, "emitted_at": 1690884067029} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": 36694549, "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 3}, "emitted_at": 1690884067354} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:33:18Z", "join_date": null, "email": "iryna.grankova@airbyte.io", "enabled": true, "id": 36695702, "is_admin": false, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Iryna Grankova", "phone": null, "photo_original": "https://files.monday.com/use1/photos/36695702/original/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_small": "https://files.monday.com/use1/photos/36695702/small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb": "https://files.monday.com/use1/photos/36695702/thumb/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb_small": "https://files.monday.com/use1/photos/36695702/thumb_small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_tiny": "https://files.monday.com/use1/photos/36695702/tiny/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "time_zone_identifier": "Europe/Athens", "title": null, "url": "https://airbyte-unit.monday.com/users/36695702", "utc_hours_diff": 3}, "emitted_at": 1690884067356} -{"stream": "workspaces", "data": {"created_at": "2023-06-08T11:26:44Z", "description": null, "id": 2845647, "kind": "open", "name": "Test workspace", "state": "active", "account_product": {"id": 2248222, "kind": "core"}, "owners_subscribers": [{"id": 36694549}], "settings": {"icon": {"color": "#FDAB3D", "image": null}}, "team_owners_subscribers": [], "teams_subscribers": [], "users_subscribers": [{"id": 36694549}]}, "emitted_at": 1690884067856} -{"stream": "activity_logs", "data": {"id": "81d07d4d-414d-458e-b44c-fef36e44c424", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"new_group\",\"group_name\":\"New Group unit board\",\"group_color\":\"#808080\",\"is_top_group\":false,\"pulse_id\":4672924165,\"pulse_name\":\"Item 7\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631837419768", "created_at_int": 1687263183, "pulse_id": 4672924165}, "emitted_at": 1690884068262} -{"stream": "activity_logs", "data": {"id": "c0aa4bab-d3a4-4f13-8942-934178c0238a", "event": "update_column_value", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_id\":\"status\",\"column_type\":\"color\",\"column_title\":\"Status\",\"value\":{\"label\":{\"index\":1,\"text\":\"Done\",\"style\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"is_done\":true},\"post_id\":null},\"previous_value\":null,\"is_column_with_hide_permissions\":false}", "entity": "pulse", "created_at": "16872631743009674", "created_at_int": 1687263174, "pulse_id": 4672922929}, "emitted_at": 1690884068266} -{"stream": "activity_logs", "data": {"id": "4e8f926c-1b4e-43d8-a3de-09af876ccb9e", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"group_name\":\"Group Title\",\"group_color\":\"#a25ddc\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631712788820", "created_at_int": 1687263171, "pulse_id": 4672922929}, "emitted_at": 1690884068269} +{"stream": "items", "data": {"id": "4635211945", "name": "Item 1", "assets": [], "board": {"id": "4635211873"}, "column_values": [{"id": "person", "text": "", "type": "people", "value": null}, {"id": "status", "text": "Working on it", "type": "status", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-03-01T17:24:57.321Z\"}"}, {"id": "date4", "text": "2023-06-11", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"id": "tags", "text": "open", "type": "tags", "value": "{\"tag_ids\":[19038090]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "parent_item": null, "state": "active", "subscribers": [{"id": "36694549"}], "updated_at": "2023-06-13T13:59:32Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "updated_at_int": 1686664772}, "emitted_at": 1708603933827} +{"stream": "boards", "data": {"id": "3555407826", "name": "Procurement process", "board_kind": "public", "type": "board", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "description": null, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "status1", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "status", "width": null}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Manager", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "budget_owner_approval4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "status", "width": null}, {"archived": false, "description": null, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "status4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "status", "width": 185}, {"archived": false, "description": null, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "people", "width": null}, {"archived": false, "description": null, "id": "budget_owner_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "people", "width": null}, {"archived": false, "description": null, "id": "procurement_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "people", "width": null}, {"archived": false, "description": null, "id": "finance_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "description": null, "id": "legal", "settings_str": "{}", "title": "Security", "type": "people", "width": null}, {"archived": false, "description": null, "id": "legal_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "last_updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": false, "color": "#FF642E", "deleted": false, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": false, "color": "#037f4c", "deleted": false, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "owners": [{"id": "36694549"}], "creator": {"id": "36694549"}, "permissions": "everyone", "state": "active", "subscribers": [{"id": "36694549"}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:36:50Z", "updates": [], "views": [], "workspace": null, "updated_at_int": 1669041410}, "emitted_at": 1705073472066} +{"stream": "tags", "data": {"color": "#00c875", "id": "19038090", "name": "open"}, "emitted_at": 1690884065804} +{"stream": "tags", "data": {"color": "#fdab3d", "id": "19038091", "name": "closed"}, "emitted_at": 1690884065806} +{"stream": "updates", "data": {"assets": [{"created_at": "2023-06-15T16:19:31Z", "file_extension": ".jpg", "file_size": 116107, "id": "919077184", "name": "black_cat.jpg", "original_geometry": "473x600", "public_url": "https://files-monday-com.s3.amazonaws.com/14202902/resources/919077184/black_cat.jpg?response-content-disposition=attachment&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA4MPVJMFXILAOBJXD%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T154009Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=b4f14a9dd800d70520f428ff7f4a29aa1b6a259d761f3b073fe83c41010c729a", "uploaded_by": {"id": "36694549"}, "url": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/black_cat.jpg", "url_thumbnail": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/thumb_small-black_cat.jpg"}], "body": "", "created_at": "2023-06-15T16:19:36Z", "creator_id": "36694549", "id": "2223820299", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:19:36Z"}, "emitted_at": 1705074009909} +{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": "36694549", "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 2}, "emitted_at": 1702496564648} +{"stream": "workspaces", "data": {"created_at": "2023-06-08T11:26:44Z", "description": null, "id": "2845647", "kind": "open", "name": "Test workspace", "state": "active", "account_product": {"id": "2248222", "kind": "core"}, "owners_subscribers": [{"id": "36694549"}], "settings": {"icon": {"color": "#FDAB3D", "image": null}}, "team_owners_subscribers": [], "teams_subscribers": [], "users_subscribers": [{"id": "36694549"}]}, "emitted_at": 1705074164892} +{"stream": "activity_logs", "data": {"id": "81d07d4d-414d-458e-b44c-fef36e44c424", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"new_group\",\"group_name\":\"New Group unit board\",\"group_color\":\"#808080\",\"is_top_group\":false,\"pulse_id\":4672924165,\"pulse_name\":\"Item 7\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631837419768", "created_at_int": 1687263183, "pulse_id": 4672924165}, "emitted_at": 1705074202226} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-monday/main.py b/airbyte-integrations/connectors/source-monday/main.py index 6fec8ed55e58..14f4fa2d0439 100644 --- a/airbyte-integrations/connectors/source-monday/main.py +++ b/airbyte-integrations/connectors/source-monday/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_monday import SourceMonday +from source_monday.run import run if __name__ == "__main__": - source = SourceMonday() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-monday/metadata.yaml b/airbyte-integrations/connectors/source-monday/metadata.yaml index ac999afd827d..abda37283e00 100644 --- a/airbyte-integrations/connectors/source-monday/metadata.yaml +++ b/airbyte-integrations/connectors/source-monday/metadata.yaml @@ -1,28 +1,50 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.monday.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 80a54ea2-9959-4040-aac1-eee42423ec9b - dockerImageTag: 1.1.3 + dockerImageTag: 2.0.3 + releases: + breakingChanges: + 2.0.0: + message: "Source Monday has deprecated API version 2023-07. We have upgraded the connector to the latest API version 2024-01. In this new version, the Id field has changed from an integer to a string in the streams Boards, Items, Tags, Teams, Updates, Users and Workspaces. Please reset affected streams." + upgradeDeadline: "2024-01-15" + scopedImpact: + - scopeType: stream + impactedScopes: + [ + "boards", + "items", + "tags", + "teams", + "updates", + "users", + "workspaces", + ] dockerRepository: airbyte/source-monday + documentationUrl: https://docs.airbyte.com/integrations/sources/monday githubIssueLabel: source-monday icon: monday.svg license: MIT name: Monday + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-monday registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/monday + supportLevel: certified tags: - language:low-code - - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-monday/poetry.lock b/airbyte-integrations/connectors/source-monday/poetry.lock new file mode 100644 index 000000000000..b6f4aa5d14b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.0.tar.gz", hash = "sha256:622f56bd7101493a74f11c33a45a31c251032333989996f137cac8370873c614"}, + {file = "airbyte_cdk-0.62.0-py3-none-any.whl", hash = "sha256:b21330a566b33dbdddde33243eb9855f086ad4272e3585ca626be1225451a3b8"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "1ba7362086cf24723e7be8cf2bc3ea3414137e3e56c9721b6593a0b8c89d269e" diff --git a/airbyte-integrations/connectors/source-monday/pyproject.toml b/airbyte-integrations/connectors/source-monday/pyproject.toml new file mode 100644 index 000000000000..d40e70477332 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.0.3" +name = "source-monday" +description = "Source implementation for Monday." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/monday" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_monday" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.0" + +[tool.poetry.scripts] +source-monday = "source_monday.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-monday/requirements.txt b/airbyte-integrations/connectors/source-monday/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-monday/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-monday/setup.py b/airbyte-integrations/connectors/source-monday/setup.py deleted file mode 100644 index 4af4bd4dac1c..000000000000 --- a/airbyte-integrations/connectors/source-monday/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.44.1", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] - -setup( - name="source_monday", - description="Source implementation for Monday.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py index bd8524044024..830aafc5cf9a 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py @@ -72,11 +72,12 @@ class MondayIncrementalItemsExtractor(RecordExtractor): field_path: List[Union[InterpolatedString, str]] config: Config parameters: InitVar[Mapping[str, Any]] - additional_field_path: List[Union[InterpolatedString, str]] = field(default_factory=list) + field_path_pagination: List[Union[InterpolatedString, str]] = field(default_factory=list) + field_path_incremental: List[Union[InterpolatedString, str]] = field(default_factory=list) decoder: Decoder = JsonDecoder(parameters={}) def __post_init__(self, parameters: Mapping[str, Any]): - for field_list in (self.field_path, self.additional_field_path): + for field_list in (self.field_path, self.field_path_pagination, self.field_path_incremental): for path_index in range(len(field_list)): if isinstance(field_list[path_index], str): field_list[path_index] = InterpolatedString.create(field_list[path_index], parameters=parameters) @@ -100,8 +101,10 @@ def try_extract_records(self, response: requests.Response, field_path: List[Unio def extract_records(self, response: requests.Response) -> List[Record]: result = self.try_extract_records(response, field_path=self.field_path) - if not result and self.additional_field_path: - result = self.try_extract_records(response, self.additional_field_path) + if not result and self.field_path_pagination: + result = self.try_extract_records(response, self.field_path_pagination) + if not result and self.field_path_incremental: + result = self.try_extract_records(response, self.field_path_incremental) for item_index in range(len(result)): if "updated_at" in result[item_index]: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py index 0e5fd049583c..fb360b133ec9 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py @@ -79,18 +79,59 @@ def _build_query(self, object_name: str, field_schema: dict, **object_arguments) arguments = f"({arguments})" if arguments else "" fields = ",".join(fields) - return f"{object_name}{arguments}{{{fields}}}" + # Essentially, we construct a query based on schema properties; however, some fields in the schema are conditional. + # These conditional fields can be obtained by defining them as inline fragments (The docs: https://spec.graphql.org/October2021/#sec-Inline-Fragments). + # This is an example of a query built for the Items stream, with a `display_value` property defined as an `MirrorValue` inline fragment: + # query { + # boards (limit:1) { + # items_page (limit:20) { + # , + # ..., + # column_values { + # id, + # text, + # type, + # value, + # ... on MirrorValue {display_value} + # } + # } + # } + # } + # When constructing a query, we replace the `display_value` field with the `... on MirrorValue {display_value}` inline fragment. + if object_name == "column_values" and "display_value" in fields: + fields = fields.replace("display_value", "... on MirrorValue{display_value}") + + if object_name in ["items_page", "next_items_page"]: + query = f"{object_name}{arguments}{{cursor,items{{{fields}}}}}" + else: + query = f"{object_name}{arguments}{{{fields}}}" + return query def _build_items_query(self, object_name: str, field_schema: dict, sub_page: Optional[int], **object_arguments) -> str: """ Special optimization needed for items stream. Starting October 3rd, 2022 items can only be reached through boards. See https://developer.monday.com/api-reference/docs/items-queries#items-queries + + Comparison of different APIs queries: + 2023-07: + boards(limit: 1) { items(limit: 20) { field1, field2, ... }} + boards(limit: 1, page:2) { items(limit: 20, page:2) { field1, field2, ... }} boards and items paginations + 2024_01: + boards(limit: 1) { items_page(limit: 20) {cursor, items{field1, field2, ...} }} + boards(limit: 1, page:2) { items_page(limit: 20) {cursor, items{field1, field2, ...} }} - boards pagination + next_items_page(limit: 20, cursor: "blaa") {cursor, items{field1, field2, ...} } - items pagination + """ nested_limit = self.nested_limit.eval(self.config) - query = self._build_query("items", field_schema, limit=nested_limit, page=sub_page) - arguments = self._get_object_arguments(**object_arguments) - return f"boards({arguments}){{{query}}}" + if sub_page: + query = self._build_query("next_items_page", field_schema, limit=nested_limit, cursor=f'"{sub_page}"') + else: + query = self._build_query("items_page", field_schema, limit=nested_limit) + arguments = self._get_object_arguments(**object_arguments) + query = f"boards({arguments}){{{query}}}" + + return query def _build_items_incremental_query(self, object_name: str, field_schema: dict, stream_slice: dict, **object_arguments) -> str: """ @@ -133,6 +174,17 @@ def _build_activity_query(self, object_name: str, field_schema: dict, sub_page: arguments = self._get_object_arguments(**object_arguments) return f"boards({arguments}){{{query}}}" + def get_request_headers( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + headers = super().get_request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + headers["API-Version"] = "2024-01" + return headers + def get_request_params( self, *, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py index 5b18cb4b37b7..a6276416d2e5 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py @@ -6,6 +6,10 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + class ItemPaginationStrategy(PageIncrement): """ @@ -45,3 +49,53 @@ def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Op return None return self._page, self._sub_page + + +class ItemCursorPaginationStrategy(PageIncrement): + """ + Page increment strategy with subpages for the `items` stream. + + From the `items` documentation https://developer.monday.com/api-reference/docs/items: + Please note that you cannot return more than 100 items per query when using items at the root. + To adjust your query, try only returning items on a specific board, nesting items inside a boards query, + looping through the boards on your account, or querying less than 100 items at a time. + + This pagination strategy supports nested loop through `boards` on the top level and `items` on the second. + See boards documentation for more details: https://developer.monday.com/api-reference/docs/boards#queries. + """ + + def __post_init__(self, parameters: Mapping[str, Any]): + # `self._page` corresponds to board page number + # `self._sub_page` corresponds to item page number within its board + self.start_from_page = 1 + self._page: Optional[int] = self.start_from_page + self._sub_page: Optional[int] = self.start_from_page + + def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Optional[Tuple[Optional[int], Optional[int]]]: + """ + `items` stream use a separate 2 level pagination strategy where: + 1st level `boards` - incremental pagination + 2nd level `items_page` - cursor pagination + + Attributes: + response: Contains `boards` and corresponding lists of `items` for each `board` + last_records: Parsed `items` from the response + """ + data = response.json()["data"] + boards = data.get("boards", []) + next_items_page = data.get("next_items_page", {}) + if boards: + # there is always only one board due to limit=1, so in one request we extract all 'items_page' for one board only + board = boards[0] + cursor = board.get("items_page", {}).get("cursor", None) + elif next_items_page: + cursor = next_items_page.get("cursor", None) + else: + # Finish pagination if there is no more data + return None + + if cursor: + return self._page, cursor + else: + self._page += 1 + return self._page, None diff --git a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml index dc482bae0f68..658c635cf206 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml +++ b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml @@ -11,11 +11,11 @@ definitions: field_path: - "data" - "{{ parameters['name'] }}" + requester: type: CustomRequester class_name: "source_monday.MondayGraphqlRequester" url_base: "https://api.monday.com/v2" - http_method: "GET" authenticator: type: BearerAuthenticator api_token: "{{ config.get('credentials', {}).get('api_token') if config.get('credentials', {}).get('auth_type') == 'api_token' else config.get('credentials', {}).get('access_token') if config.get('credentials', {}).get('auth_type') == 'oauth2.0' else config.get('api_token', '') }}" @@ -49,6 +49,7 @@ definitions: action: RETRY backoff_strategies: - type: ExponentialBackoffStrategy + default_paginator: type: "DefaultPaginator" pagination_strategy: @@ -62,17 +63,20 @@ definitions: $ref: "#/definitions/requester" paginator: $ref: "#/definitions/default_paginator" + base_stream: retriever: $ref: "#/definitions/retriever" schema_loader: $ref: "#/definitions/schema_loader" primary_key: "id" + base_nopagination_stream: retriever: $ref: "#/definitions/retriever" paginator: type: NoPagination + tags_stream: $ref: "#/definitions/base_nopagination_stream" $parameters: @@ -105,6 +109,12 @@ definitions: class_name: "source_monday.item_pagination_strategy.ItemPaginationStrategy" type: "CustomPaginationStrategy" + cursor_paginator: + $ref: "#/definitions/default_paginator" + pagination_strategy: + class_name: "source_monday.item_pagination_strategy.ItemCursorPaginationStrategy" + type: "CustomPaginationStrategy" + activity_logs_stream: description: "https://developers.intercom.com/intercom-api-reference/reference/scroll-over-all-companies" incremental_sync: @@ -173,12 +183,13 @@ definitions: page_size: 20 nested_items_per_page: 20 parent_key: "pulse_id" - field_path: ["data", "items", "*"] - additional_field_path: ["data", "boards", "*", "items", "*"] + field_path: ["data", "boards", "*", "items_page", "items", "*"] # for first and further incremental pagination responses + field_path_pagination: ["data", "next_items_page", "items", "*"] # for cursor pagination responses + field_path_incremental: ["data", "items", "*"] # for incremental sync responses retriever: $ref: "#/definitions/base_stream/retriever" paginator: - $ref: "#/definitions/double_paginator" + $ref: "#/definitions/cursor_paginator" record_selector: $ref: "#/definitions/selector" extractor: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/run.py b/airbyte-integrations/connectors/source-monday/source_monday/run.py new file mode 100644 index 000000000000..9ef2d048d213 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_monday import SourceMonday + + +def run(): + source = SourceMonday() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json index da156dbb7f61..992636621db7 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json @@ -2,6 +2,8 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "board_kind": { "type": ["null", "string"] }, "type": { "type": ["null", "string"] }, "columns": { @@ -37,26 +39,23 @@ } } }, - "id": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, "owners": { "type": ["null", "array"], "items": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, "creator": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "permissions": { "type": ["null", "string"] }, - "pos": { "type": ["null", "string"] }, "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], @@ -64,7 +63,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, @@ -113,7 +112,7 @@ "workspace": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json index 27f02d52354f..fb682608e71b 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json @@ -2,6 +2,8 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "assets": { "type": ["null", "array"], "items": { @@ -18,7 +20,7 @@ "uploaded_by": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "url": { "type": ["null", "string"] }, @@ -38,13 +40,11 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "additional_info": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, "id": { "type": ["null", "string"] }, "text": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, "type": { "type": ["null", "string"] }, - "value": { "type": ["null", "string"] } + "value": { "type": ["null", "string"] }, + "display_value": { "type": ["null", "string"] } } } }, @@ -56,8 +56,6 @@ "id": { "type": ["null", "string"] } } }, - "id": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, "parent_item": { "type": ["null", "object"], "properties": { @@ -71,7 +69,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json index c96a58d1d42a..e1a4faeb63db 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "color": { "type": ["null", "string"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json index 16cb865fcc92..0bccac5fd4fa 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json @@ -11,7 +11,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json index d0e004a69982..8dc809329358 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json @@ -18,7 +18,7 @@ "uploaded_by": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "url": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json index a064bdc3f4bc..bd2347a4fc2b 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json @@ -8,7 +8,7 @@ "join_date": { "type": ["null", "string"], "format": "date" }, "email": { "type": ["null", "string"] }, "enabled": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "is_admin": { "type": ["null", "boolean"] }, "is_guest": { "type": ["null", "boolean"] }, "is_pending": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json index af7bf79b2d97..3f8439055873 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json @@ -4,14 +4,14 @@ "properties": { "created_at": { "type": ["null", "string"], "format": "date-time" }, "description": { "type": ["null", "string"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, "state": { "type": ["null", "string"] }, "account_product": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] } } }, @@ -21,7 +21,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, @@ -43,7 +43,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } @@ -54,7 +54,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } @@ -65,7 +65,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/config.py new file mode 100644 index 000000000000..e64add53f704 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/config.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._credentials: Dict[str, str] = {} + + def with_oauth_credentials(self, client_id: str, client_secret: str, access_token: str, subdomain: str) -> "ConfigBuilder": + self._credentials["auth_type"] = "oauth2.0" + self._credentials["client_id"] = client_id + self._credentials["client_secret"] = client_secret + self._credentials["access_token"] = access_token + self._credentials["subdomain"] = subdomain + return self + + def with_api_token_credentials(self, api_token: str) -> "ConfigBuilder": + self._credentials["api_token"] = api_token + self._credentials["auth_type"] = "api_token" + return self + + def build(self) -> Dict[str, Any]: + config = {} + if self._credentials: + config["credentials"] = self._credentials + return config diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/__init__.py new file mode 100644 index 000000000000..85ff5a122786 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/__init__.py @@ -0,0 +1 @@ +from .teams_requests_builder import TeamsRequestBuilder diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py new file mode 100644 index 000000000000..3dd017d476b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import abc +from typing import Any, Dict, Optional + +from airbyte_cdk.test.mock_http import HttpRequest + +from .request_authenticators.authenticator import Authenticator + + +class MondayRequestBuilder(abc.ABC): + @property + @abc.abstractmethod + def url(self) -> str: + """A url""" + + @property + @abc.abstractmethod + def query_params(self) -> Dict[str, Any]: + """Query params""" + + @property + @abc.abstractmethod + def headers(self) -> Dict[str, Any]: + """Headers""" + + @property + @abc.abstractmethod + def request_body(self) -> Optional[str]: + """A request body""" + + def build(self) -> HttpRequest: + return HttpRequest( + url=self.url, + query_params=self.query_params, + headers=self.headers, + body=self.request_body + ) + + +class MondayBaseRequestBuilder(MondayRequestBuilder): + def __init__(self, resource: str = "") -> None: + self._resource: str = resource + self._authenticator: str = None + + @property + def url(self) -> str: + return f"https://api.monday.com/v2/{self._resource}" + + @property + def headers(self) -> Dict[str, Any]: + return (super().headers or {}) | { + "Authorization": self._authenticator.client_access_token, + } + + @property + def request_body(self): + return super().request_body + + def with_authenticator(self, authenticator: Authenticator) -> "MondayBaseRequestBuilder": + self._authenticator: Authenticator = authenticator + return self diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/__init__.py new file mode 100644 index 000000000000..c670d8c3e6c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/__init__.py @@ -0,0 +1 @@ +from .api_token_authenticator import ApiTokenAuthenticator diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/api_token_authenticator.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/api_token_authenticator.py new file mode 100644 index 000000000000..dcd4fdcf3b54 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/api_token_authenticator.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import base64 + +from .authenticator import Authenticator + + +class ApiTokenAuthenticator(Authenticator): + def __init__(self, api_token: str) -> None: + super().__init__() + self._api_token = api_token + + @property + def client_access_token(self) -> str: + return f"Bearer {self._api_token}" diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/authenticator.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/authenticator.py new file mode 100644 index 000000000000..0d85143998b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/request_authenticators/authenticator.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import abc + + +class Authenticator(abc.ABC): + @abc.abstractproperty + def client_access_token(self) -> str: + """""" diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/teams_requests_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/teams_requests_builder.py new file mode 100644 index 000000000000..4fa39ecdcd2b --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/teams_requests_builder.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from .base_requests_builder import MondayBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class TeamsRequestBuilder(MondayBaseRequestBuilder): + @classmethod + def teams_endpoint(cls, authenticator: Authenticator) -> "TeamsRequestBuilder": + return cls().with_authenticator(authenticator) + + @property + def query_params(self): + params = super().query_params or {} + params["query"] = "query{teams{id,name,picture_url,users{id}}}" + return params diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/__init__.py new file mode 100644 index 000000000000..3b3481e236d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/__init__.py @@ -0,0 +1,2 @@ +from .teams_response_builder import TeamsResponseBuilder +from .error_response_builder import ErrorResponseBuilder diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py new file mode 100644 index 000000000000..a746a8d68a05 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py @@ -0,0 +1,18 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + + +class ErrorResponseBuilder: + def __init__(self, status_code: int): + self._status_code: int = status_code + + @classmethod + def response_with_status(cls, status_code) -> "ErrorResponseBuilder": + return cls(status_code) + + def build(self) -> HttpResponse: + return HttpResponse(json.dumps(find_template(str(self._status_code), __file__)), self._status_code) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/__init__.py new file mode 100644 index 000000000000..dcbc806ca42a --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/__init__.py @@ -0,0 +1 @@ +from .teams_record_builder import TeamsRecordBuilder diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/record_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/record_builder.py new file mode 100644 index 000000000000..8833818f3926 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/record_builder.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import Path, RecordBuilder, find_template + + +class MondayRecordBuilder(RecordBuilder): + @staticmethod + def extract_record(resource: str, execution_folder: str, data_field: Path): + return data_field.extract(find_template(resource=resource, execution_folder=execution_folder)) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/teams_record_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/teams_record_builder.py new file mode 100644 index 000000000000..38f5c9219574 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/records/teams_record_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .record_builder import MondayRecordBuilder + + +class TeamsRecordBuilder(MondayRecordBuilder): + @classmethod + def teams_record(cls) -> "TeamsRecordBuilder": + record_template = cls.extract_record("teams", __file__, NestedPath(["data", "teams", 0])) + return cls(record_template, FieldPath("id"), None) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/teams_response_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/teams_response_builder.py new file mode 100644 index 000000000000..6b479b9aa0b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/teams_response_builder.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, NestedPath, find_template + + +class TeamsResponseBuilder(HttpResponseBuilder): + @classmethod + def teams_response(cls) -> "TeamsResponseBuilder": + return cls(find_template("teams", __file__), NestedPath(["data", "teams"]), None) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/test_teams_stream.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/test_teams_stream.py new file mode 100644 index 000000000000..ff8322903b5c --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/test_teams_stream.py @@ -0,0 +1,135 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase +from unittest.mock import patch + +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .config import ConfigBuilder +from .monday_requests import TeamsRequestBuilder +from .monday_requests.request_authenticators import ApiTokenAuthenticator +from .monday_responses import ErrorResponseBuilder, TeamsResponseBuilder +from .monday_responses.records import TeamsRecordBuilder +from .utils import get_log_messages_by_log_level, read_stream + + +class TestTeamsStreamFullRefresh(TestCase): + @property + def _config(self): + return ConfigBuilder().with_api_token_credentials("api-token").build() + + def get_authenticator(self, config): + return ApiTokenAuthenticator(api_token=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_one_page_when_read_teams_then_return_records(self, http_mocker): + """ + A normal full refresh sync without pagination + """ + api_token_authenticator = self.get_authenticator(self._config) + + http_mocker.get( + TeamsRequestBuilder.teams_endpoint(api_token_authenticator).build(), + TeamsResponseBuilder.teams_response().with_record(TeamsRecordBuilder.teams_record()).build() + ) + + output = read_stream("teams", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_retryable_error_and_one_page_when_read_teams_then_return_records(self, http_mocker): + """ + A full refresh sync without pagination completes successfully after one retry + """ + api_token_authenticator = self.get_authenticator(self._config) + + http_mocker.get( + TeamsRequestBuilder.teams_endpoint(api_token_authenticator).build(), + [ + ErrorResponseBuilder.response_with_status(200).build(), + TeamsResponseBuilder.teams_response().with_record(TeamsRecordBuilder.teams_record()).build() + ] + ) + + with patch('time.sleep', return_value=None): + output = read_stream("teams", SyncMode.full_refresh, self._config) + + assert len(output.records) == 1 + + error_logs = [ + error for error in get_log_messages_by_log_level(output.logs, LogLevel.INFO) + if f'Response Code: 200, Response Text: {json.dumps({"error_code": "ComplexityException", "status_code": 200})}' in error + ] + assert len(error_logs) == 1 + + @HttpMocker() + def test_given_retryable_error_when_read_teams_then_stop_syncing(self, http_mocker): + """ + A full refresh sync without pagination give up after 6 retries + """ + api_token_authenticator = self.get_authenticator(self._config) + + http_mocker.get( + TeamsRequestBuilder.teams_endpoint(api_token_authenticator).build(), + ErrorResponseBuilder.response_with_status(200).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("teams", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = [ + error for error in get_log_messages_by_log_level(output.logs, LogLevel.INFO) + if f'Response Code: 200, Response Text: {json.dumps({"error_code": "ComplexityException", "status_code": 200})}' in error + ] + assert len(error_logs) == 5 + + @HttpMocker() + def test_given_retryable_500_error_when_read_teams_then_stop_syncing(self, http_mocker): + """ + A full refresh sync without pagination give up after 6 retries + """ + api_token_authenticator = self.get_authenticator(self._config) + + http_mocker.get( + TeamsRequestBuilder.teams_endpoint(api_token_authenticator).build(), + ErrorResponseBuilder.response_with_status(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("teams", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = [ + error for error in get_log_messages_by_log_level(output.logs, LogLevel.INFO) + if f'Response Code: 500, Response Text: {json.dumps({"error_message": "Internal server error", "status_code": 500})}' in error + ] + assert len(error_logs) == 5 + + @HttpMocker() + def test_given_403_error_when_read_teams_then_ignore_the_stream(self, http_mocker): + """ + A full refresh sync without pagination ignore failed stream + """ + api_token_authenticator = self.get_authenticator(self._config) + + http_mocker.get( + TeamsRequestBuilder.teams_endpoint(api_token_authenticator).build(), + ErrorResponseBuilder.response_with_status(403).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("teams", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = [ + error for error in get_log_messages_by_log_level(output.logs, LogLevel.INFO) + if f'Ignoring response for failed request with error message None' in error + ] + assert len(error_logs) == 1 diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py new file mode 100644 index 000000000000..473ddcc61f0f --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import operator +from typing import Any, Dict, List, Optional + +from airbyte_cdk.models import AirbyteMessage +from airbyte_cdk.models import Level as LogLevel +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_protocol.models import SyncMode +from source_monday import SourceMonday + + +def read_stream( + stream_name: str, + sync_mode: SyncMode, + config: Dict[str, Any], + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build() + return read(SourceMonday(), config, catalog, state, expecting_exception) + + +def get_log_messages_by_log_level(logs: List[AirbyteMessage], log_level: LogLevel) -> List[str]: + return map(operator.attrgetter("log.message"), filter(lambda x: x.log.level == log_level, logs)) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/200.json b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/200.json new file mode 100644 index 000000000000..825533274645 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/200.json @@ -0,0 +1,4 @@ +{ + "error_code": "ComplexityException", + "status_code": 200 +} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/403.json b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/403.json new file mode 100644 index 000000000000..2a6b669b275a --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/403.json @@ -0,0 +1,4 @@ +{ + "error_code": "UserUnauthorizedException", + "status_code": 403 +} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/500.json b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/500.json new file mode 100644 index 000000000000..c1a1a6085825 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/500.json @@ -0,0 +1,4 @@ +{ + "error_message": "Internal server error", + "status_code": 500 +} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/teams.json b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/teams.json new file mode 100644 index 000000000000..27b005a1d331 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/resource/http/response/teams.json @@ -0,0 +1,17 @@ +{ + "data": { + "teams": [ + { + "id": "520480", + "name": "Team1", + "picture_url": "https://cdn.monday.com/images/dapulse_team_default.png", + "users": [ + { + "id": "25479561" + } + ] + } + ] + }, + "account_id": 10239240 +} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py index 49be96073f1e..670aff5e4e44 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock import pytest +from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from airbyte_cdk.sources.streams import Stream from requests import Response @@ -99,3 +100,87 @@ def test_null_records(caplog): {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z", "updated_at_int": 1692095454}, ] assert records == expected_records + + +@pytest.fixture +def mock_parent_stream(): + + def mock_parent_stream_slices(*args, **kwargs): + return iter([{"ids": [123]}]) + + mock_stream = MagicMock(spec=Stream) + mock_stream.primary_key = "id" # Example primary key + mock_stream.stream_slices = mock_parent_stream_slices + mock_stream.parent_config = ParentStreamConfig( + stream=mock_stream, + parent_key="id", + partition_field="parent_stream_id", + parameters={}, + config={}, + ) + + return mock_stream + +@pytest.mark.parametrize("stream_state, parent_records, expected_slices", + [ + ({}, [], [{}]), + ( + {"updated_at": "2022-01-01T00:00:00Z"}, + [AirbyteMessage( + type=Type.RECORD, + record={ "data": {"id": 123, "name": "Sample Record", "updated_at": "2023-01-01T00:00:00Z"}, "stream": "projects", "emitted_at": 1632095449} + )], + [{'parent_stream_id': [123]}] + ), + ( + {"updated_at": "2022-01-01T00:00:00Z"}, + AirbyteMessage(type=Type.LOG), + [] + ) + ], + ids=[ + "no stream state", + "successfully read parent record", + "skip non_record AirbyteMessage" + ] +) +def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, expected_slices): + + slicer = IncrementalSubstreamSlicer( + config={}, + parameters={}, + cursor_field="updated_at", + parent_stream_configs=[mock_parent_stream.parent_config], + nested_items_per_page=10 + ) + + mock_parent_stream.read_records = MagicMock(return_value=parent_records) + slicer.parent_cursor_field = "updated_at" + + slices = list(slicer.read_parent_stream( + sync_mode=SyncMode.full_refresh, + cursor_field="updated_at", + stream_state=stream_state + )) + + assert slices == expected_slices + + +def test_set_initial_state(): + + slicer = IncrementalSubstreamSlicer( + config={}, + parameters={}, + cursor_field="updated_at_int", + parent_stream_configs=[MagicMock(parent_stream_name="parent_stream")], + nested_items_per_page=10 + ) + + initial_stream_state = { + "updated_at_int": 1662459010, + "parent_stream": {"parent_cursor_field": 1662459011} + } + + expected_state = { "updated_at_int": 1662459010 } + slicer.set_initial_state(initial_stream_state) + assert slicer._state == expected_state diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py new file mode 100644 index 000000000000..869c7ab7bbca --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_monday.extractor import MondayActivityExtractor, MondayIncrementalItemsExtractor + + +def test_extract_records(): + # Mock the response + response = MagicMock() + response_body = { + "data": { + "boards": [ + { + "activity_logs": [ + { + "data": "{\"pulse_id\": 123}", + "entity": "pulse", + "created_at": "16367386880000000" + } + ] + } + ] + } + } + + response.json.return_value = response_body + extractor = MondayActivityExtractor(parameters={}) + records = extractor.extract_records(response) + + # Assertions + assert len(records) == 1 + assert records[0]["pulse_id"] == 123 + assert records[0]["created_at_int"] == 1636738688 + + +def test_extract_records_incremental(): + # Mock the response + response = MagicMock() + response_body = { + "data": { + "boards": [ + { + "id": 1 + } + ] + } + } + + response.json.return_value = response_body + extractor = MondayIncrementalItemsExtractor( + parameters={}, + field_path=["data", "ccccc"], + config=MagicMock(), + field_path_pagination=["data", "bbbb"], + field_path_incremental=["data", "boards", "*"] + ) + records = extractor.extract_records(response) + + # Assertions + assert records == [{'id': 1}] diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index 2658134d6cb5..b4f46146b6bc 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -5,7 +5,9 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod +from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader from source_monday import MondayGraphqlRequester nested_object_schema = { @@ -51,8 +53,8 @@ nested_array_schema, "items", {}, - {"query": "query{boards(limit:100,page:2){items(limit:100,page:1){root{nested{nested_of_nested}},sibling}}}"}, - {"next_page_token": (2, 1)}, + {"query": 'query{next_items_page(limit:100,cursor:"cursor_bla"){cursor,items{root{nested{nested_of_nested}},sibling}}}'}, + {"next_page_token": (2, "cursor_bla")}, id="test_get_request_params_produces_graphql_query_for_items_stream", ), pytest.param( @@ -89,3 +91,69 @@ def test_get_request_params(mocker, input_schema, graphql_query, stream_name, co config=config, ) assert requester.get_request_params(stream_state={}, stream_slice={}, next_page_token=next_page_token) == graphql_query + + +@pytest.fixture +def monday_requester(): + return MondayGraphqlRequester( + name="a name", + url_base="https://api.monday.com/v2", + path="a-path", + config={}, + parameters={"name": "activity_logs"}, + limit=InterpolatedString.create("100", parameters={"name": "activity_logs"}), + nested_limit=InterpolatedString.create("100", parameters={"name": "activity_logs"}), + ) + +def test_get_schema_root_properties(mocker, monday_requester): + mock_schema = { + "properties": { + "updated_at_int": {"type": "integer"}, + "created_at_int": {"type": "integer"}, + "pulse_id": {"type": "integer"}, + "board_id": {"type": "integer"}, + "other_field": {"type": "string"}, + "yet_another_field": {"type": "boolean"} + } + } + + mocker.patch.object(JsonFileSchemaLoader, 'get_json_schema', return_value=mock_schema) + requester = monday_requester + result_schema = requester._get_schema_root_properties() + + assert result_schema == { + "other_field": { "type": "string" }, + "yet_another_field": { "type": "boolean" } + } + + +def test_build_activity_query(mocker, monday_requester): + + mock_stream_state = { "updated_at_int": 1636738688 } + object_arguments = { "stream_state": mock_stream_state } + mocker.patch.object(MondayGraphqlRequester, '_get_object_arguments', return_value="stream_state:{{ stream_state['updated_at_int'] }}") + requester = monday_requester + + result = requester._build_activity_query(object_name="activity_logs", field_schema={}, sub_page=None, **object_arguments) + assert result == "boards(stream_state:{{ stream_state['updated_at_int'] }}){activity_logs(stream_state:{{ stream_state['updated_at_int'] }}){}}" + + +def test_build_items_incremental_query(monday_requester): + + object_name = "test_items" + field_schema = { + "id": {"type": "integer"}, + "name": {"type": "string"}, + } + stream_slice = {"ids": [1, 2, 3]} + + built_query = monday_requester._build_items_incremental_query(object_name, field_schema, stream_slice) + + assert built_query == 'items(limit:100,ids:[1, 2, 3]){id,name}' + + +def test_get_request_headers(monday_requester): + + headers = monday_requester.get_request_headers() + + assert headers == {'API-Version': '2024-01'} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py index 16988efc620d..979c72284713 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock import pytest -from source_monday.item_pagination_strategy import ItemPaginationStrategy +from source_monday.item_pagination_strategy import ItemCursorPaginationStrategy, ItemPaginationStrategy @pytest.mark.parametrize( @@ -40,3 +40,42 @@ def test_item_pagination_strategy(response_json, last_records, expected): response.json.return_value = response_json assert strategy.next_page_token(response, last_records) == expected + +@pytest.mark.parametrize( + ("response_json", "last_records", "expected"), + [ + pytest.param( + {"data": {"boards": [{"items_page": {"cursor": "bla", "items":[{"id": "1"}]}}]}}, + [], + (1, 'bla'), + id="test_cursor_in_first_request", + ), + pytest.param( + {"data": {"next_items_page": {"cursor": "bla2", "items":[{"id": "1"}]}}}, + [], + (1, 'bla2'), + id="test_cursor_in_next_page", + ), + pytest.param( + {"data": {"next_items_page": {"items": [{"id": "1"}]}}}, + [], + (2, None), + id="test_next_board_page", + ), + pytest.param( + {"data": {"boards": []}}, + [], + None, + id="test_end_pagination", + ), + ], +) +def test_item_cursor_pagination_strategy(response_json, last_records, expected): + strategy = ItemCursorPaginationStrategy( + page_size=1, + parameters={"items_per_page": 1}, + ) + response = MagicMock() + response.json.return_value = response_json + + assert strategy.next_page_token(response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle deleted file mode 100644 index d2557caffc6b..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -plugins { - id 'application' - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-sources'] - useLocalCdk = false -} - -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - -application { - mainClass = 'io.airbyte.integrations.source.mongodb.MongodbSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-integrations:connectors:source-mongodb-v2') - implementation project(':airbyte-integrations:connectors:destination-mongodb') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') - - - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' -} diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java deleted file mode 100644 index 834c97308143..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.base.Source; -import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.mongodb.MongoUtils; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MongodbSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongodbSourceStrictEncrypt.class); - - public MongodbSourceStrictEncrypt() { - super(new MongoDbSource()); - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE); - final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); - // If the MongoDb source connector is not set up to use a TLS connection, then we should fail the - // check. - if (instance.equals(MongoInstanceType.STANDALONE) && !MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig)) { - throw new ConfigErrorException("TLS connection must be used to read from MongoDB."); - } - - return super.check(config); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - // removing tls property for a standalone instance to disable possibility to switch off a tls - // connection - ((ObjectNode) spec.getConnectionSpecification().get("properties").get("instance_type").get("oneOf").get(0).get("properties")).remove("tls"); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new MongodbSourceStrictEncrypt(); - LOGGER.info("starting source: {}", MongodbSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MongodbSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java deleted file mode 100644 index 6e47cc8693bd..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.mongodb.client.MongoCollection; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.mongodb.MongoDatabase; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import org.bson.BsonArray; -import org.bson.BsonString; -import org.bson.Document; -import org.junit.jupiter.api.Test; - -public class MongodbSourceStrictEncryptAcceptanceTest extends SourceAcceptanceTest { - - private static final String DATABASE_NAME = "test"; - private static final String COLLECTION_NAME = "acceptance_test1"; - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - private static final String INSTANCE_TYPE = "instance_type"; - - protected JsonNode config; - protected MongoDatabase database; - - @Override - protected String getImageName() { - return "airbyte/source-mongodb-strict-encrypt:dev"; - } - - @Override - protected JsonNode getConfig() throws Exception { - return config; - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - - config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - - final String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", - config.get("user").asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - config.get("instance_type").get("cluster_url").asText(), - config.get(JdbcUtils.DATABASE_KEY).asText()); - - database = new MongoDatabase(connectionString, DATABASE_NAME); - - final MongoCollection collection = database.createCollection(COLLECTION_NAME); - final var doc1 = new Document("id", "0001").append("name", "Test") - .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) - .append("double_test", 100.12).append("int_test", 100); - final var doc2 = new Document("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201); - final var doc3 = new Document("id", "0003").append("name", "Source").append("test", null) - .append("double_test", 212.11).append("int_test", 302); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - for (final String collectionName : database.getCollectionNames()) { - database.getDatabase().getCollection(collectionName).drop(); - } - database.close(); - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("_id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withCursorField(List.of("_id")) - .withStream(CatalogHelpers.createAirbyteStream( - DATABASE_NAME + "." + COLLECTION_NAME, - Field.of("_id", JsonSchemaType.STRING), - Field.of("id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("empty_test", JsonSchemaType.STRING), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("int_test", JsonSchemaType.NUMBER)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) - .withDefaultCursorField(List.of("_id"))))); - } - - @Override - protected JsonNode getState() throws Exception { - return Jsons.jsonNode(new HashMap<>()); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = new MongodbSourceStrictEncrypt().spec(); - final ConnectorSpecification expected = getSpec(); - - assertEquals(expected, actual); - } - - @Test - void testCheck() throws Exception { - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", MongoInstanceType.STANDALONE.getType()) - .put("tls", false) - .build()); - - final JsonNode invalidStandaloneConfig = Jsons.clone(getConfig()); - - ((ObjectNode) invalidStandaloneConfig).put(INSTANCE_TYPE, instanceConfig); - - final Throwable throwable = catchThrowable(() -> new MongodbSourceStrictEncrypt().check(invalidStandaloneConfig)); - assertThat(throwable).isInstanceOf(ConfigErrorException.class); - assertThat(((ConfigErrorException) throwable) - .getDisplayMessage() - .contains("TLS connection must be used to read from MongoDB.")); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 3aa358d11397..b160c88b04b3 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -1,17 +1,14 @@ plugins { - id 'application' id 'airbyte-java-connector' - alias(libs.plugins.kotlin.jvm) + id 'org.jetbrains.kotlin.jvm' version '1.9.22' } airbyteJavaConnector { - cdkVersionRequired = '0.7.1' - features = ['db-sources'] + cdkVersionRequired = '0.20.6' + features = ['db-sources', 'datastore-mongo'] useLocalCdk = false } -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.mongodb.MongoDbSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -35,39 +32,32 @@ sourceSets { } } -dependencies { - implementation libs.mongo.driver.sync - - testImplementation libs.testcontainers.mongodb +java { + compileJava { + options.compilerArgs += "-Xlint:-try,-rawtypes" + } +} - integrationTestJavaImplementation libs.apache.commons.lang - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') +dependencies { + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'io.debezium:debezium-connector-mongodb:2.4.0.Final' - dataGeneratorImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testImplementation 'org.testcontainers:mongodb:1.19.0' dataGeneratorImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') - dataGeneratorImplementation libs.mongo.driver.sync - dataGeneratorImplementation libs.kotlin.logging - dataGeneratorImplementation libs.kotlinx.cli - dataGeneratorImplementation (libs.java.faker) { - exclude module: 'snakeyaml' - } - dataGeneratorImplementation libs.jackson.databind - dataGeneratorImplementation libs.bundles.slf4j - dataGeneratorImplementation libs.slf4j.simple - dataGeneratorImplementation libs.kotlinx.cli.jvm - dataGeneratorImplementation 'org.yaml:snakeyaml:2.2' - - debeziumTestImplementation libs.debezium.api - debeziumTestImplementation libs.debezium.embedded - debeziumTestImplementation libs.debezium.sqlserver - debeziumTestImplementation libs.debezium.mysql - debeziumTestImplementation libs.debezium.postgres - debeziumTestImplementation libs.debezium.mongodb - debeziumTestImplementation libs.bundles.slf4j - debeziumTestImplementation libs.slf4j.simple - debeziumTestImplementation libs.kotlinx.cli.jvm - debeziumTestImplementation libs.spotbugs.annotations + dataGeneratorImplementation platform('com.fasterxml.jackson:jackson-bom:2.15.2') + dataGeneratorImplementation 'com.fasterxml.jackson.core:jackson-databind' + dataGeneratorImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + + dataGeneratorImplementation ('com.github.javafaker:javafaker:1.0.2') { exclude module: 'snakeyaml' } + dataGeneratorImplementation 'io.github.oshai:kotlin-logging-jvm:5.1.0' + dataGeneratorImplementation 'org.jetbrains.kotlinx:kotlinx-cli-jvm:0.3.5' + dataGeneratorImplementation 'org.mongodb:mongodb-driver-sync:4.10.2' + + debeziumTestImplementation 'io.debezium:debezium-embedded:2.4.0.Final' + debeziumTestImplementation 'io.debezium:debezium-connector-mongodb:2.4.0.Final' + debeziumTestImplementation 'org.jetbrains.kotlinx:kotlinx-cli-jvm:0.3.5' + debeziumTestImplementation 'com.github.spotbugs:spotbugs-annotations:4.7.3' } /* diff --git a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json index d20aff3bcb5f..bd7f8b04829e 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json @@ -67,6 +67,14 @@ "default": "admin", "examples": ["admin"], "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } }, @@ -117,6 +125,14 @@ "default": "admin", "examples": ["admin"], "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } } @@ -127,7 +143,7 @@ "title": "Initial Waiting Time in Seconds (Advanced)", "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds.", "default": 300, - "order": 7, + "order": 8, "min": 120, "max": 1200, "group": "advanced" @@ -137,7 +153,7 @@ "title": "Size of the queue (Advanced)", "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", "default": 10000, - "order": 8, + "order": 9, "min": 1000, "max": 10000, "group": "advanced" @@ -147,10 +163,19 @@ "title": "Document discovery sample size (Advanced)", "description": "The maximum number of documents to sample when attempting to discover the unique fields for a collection.", "default": 10000, - "order": 9, - "minimum": 1000, + "order": 10, + "minimum": 10, "maximum": 100000, "group": "advanced" + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 11, + "group": "advanced" } }, "groups": [ diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index d7171d2fcb5d..7cfabf8d314a 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -1,11 +1,11 @@ data: ab_internal: ql: 200 - sl: 100 + sl: 200 connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 1.0.11 + dockerImageTag: 1.2.13 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java index 3cee7c45a280..f9863945d8fd 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java @@ -27,6 +27,10 @@ import java.util.List; import java.util.Optional; import org.bson.BsonDocument; +import org.bson.BsonInt32; +import org.bson.BsonInt64; +import org.bson.BsonObjectId; +import org.bson.BsonString; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; @@ -51,7 +55,8 @@ public List> getIterators( final MongoDatabase database, final MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector, final Instant emittedAt, - final int checkpointInterval) { + final int checkpointInterval, + final boolean isEnforceSchema) { return streams .stream() .peek(airbyteStream -> { @@ -85,21 +90,33 @@ public List> getIterators( // "where _id > [last saved state] order by _id ASC". // If no state exists, it will create a query akin to "where 1=1 order by _id ASC" final Bson filter = existingState - // TODO add type support here when we add support for _id fields that are not ObjectId types - .map(state -> Filters.gt(MongoConstants.ID_FIELD, new ObjectId(state.id()))) + .map(state -> Filters.gt(MongoConstants.ID_FIELD, + switch (state.idType()) { + case STRING -> new BsonString(state.id()); + case OBJECT_ID -> new BsonObjectId(new ObjectId(state.id())); + case INT -> new BsonInt32(Integer.parseInt(state.id())); + case LONG -> new BsonInt64(Long.parseLong(state.id())); + })) // if nothing was found, return a new BsonDocument .orElseGet(BsonDocument::new); - final var cursor = collection.find() + // When schema is enforced we query for the selected fields + // Otherwise we retreive the entire set of fields + final var cursor = isEnforceSchema ? collection.find() .filter(filter) .projection(fields) .sort(Sorts.ascending(MongoConstants.ID_FIELD)) .allowDiskUse(true) - .cursor(); + .cursor() + : collection.find() + .filter(filter) + .sort(Sorts.ascending(MongoConstants.ID_FIELD)) + .allowDiskUse(true) + .cursor(); final var stateIterator = new MongoDbStateIterator(cursor, stateManager, Optional.ofNullable(cdcConnectorMetadataInjector), - airbyteStream, emittedAt, checkpointInterval, MongoConstants.CHECKPOINT_DURATION); + airbyteStream, emittedAt, checkpointInterval, MongoConstants.CHECKPOINT_DURATION, isEnforceSchema); return AutoCloseableIterators.fromIterator(stateIterator, cursor::close, null); }) .toList(); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java index aeab43f94afe..37f0c51dd1ba 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java @@ -4,7 +4,10 @@ package io.airbyte.integrations.source.mongodb; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -12,6 +15,7 @@ import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.SyncMode; import java.util.List; +import java.util.stream.Collectors; /** * Collection of utility methods for generating the {@link AirbyteCatalog}. @@ -55,6 +59,21 @@ public static AirbyteStream buildAirbyteStream(final String streamName, final St .withSourceDefinedPrimaryKey(List.of(List.of(DEFAULT_PRIMARY_KEY)))); } + /** + * Builds an {@link AirbyteStream} with the correct configuration for this source, in schemaless + * mode. All fields are stripped out and the only fields kept are _id, _data, and the CDC fields. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @param fields The fields associated with the stream. + * @return The configured {@link AirbyteStream} for this source. + */ + public static AirbyteStream buildSchemalessAirbyteStream(final String streamName, final String streamNamespace, final List fields) { + // The packed airbyte catalog should only contain the _id field. + final List idFieldList = fields.stream().filter(field -> field.getName().equals(MongoConstants.ID_FIELD)).collect(Collectors.toList()); + return addDataMetadataColumn(buildAirbyteStream(streamName, streamNamespace, idFieldList)); + } + /** * Adds CDC metadata columns to the stream. * @@ -68,4 +87,23 @@ private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { return stream; } + /** + * Adds the data metadata columns to the stream, for schemaless (packed) mode. + * + * @param stream An {@link AirbyteStream}. + * @return The modified {@link AirbyteStream}. + */ + private static AirbyteStream addDataMetadataColumn(final AirbyteStream stream) { + final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); + final ObjectNode properties = (ObjectNode) jsonSchema.get(AIRBYTE_STREAM_PROPERTIES); + addSchemalessModeDataColumn(properties); + return stream; + } + + private static ObjectNode addSchemalessModeDataColumn(final ObjectNode properties) { + final JsonNode objectType = Jsons.jsonNode(ImmutableMap.of("type", "object")); + properties.set(MongoConstants.SCHEMALESS_MODE_DATA_FIELD, objectType); + return properties; + } + } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java index c3a363adc63e..7e48aa7468a4 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java @@ -13,7 +13,7 @@ import com.mongodb.ReadPreference; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -50,7 +50,7 @@ public static MongoClient createMongoClient(final MongoDbSourceConfig config) { } private static String buildConnectionString(final MongoDbSourceConfig config) { - return MongoDbDebeziumPropertiesManager.buildConnectionString(config.rawConfig(), true); + return MongoDbDebeziumPropertiesManager.buildConnectionString(config.getDatabaseConfig(), true); } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java index c9b25a4910cd..6fb2bc792b19 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.source.mongodb; import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration; import java.time.Duration; public class MongoConstants { @@ -31,6 +32,14 @@ public class MongoConstants { public static final String QUEUE_SIZE_CONFIGURATION_KEY = "queue_size"; public static final String STORAGE_STATS_KEY = "storageStats"; public static final String USERNAME_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; + public static final String SCHEMA_ENFORCED_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY; + public static final String SCHEMALESS_MODE_DATA_FIELD = Configuration.SCHEMALESS_MODE_DATA_FIELD; + public static final String INITIAL_RECORD_WAITING_TIME_SEC = "initial_waiting_seconds"; + public static final Integer DEFAULT_INITIAL_RECORD_WAITING_TIME_SEC = 300; + + public static final String INVALID_CDC_CURSOR_POSITION_PROPERTY = "invalid_cdc_cursor_position_behavior"; + public static final String FAIL_SYNC_OPTION = "Fail sync"; + public static final String RESYNC_DATA_OPTION = "Re-sync data"; private MongoConstants() {} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java index 57c820eba1fd..d7103f6fe854 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java @@ -6,10 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import com.mongodb.MongoCommandException; import com.mongodb.MongoSecurityException; import com.mongodb.client.MongoClient; import com.mongodb.connection.ClusterType; import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.Source; @@ -40,6 +42,7 @@ public MongoDbSource() { } public static void main(final String[] args) throws Exception { + AirbyteExceptionHandler.addThrowableForDeinterpolation(MongoCommandException.class); final Source source = new MongoDbSource(); LOGGER.info("starting source: {}", MongoDbSource.class); new IntegrationRunner(source).run(args); @@ -103,7 +106,8 @@ public AirbyteCatalog discover(final JsonNode config) { try (final MongoClient mongoClient = createMongoClient(sourceConfig)) { final String databaseName = sourceConfig.getDatabaseName(); final Integer sampleSize = sourceConfig.getSampleSize(); - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, sampleSize); + final boolean isSchemaEnforced = sourceConfig.getEnforceSchema(); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, sampleSize, isSchemaEnforced); return new AirbyteCatalog().withStreams(streams); } } catch (final IllegalArgumentException e) { @@ -119,10 +123,13 @@ public AutoCloseableIterator read(final JsonNode config, final var emittedAt = Instant.now(); final var cdcMetadataInjector = MongoDbCdcConnectorMetadataInjector.getInstance(emittedAt); final var stateManager = MongoDbStateManager.createStateManager(state); + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); + if (catalog != null) { + MongoUtil.checkSchemaModeMismatch(sourceConfig.getEnforceSchema(), + stateManager.getCdcState() != null ? stateManager.getCdcState().schema_enforced() : sourceConfig.getEnforceSchema(), catalog); + } try { - final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); - // WARNING: do not close the client here since it needs to be used by the iterator final MongoClient mongoClient = createMongoClient(sourceConfig); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java index 111bb62ef34e..8f3f572afad2 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java @@ -11,8 +11,13 @@ import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_AUTH_SOURCE; import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_INITIAL_RECORD_WAITING_TIME_SEC; import static io.airbyte.integrations.source.mongodb.MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.INITIAL_RECORD_WAITING_TIME_SEC; +import static io.airbyte.integrations.source.mongodb.MongoConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; import static io.airbyte.integrations.source.mongodb.MongoConstants.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.RESYNC_DATA_OPTION; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.USERNAME_CONFIGURATION_KEY; import com.fasterxml.jackson.databind.JsonNode; @@ -26,33 +31,32 @@ */ public record MongoDbSourceConfig(JsonNode rawConfig) { - /** - * Constructs a new {@link MongoDbSourceConfig} from the provided raw configuration. - * - * @param rawConfig The underlying JSON configuration provided by the connector framework. - * @throws IllegalArgumentException if the raw configuration does not contain the - * {@link MongoConstants#DATABASE_CONFIG_CONFIGURATION_KEY} key. - */ - public MongoDbSourceConfig(final JsonNode rawConfig) { - if (rawConfig.has(DATABASE_CONFIG_CONFIGURATION_KEY)) { - this.rawConfig = rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY); - } else { + public MongoDbSourceConfig { + if (rawConfig == null) { + throw new IllegalArgumentException("MongoDbSourceConfig cannot accept a null config."); + } + if (!rawConfig.hasNonNull(DATABASE_CONFIG_CONFIGURATION_KEY)) { throw new IllegalArgumentException("Database configuration is missing required '" + DATABASE_CONFIG_CONFIGURATION_KEY + "' property."); } } + public JsonNode getDatabaseConfig() { + return rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY); + } + public String getAuthSource() { - return rawConfig.has(AUTH_SOURCE_CONFIGURATION_KEY) ? rawConfig.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(DEFAULT_AUTH_SOURCE) + return getDatabaseConfig().has(AUTH_SOURCE_CONFIGURATION_KEY) ? getDatabaseConfig().get(AUTH_SOURCE_CONFIGURATION_KEY).asText(DEFAULT_AUTH_SOURCE) : DEFAULT_AUTH_SOURCE; } public Integer getCheckpointInterval() { - return rawConfig.has(CHECKPOINT_INTERVAL_CONFIGURATION_KEY) ? rawConfig.get(CHECKPOINT_INTERVAL_CONFIGURATION_KEY).asInt(CHECKPOINT_INTERVAL) + return getDatabaseConfig().has(CHECKPOINT_INTERVAL_CONFIGURATION_KEY) + ? getDatabaseConfig().get(CHECKPOINT_INTERVAL_CONFIGURATION_KEY).asInt(CHECKPOINT_INTERVAL) : CHECKPOINT_INTERVAL; } public String getDatabaseName() { - return rawConfig.has(DATABASE_CONFIGURATION_KEY) ? rawConfig.get(DATABASE_CONFIGURATION_KEY).asText() : null; + return getDatabaseConfig().has(DATABASE_CONFIGURATION_KEY) ? getDatabaseConfig().get(DATABASE_CONFIGURATION_KEY).asText() : null; } public OptionalInt getQueueSize() { @@ -62,15 +66,15 @@ public OptionalInt getQueueSize() { } public String getPassword() { - return rawConfig.has(PASSWORD_CONFIGURATION_KEY) ? rawConfig.get(PASSWORD_CONFIGURATION_KEY).asText() : null; + return getDatabaseConfig().has(PASSWORD_CONFIGURATION_KEY) ? getDatabaseConfig().get(PASSWORD_CONFIGURATION_KEY).asText() : null; } public String getUsername() { - return rawConfig.has(USERNAME_CONFIGURATION_KEY) ? rawConfig.get(USERNAME_CONFIGURATION_KEY).asText() : null; + return getDatabaseConfig().has(USERNAME_CONFIGURATION_KEY) ? getDatabaseConfig().get(USERNAME_CONFIGURATION_KEY).asText() : null; } public boolean hasAuthCredentials() { - return rawConfig.has(USERNAME_CONFIGURATION_KEY) && rawConfig.has(PASSWORD_CONFIGURATION_KEY); + return getDatabaseConfig().has(USERNAME_CONFIGURATION_KEY) && getDatabaseConfig().has(PASSWORD_CONFIGURATION_KEY); } public Integer getSampleSize() { @@ -81,4 +85,26 @@ public Integer getSampleSize() { } } + public boolean getEnforceSchema() { + return getDatabaseConfig().has(SCHEMA_ENFORCED_CONFIGURATION_KEY) ? getDatabaseConfig().get(SCHEMA_ENFORCED_CONFIGURATION_KEY).asBoolean(true) + : true; + } + + public Integer getInitialWaitingTimeSeconds() { + if (rawConfig.has(INITIAL_RECORD_WAITING_TIME_SEC)) { + return rawConfig.get(INITIAL_RECORD_WAITING_TIME_SEC).asInt(DEFAULT_INITIAL_RECORD_WAITING_TIME_SEC); + } else { + return DEFAULT_INITIAL_RECORD_WAITING_TIME_SEC; + } + } + + public boolean shouldFailSyncOnInvalidCursor() { + if (rawConfig.has(INVALID_CDC_CURSOR_POSITION_PROPERTY) + && rawConfig.get(INVALID_CDC_CURSOR_POSITION_PROPERTY).asText().equals(RESYNC_DATA_OPTION)) { + return false; + } else { + return true; + } + } + } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java index 06dcfb6c7958..f659c2c06af6 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java @@ -9,8 +9,8 @@ import com.mongodb.MongoException; import com.mongodb.client.MongoCursor; import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils; import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils; import io.airbyte.integrations.source.mongodb.state.IdType; import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; @@ -50,6 +50,7 @@ public class MongoDbStateIterator implements Iterator { private Instant lastCheckpoint = Instant.now(); private final Integer checkpointInterval; private final Duration checkpointDuration; + private final boolean isEnforceSchema; /** * Counts the number of records seen in this batch, resets when a state-message has been generated. @@ -68,10 +69,15 @@ public class MongoDbStateIterator implements Iterator { private boolean finalStateNext = false; /** - * Tracks if the underlying iterator threw an exception. This helps to determine the final state - * status emitted from the final next call. + * Tracks if the underlying iterator threw an exception, indicating that the snapshot for this + * stream failed. This helps to determine the final state status emitted from the final next call. */ - private boolean iterThrewException = false; + private boolean initialSnapshotFailed = false; + + /** + * Tracks the exception thrown if there initial snapshot has failed. + */ + private Exception initialSnapshotException; /** * Constructor. @@ -92,7 +98,8 @@ public MongoDbStateIterator(final MongoCursor iter, final ConfiguredAirbyteStream stream, final Instant emittedAt, final int checkpointInterval, - final Duration checkpointDuration) { + final Duration checkpointDuration, + final boolean isEnforceSchema) { this.iter = iter; this.stateManager = stateManager; this.stream = stream; @@ -103,19 +110,30 @@ public MongoDbStateIterator(final MongoCursor iter, this.lastId = stateManager.getStreamState(stream.getStream().getName(), stream.getStream().getNamespace()).map(MongoDbStreamState::id).orElse(null); this.cdcMetadataInjector = cdcMetadataInjector; + this.isEnforceSchema = isEnforceSchema; } @Override public boolean hasNext() { LOGGER.debug("Checking hasNext() for stream {}...", getStream()); + if (initialSnapshotFailed) { + // If the initial snapshot is incomplete for this stream, throw an exception failing the sync. This + // will ensure the platform retry logic + // kicks in and keeps retrying the sync until the initial snapshot is complete. + throw new RuntimeException(initialSnapshotException); + } try { if (iter.hasNext()) { return true; } } catch (final MongoException e) { - // If hasNext throws an exception, log it and then treat it as if hasNext returned false. - iterThrewException = true; + // If hasNext throws an exception, log it and set the flag to indicate that the initial snapshot + // failed. This indicates to the main iterator + // to emit state associated with what has been processed so far. + initialSnapshotFailed = true; + initialSnapshotException = e; LOGGER.info("hasNext threw an exception for stream {}: {}", getStream(), e.getMessage(), e); + return true; } // no more records in cursor + no record messages have been emitted => collection is empty @@ -142,9 +160,9 @@ public AirbyteMessage next() { // Should a state message be emitted based on then last time a state message was emitted? final var emitStateDueToDuration = count > 0 && Duration.between(lastCheckpoint, Instant.now()).compareTo(checkpointDuration) > 0; - if (finalStateNext) { + if (finalStateNext || initialSnapshotFailed) { LOGGER.debug("Emitting final state status for stream {}:{}...", stream.getStream().getNamespace(), stream.getStream().getName()); - final var finalStateStatus = iterThrewException ? InitialSnapshotStatus.IN_PROGRESS : InitialSnapshotStatus.COMPLETE; + final var finalStateStatus = initialSnapshotFailed ? InitialSnapshotStatus.IN_PROGRESS : InitialSnapshotStatus.COMPLETE; final var idType = IdType.findByJavaType(lastId.getClass().getSimpleName()) .orElseThrow(() -> new ConfigErrorException("Unsupported _id type " + lastId.getClass().getSimpleName())); final var state = new MongoDbStreamState(lastId.toString(), finalStateStatus, idType); @@ -172,7 +190,7 @@ public AirbyteMessage next() { count++; final var document = iter.next(); - final var jsonNode = MongoDbCdcEventUtils.toJsonNode(document, fields); + final var jsonNode = isEnforceSchema ? MongoDbCdcEventUtils.toJsonNode(document, fields) : MongoDbCdcEventUtils.toJsonNodeNoSchema(document); lastId = document.get(MongoConstants.ID_FIELD); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java index 8ee4c644ddcd..6becadb3225d 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java @@ -4,6 +4,14 @@ package io.airbyte.integrations.source.mongodb; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.AIRBYTE_STREAM_PROPERTIES; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_PRIMARY_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMALESS_MODE_DATA_FIELD; + +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoClient; @@ -12,11 +20,15 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.MongoIterable; import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Projections; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -25,6 +37,7 @@ import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.bson.Document; import org.bson.conversions.Bson; @@ -55,6 +68,9 @@ public class MongoUtil { @VisibleForTesting static final int MAX_QUEUE_SIZE = 10000; + static final Set SCHEMALESS_FIELDS = + Set.of(CDC_UPDATED_AT, CDC_DELETED_AT, DEFAULT_CURSOR_FIELD, DEFAULT_PRIMARY_KEY, SCHEMALESS_MODE_DATA_FIELD); + /** * Tests whether the database exists in target MongoDB instance. * @@ -105,15 +121,20 @@ public static Set getAuthorizedCollections(final MongoClient mongoClient * @param databaseName The name of the database to query for collections. * @param sampleSize The maximum number of documents to sample when attempting to discover the * unique fields for a collection. + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode * @return The list of {@link AirbyteStream}s that map to the available collections in the provided * database. */ - public static List getAirbyteStreams(final MongoClient mongoClient, final String databaseName, final Integer sampleSize) { + public static List getAirbyteStreams(final MongoClient mongoClient, + final String databaseName, + final Integer sampleSize, + final boolean isSchemaEnforced) { final Set authorizedCollections = getAuthorizedCollections(mongoClient, databaseName); return authorizedCollections.parallelStream() - .map(collectionName -> discoverFields(collectionName, mongoClient, databaseName, sampleSize)) - .filter(stream -> stream.isPresent()) - .map(stream -> stream.get()) + .map(collectionName -> discoverFields(collectionName, mongoClient, databaseName, sampleSize, isSchemaEnforced)) + .filter(Optional::isPresent) + .map(Optional::get) .collect(Collectors.toList()); } @@ -126,7 +147,7 @@ public static List getAirbyteStreams(final MongoClient mongoClien * @param config The source connector's configuration. * @return The size of the Debezium event queue. */ - public static OptionalInt getDebeziumEventQueueSize(final MongoDbSourceConfig config) { + public static int getDebeziumEventQueueSize(final MongoDbSourceConfig config) { final OptionalInt sizeFromConfig = config.getQueueSize(); if (sizeFromConfig.isPresent()) { @@ -134,15 +155,15 @@ public static OptionalInt getDebeziumEventQueueSize(final MongoDbSourceConfig co if (size < MIN_QUEUE_SIZE) { LOGGER.warn("Queue size is overridden to {} , which is the min allowed for safety.", MIN_QUEUE_SIZE); - return OptionalInt.of(MIN_QUEUE_SIZE); + return MIN_QUEUE_SIZE; } else if (size > MAX_QUEUE_SIZE) { LOGGER.warn("Queue size is overridden to {} , which is the max allowed for safety.", MAX_QUEUE_SIZE); - return OptionalInt.of(MAX_QUEUE_SIZE); + return MAX_QUEUE_SIZE; } - return OptionalInt.of(size); + return size; } - return OptionalInt.of(MAX_QUEUE_SIZE); + return MAX_QUEUE_SIZE; } /** @@ -184,16 +205,57 @@ public static Optional getCollectionStatistics(final Mongo return Optional.empty(); } + /** + * Checks whether the user's config + catalog does not match. This can happen in the following cases + * : 1. User is in schemaless mode + catalog corresponds to schema enabled mode. 2. User is in + * schema enabled mode + catalog corresponds to schemaless mode + * + * @param isConfigSchemaEnforced true if schema is enforced in configuration, false if in schemaless + * mode. + * @param isStateSchemaEnforced true if schema is enforced in saved state, false if in schemaless + * mode. + * @param catalog User's configured catalog. + */ + public static void checkSchemaModeMismatch(final boolean isConfigSchemaEnforced, + final boolean isStateSchemaEnforced, + final ConfiguredAirbyteCatalog catalog) { + final boolean isCatalogSchemaEnforcing = !catalog.getStreams().stream() + .allMatch(stream -> verifySchemaless(stream.getStream().getJsonSchema())); + + final String remedy = isConfigSchemaEnforced == isCatalogSchemaEnforcing + ? "Please reset your data." + : "Please refresh source schema and reset streams."; + if (Stream.of(isConfigSchemaEnforced, isStateSchemaEnforced, isCatalogSchemaEnforcing).distinct().count() > 1) { + throw new ConfigErrorException("Mismatch between schema enforcing mode in sync configuration (%b), catalog (%b) and saved state (%b). %s" + .formatted(isConfigSchemaEnforced, isCatalogSchemaEnforcing, isStateSchemaEnforced, remedy)); + } + } + + private static boolean verifySchemaless(final JsonNode jsonSchema) { + final JsonNode airbyteStreamProperties = jsonSchema.get(AIRBYTE_STREAM_PROPERTIES); + return airbyteStreamProperties.size() == SCHEMALESS_FIELDS.size() && + SCHEMALESS_FIELDS.stream().allMatch(field -> airbyteStreamProperties.get(field) != null); + } + /** * Creates an {@link AirbyteStream} from the provided data. * * @param collectionName The name of the collection represented by the stream (stream name). * @param databaseName The name of the database represented by the stream (stream namespace). * @param fields The fields available to the stream. + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode * @return A {@link AirbyteStream} object representing the stream. */ - private static AirbyteStream createAirbyteStream(final String collectionName, final String databaseName, final List fields) { - return MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); + private static AirbyteStream createAirbyteStream(final String collectionName, + final String databaseName, + final List fields, + final boolean isSchemaEnforced) { + if (isSchemaEnforced) { + return MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); + } else { + return MongoCatalogHelper.buildSchemalessAirbyteStream(collectionName, databaseName, fields); + } } /** @@ -204,21 +266,33 @@ private static AirbyteStream createAirbyteStream(final String collectionName, fi * @param databaseName The name of the database associated with the stream (stream namespace). * @param sampleSize The maximum number of documents to sample when attempting to discover the * unique fields for a collection + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode * @return The {@link AirbyteStream} that contains the discovered fields or an empty * {@link Optional} if the underlying collection is empty. */ private static Optional discoverFields(final String collectionName, final MongoClient mongoClient, final String databaseName, - final Integer sampleSize) { + final Integer sampleSize, + final boolean isSchemaEnforced) { /* * Fetch the keys/types from the first N documents and the last N documents from the collection. * This is an attempt to "survey" the documents in the collection for variance in the schema keys. */ + final Set discoveredFields; final MongoCollection mongoCollection = mongoClient.getDatabase(databaseName).getCollection(collectionName); - final Set discoveredFields = new HashSet<>(getFieldsInCollection(mongoCollection, sampleSize)); + if (isSchemaEnforced) { + discoveredFields = new HashSet<>(getFieldsInCollection(mongoCollection, sampleSize)); + } else { + // In schemaless mode, we only sample one record as we're only interested in the _id field (which + // exists on every record). + discoveredFields = new HashSet<>(getFieldsForSchemaless(mongoCollection)); + } return Optional - .ofNullable(!discoveredFields.isEmpty() ? createAirbyteStream(collectionName, databaseName, new ArrayList<>(discoveredFields)) : null); + .ofNullable( + !discoveredFields.isEmpty() ? createAirbyteStream(collectionName, databaseName, new ArrayList<>(discoveredFields), isSchemaEnforced) + : null); } private static Set getFieldsInCollection(final MongoCollection collection, final Integer sampleSize) { @@ -264,6 +338,26 @@ private static Set getFieldsInCollection(final MongoCollection return discoveredFields; } + private static Set getFieldsForSchemaless(final MongoCollection collection) { + final Set discoveredFields = new HashSet<>(); + + final AggregateIterable output = collection.aggregate(Arrays.asList( + Aggregates.sample(1), // Selects one random document + Aggregates.project(Projections.fields( + Projections.excludeId(), // Excludes the _id field from the result + Projections.computed("_idType", new Document("$type", "$_id")) // Gets the type of the _id field + )))); + + try (final MongoCursor cursor = output.allowDiskUse(true).cursor()) { + while (cursor.hasNext()) { + final JsonSchemaType schemaType = convertToSchemaType((String) cursor.next().get("_idType")); + discoveredFields.add(new MongoField(MongoConstants.ID_FIELD, schemaType)); + } + } + + return discoveredFields; + } + private static JsonSchemaType convertToSchemaType(final String type) { return switch (type) { case "boolean" -> JsonSchemaType.BOOLEAN; diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java index 3a6d9a39b489..ed0731d55561 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java @@ -4,14 +4,13 @@ package io.airbyte.integrations.source.mongodb.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; import io.airbyte.commons.json.Jsons; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtils.java similarity index 81% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtils.java index 82d6a0fb3e53..1e9e296a51e5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtils.java @@ -2,8 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.SCHEMALESS_MODE_DATA_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY; import static java.util.Arrays.asList; import static org.bson.BsonType.ARRAY; import static org.bson.BsonType.DOCUMENT; @@ -18,6 +20,8 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import java.util.Collections; +import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.bson.BsonBinary; @@ -73,7 +77,7 @@ public static String generateObjectIdDocument(final JsonNode debeziumEventKey) { if (StringUtils.contains(idField, OBJECT_ID_FIELD)) { return idField.replaceAll(OBJECT_ID_FIELD_PATTERN, DOCUMENT_OBJECT_ID_FIELD); } else { - return Jsons.serialize(debeziumEventKey).replaceAll(ID_FIELD, DOCUMENT_OBJECT_ID_FIELD); + return Jsons.serialize(Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, idField.replaceAll("^\"|\"$", "")))); } } @@ -109,6 +113,14 @@ public static ObjectNode normalizeObjectId(final ObjectNode data) { return data; } + public static ObjectNode normalizeObjectIdNoSchema(final ObjectNode data) { + normalizeObjectId(data); + // normalize _id in "data" if key exists + final Optional maybeDataField = Optional.ofNullable(data.get(SCHEMALESS_MODE_DATA_FIELD)); + maybeDataField.ifPresent(d -> normalizeObjectId((ObjectNode) d)); + return data; + } + /** * Transforms the Debezium event data to ensure that all data types are consistent with those in * documents generated by initial snapshots. @@ -123,12 +135,25 @@ public static ObjectNode transformDataTypes(final String json, final Set return normalizeObjectId(objectNode); } + public static ObjectNode transformDataTypesNoSchema(final String json) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + final Document document = Document.parse(json); + formatDocumentNoSchema(document, objectNode); + return normalizeObjectIdNoSchema(objectNode); + } + public static JsonNode toJsonNode(final Document document, final Set columnNames) { final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); formatDocument(document, objectNode, columnNames); return normalizeObjectId(objectNode); } + public static JsonNode toJsonNodeNoSchema(final Document document) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + formatDocumentNoSchema(document, objectNode); + return normalizeObjectIdNoSchema(objectNode); + } + private static void formatDocument(final Document document, final ObjectNode objectNode, final Set columnNames) { final BsonDocument bsonDocument = toBsonDocument(document); try (final BsonReader reader = new BsonDocumentReader(bsonDocument)) { @@ -139,6 +164,19 @@ private static void formatDocument(final Document document, final ObjectNode obj } } + private static void formatDocumentNoSchema(final Document document, final ObjectNode objectNode) { + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, Jsons.jsonNode(Collections.emptyMap())); + final BsonDocument bsonDocument = toBsonDocument(document); + try (final BsonReader reader = new BsonDocumentReader(bsonDocument)) { + readDocument(reader, (ObjectNode) objectNode.get(SCHEMALESS_MODE_DATA_FIELD), Collections.emptySet(), true); + final Optional maybeId = Optional.ofNullable(objectNode.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD)); + maybeId.ifPresent(id -> objectNode.set(DOCUMENT_OBJECT_ID_FIELD, id)); + } catch (final Exception e) { + LOGGER.error("Exception while parsing BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + private static ObjectNode readDocument(final BsonReader reader, final ObjectNode jsonNodes, final Set includedFields, @@ -292,7 +330,20 @@ public static void transformToStringIfMarked(final ObjectNode jsonNodes, final S * otherwise. */ private static boolean shouldIncludeField(final String fieldName, final Set includedFields, final boolean allowAll) { - return includedFields.contains(fieldName) || allowAll; + return allowAll || includedFields.contains(fieldName); + } + + /** + * Parses source-mongodbv2 configuration json for the value of schema_enforced. + * + * @param config config json + * @return true unless a schema_enforced configured to false + */ + public static boolean isEnforceSchema(final JsonNode config) { + return config == null || !config.has(SCHEMA_ENFORCED_CONFIGURATION_KEY) + || (config.has(SCHEMA_ENFORCED_CONFIGURATION_KEY) && config.get( + SCHEMA_ENFORCED_CONFIGURATION_KEY).asBoolean(true)); + } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java index 02c31e96c346..1e844f4949ca 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java @@ -67,7 +67,7 @@ public static List getStreamsForInitialSnapshot( final List initialSnapshotStreams = new ArrayList<>(); if (!savedOffsetIsValid) { - LOGGER.debug("Offset state is invalid. Add all {} stream(s) from the configured catalog to perform an initial snapshot.", + LOGGER.info("Offset state is invalid. Add all {} stream(s) from the configured catalog to perform an initial snapshot.", fullCatalog.getStreams().size()); /* @@ -87,7 +87,7 @@ public static List getStreamsForInitialSnapshot( .map(Map.Entry::getKey) .collect(Collectors.toSet()); - LOGGER.debug("There are {} stream(s) that are still in progress of an initial snapshot sync.", streamsStillInInitialSnapshot.size()); + LOGGER.info("There are {} stream(s) that are still in progress of an initial snapshot sync.", streamsStillInInitialSnapshot.size()); // Fetch the streams from the catalog that still need to complete the initial snapshot sync initialSnapshotStreams.addAll(fullCatalog.getStreams().stream() @@ -98,7 +98,7 @@ public static List getStreamsForInitialSnapshot( // Fetch the streams added to the catalog since the last sync final List newStreams = identifyStreamsToSnapshot(fullCatalog, new HashSet<>(stateManager.getStreamStates().keySet())); - LOGGER.debug("There are {} stream(s) that have been added to the catalog since the last sync.", newStreams.size()); + LOGGER.info("There are {} stream(s) that have been added to the catalog since the last sync.", newStreams.size()); initialSnapshotStreams.addAll(newStreams); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java index edef7fa2987e..279eb053f3f2 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java @@ -4,15 +4,15 @@ package io.airbyte.integrations.source.mongodb.cdc; +import static io.airbyte.cdk.db.DbAnalyticsUtils.cdcCursorInvalidMessage; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; -import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbResumeTokenHelper; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; @@ -27,11 +27,11 @@ import java.time.Instant; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.Properties; import java.util.function.Supplier; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,44 +83,56 @@ public List> createCdcIterators( final Instant emittedAt, final MongoDbSourceConfig config) { - final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(config.rawConfig()); - final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(config.rawConfig()); - final OptionalInt queueSize = MongoUtil.getDebeziumEventQueueSize(config); + final Duration firstRecordWaitTime = Duration.ofSeconds(config.getInitialWaitingTimeSeconds()); + // #35059: debezium heartbeats are not sent on the expected interval. this is + // a worksaround to allow making subsequent wait time configurable. + final Duration subsequentRecordWaitTime = firstRecordWaitTime; + LOGGER.info("Subsequent cdc record wait time: {} seconds", subsequentRecordWaitTime); + final int queueSize = MongoUtil.getDebeziumEventQueueSize(config); final String databaseName = config.getDatabaseName(); + final boolean isEnforceSchema = config.getEnforceSchema(); final Properties defaultDebeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); - final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + logOplogInfo(mongoClient); + final BsonDocument initialResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, catalog); final JsonNode initialDebeziumState = - mongoDbDebeziumStateUtil.constructInitialDebeziumState(resumeToken, mongoClient, databaseName); - final JsonNode cdcState = (stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) ? initialDebeziumState - : Jsons.clone(stateManager.getCdcState().state()); + mongoDbDebeziumStateUtil.constructInitialDebeziumState(initialResumeToken, mongoClient, databaseName); + final MongoDbCdcState cdcState = (stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) + ? new MongoDbCdcState(initialDebeziumState, isEnforceSchema) + : new MongoDbCdcState(Jsons.clone(stateManager.getCdcState().state()), stateManager.getCdcState().schema_enforced()); final Optional optSavedOffset = mongoDbDebeziumStateUtil.savedOffset( Jsons.clone(defaultDebeziumProperties), catalog, - cdcState, - config.rawConfig(), + cdcState.state(), + config.getDatabaseConfig(), mongoClient); // We should always be able to extract offset out of state if it's not null - if (cdcState != null && optSavedOffset.isEmpty()) { + if (cdcState.state() != null && optSavedOffset.isEmpty()) { throw new RuntimeException( - "Unable extract the offset out of state, State mutation might not be working. " + cdcState); + "Unable extract the offset out of state, State mutation might not be working. " + cdcState.state()); } final boolean savedOffsetIsValid = optSavedOffset.filter(savedOffset -> mongoDbDebeziumStateUtil.isValidResumeToken(savedOffset, mongoClient)).isPresent(); if (!savedOffsetIsValid) { - LOGGER.debug("Saved offset is not valid. Airbyte will trigger a full refresh."); + AirbyteTraceMessageUtility.emitAnalyticsTrace(cdcCursorInvalidMessage()); + if (config.shouldFailSyncOnInvalidCursor()) { + throw new ConfigErrorException( + "Saved offset is not valid. Please reset the connection, and then increase oplog retention or reduce sync frequency to prevent his from happening in the future. See https://docs.airbyte.com/integrations/sources/mongodb-v2#mongodb-oplog-and-change-streams for more details"); + } + LOGGER.info("Saved offset is not valid. Airbyte will trigger a full refresh."); // If the offset in the state is invalid, reset the state to the initial STATE - stateManager.resetState(new MongoDbCdcState(initialDebeziumState)); + stateManager.resetState(new MongoDbCdcState(initialDebeziumState, config.getEnforceSchema())); } else { - LOGGER.debug("Valid offset state discovered. Updating state manager with retrieved CDC state {}...", cdcState); - stateManager.updateCdcState(new MongoDbCdcState(cdcState)); + LOGGER.info("Valid offset state discovered. Updating state manager with retrieved CDC state {} {}...", cdcState.state(), + cdcState.schema_enforced()); + stateManager.updateCdcState(new MongoDbCdcState(cdcState.state(), cdcState.schema_enforced())); } final MongoDbCdcState stateToBeUsed = (!savedOffsetIsValid || stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) - ? new MongoDbCdcState(initialDebeziumState) + ? new MongoDbCdcState(initialDebeziumState, config.getEnforceSchema()) : stateManager.getCdcState(); final List initialSnapshotStreams = @@ -128,21 +140,17 @@ public List> createCdcIterators( final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); final List> initialSnapshotIterators = initialSnapshotHandler.getIterators(initialSnapshotStreams, stateManager, mongoClient.getDatabase(databaseName), cdcMetadataInjector, - emittedAt, config.getCheckpointInterval()); + emittedAt, config.getCheckpointInterval(), isEnforceSchema); - final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(config.rawConfig(), - new MongoDbCdcTargetPosition(resumeToken), false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize); + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(config.getDatabaseConfig(), + new MongoDbCdcTargetPosition(initialResumeToken), false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize, false); final MongoDbCdcStateHandler mongoDbCdcStateHandler = new MongoDbCdcStateHandler(stateManager); final MongoDbCdcSavedInfoFetcher cdcSavedInfoFetcher = new MongoDbCdcSavedInfoFetcher(stateToBeUsed); + final var propertiesManager = new MongoDbDebeziumPropertiesManager(defaultDebeziumProperties, config.getDatabaseConfig(), catalog); + final var eventConverter = new MongoDbDebeziumEventConverter(cdcMetadataInjector, catalog, emittedAt, config.getDatabaseConfig()); - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, - cdcSavedInfoFetcher, - mongoDbCdcStateHandler, - cdcMetadataInjector, - defaultDebeziumProperties, - DebeziumPropertiesManager.DebeziumConnectorType.MONGODB, - emittedAt, - false); + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators( + propertiesManager, eventConverter, cdcSavedInfoFetcher, mongoDbCdcStateHandler); // We can close the client after the initial snapshot is complete, incremental // iterator does not make use of the client. @@ -152,4 +160,18 @@ public List> createCdcIterators( return List.of(initialSnapshotIterator, AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null)); } + private void logOplogInfo(final MongoClient mongoClient) { + try { + final MongoDatabase localDatabase = mongoClient.getDatabase("local"); + final Document command = new Document("collStats", "oplog.rs"); + final Document result = localDatabase.runCommand(command); + if (result != null) { + LOGGER.info("Max oplog size is {} bytes", result.getLong("maxSize")); + LOGGER.info("Free space in oplog is {} bytes", result.getLong("freeStorageSize")); + } + } catch (final Exception e) { + LOGGER.warn("Unable to query for op log stats, exception: {}" + e.getMessage()); + } + } + } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java index e567d190da32..0e559a634c2a 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java @@ -20,6 +20,8 @@ public class MongoDbCdcProperties { static final String HEARTBEAT_INTERVAL_KEY = "heartbeat.interval.ms"; static final String SNAPSHOT_MODE_KEY = "snapshot.mode"; static final String SNAPSHOT_MODE_VALUE = "never"; + static final String CAPTURE_SCOPE_KEY = "capture.scope"; + static final String CAPTURE_SCOPE_VALUE = "database"; static final String TOMBSTONE_ON_DELETE_KEY = "tombstones.on.delete"; static final String TOMBSTONE_ON_DELETE_VALUE = Boolean.FALSE.toString(); @@ -33,6 +35,7 @@ public static Properties getDebeziumProperties() { props.setProperty(CONNECTOR_CLASS_KEY, CONNECTOR_CLASS_VALUE); props.setProperty(SNAPSHOT_MODE_KEY, SNAPSHOT_MODE_VALUE); + props.setProperty(CAPTURE_SCOPE_KEY, CAPTURE_SCOPE_VALUE); props.setProperty(CAPTURE_MODE_KEY, CAPTURE_MODE_VALUE); props.setProperty(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_FREQUENCY_MS); props.setProperty(TOMBSTONE_ON_DELETE_KEY, TOMBSTONE_ON_DELETE_VALUE); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java index 87f66a8224d2..1d94fdbd3546 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java @@ -11,4 +11,15 @@ * * @param state The Debezium offset state as a {@link JsonNode}. */ -public record MongoDbCdcState(JsonNode state) {} +public record MongoDbCdcState(JsonNode state, Boolean schema_enforced) { + + public MongoDbCdcState { + // Ensure that previously saved state with no schema_enforced will migrate to schema_enforced = true + schema_enforced = schema_enforced == null || schema_enforced; + } + + public MongoDbCdcState(final JsonNode state) { + this(state, true); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java index 9ad7c13a766f..81fd7cb417ab 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java @@ -30,7 +30,8 @@ public MongoDbCdcStateHandler(final MongoDbStateManager stateManager) { @Override public AirbyteMessage saveState(final Map offset, final AirbyteSchemaHistoryStorage.SchemaHistory ignored) { - final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset)); + final Boolean previousStateSchemaEnforced = stateManager.getCdcState() != null ? stateManager.getCdcState().schema_enforced() : null; + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset), previousStateSchemaEnforced); LOGGER.info("Saving Debezium state {}...", cdcState); stateManager.updateCdcState(cdcState); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPosition.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPosition.java index dcf907e887dc..c2d20f9bc4e9 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPosition.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoader.java similarity index 95% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoader.java index d5d0cb9f219c..ec537d60b8e4 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoader.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import io.airbyte.commons.json.Jsons; import io.debezium.connector.mongodb.MongoDbConnectorConfig; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumConstants.java similarity index 91% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumConstants.java index ad30c8166c86..170c6ae78552 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import io.debezium.connector.mongodb.SourceInfo; @@ -39,6 +39,8 @@ public static class Configuration { public static final String DATABASE_CONFIG_CONFIGURATION_KEY = "database_config"; public static final String PASSWORD_CONFIGURATION_KEY = "password"; public static final String USERNAME_CONFIGURATION_KEY = "username"; + public static final String SCHEMA_ENFORCED_CONFIGURATION_KEY = "schema_enforced"; + public static final String SCHEMALESS_MODE_DATA_FIELD = "data"; } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverter.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverter.java new file mode 100644 index 000000000000..30056743ced1 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverter.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.time.Instant; +import java.util.Set; +import java.util.stream.Collectors; + +public class MongoDbDebeziumEventConverter implements DebeziumEventConverter { + + private final CdcMetadataInjector cdcMetadataInjector; + private final ConfiguredAirbyteCatalog configuredAirbyteCatalog; + private final Instant emittedAt; + private final JsonNode config; + + public MongoDbDebeziumEventConverter( + CdcMetadataInjector cdcMetadataInjector, + ConfiguredAirbyteCatalog configuredAirbyteCatalog, + Instant emittedAt, + JsonNode config) { + this.cdcMetadataInjector = cdcMetadataInjector; + this.configuredAirbyteCatalog = configuredAirbyteCatalog; + this.emittedAt = emittedAt; + this.config = config; + } + + @Override + public AirbyteMessage toAirbyteMessage(ChangeEventWithMetadata event) { + final JsonNode debeziumEventKey = event.eventKeyAsJson(); + final JsonNode debeziumEvent = event.eventValueAsJson(); + final JsonNode before = debeziumEvent.get(DebeziumEventConverter.BEFORE_EVENT); + final JsonNode after = debeziumEvent.get(DebeziumEventConverter.AFTER_EVENT); + final JsonNode source = debeziumEvent.get(DebeziumEventConverter.SOURCE_EVENT); + final String operation = debeziumEvent.get(DebeziumEventConverter.OPERATION_FIELD).asText(); + final boolean isEnforceSchema = MongoDbCdcEventUtils.isEnforceSchema(config); + + final Set configuredFields = isEnforceSchema ? getConfiguredMongoDbCollectionFields(source, configuredAirbyteCatalog, cdcMetadataInjector) + : null; + + /* + * Delete events need to be handled separately from other CrUD events, as depending on the version + * of the MongoDB server, the contents Debezium event data will be different. See + * #formatMongoDbDeleteDebeziumData() for more details. + */ + final JsonNode data = switch (operation) { + case "c", "i", "u" -> formatMongoDbDebeziumData( + before, after, source, debeziumEventKey, cdcMetadataInjector, configuredFields, isEnforceSchema); + case "d" -> formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields, isEnforceSchema); + default -> throw new IllegalArgumentException("Unsupported MongoDB change event operation '" + operation + "'."); + }; + + return DebeziumEventConverter.buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); + } + + private static JsonNode formatMongoDbDebeziumData(final JsonNode before, + final JsonNode after, + final JsonNode source, + final JsonNode debeziumEventKey, + final CdcMetadataInjector cdcMetadataInjector, + final Set configuredFields, + final boolean isEnforceSchema) { + + if ((before == null || before.isNull()) && (after == null || after.isNull())) { + // In case a mongodb document was updated and then deleted, the update change event will not have + // any information ({after: null}) + // We are going to treat it as a delete. + return formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields, isEnforceSchema); + } else { + final String eventJson = (after.isNull() ? before : after).asText(); + return DebeziumEventConverter.addCdcMetadata( + isEnforceSchema + ? MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields) + : MongoDbCdcEventUtils.transformDataTypesNoSchema(eventJson), + source, cdcMetadataInjector, false); + } + } + + private static JsonNode formatMongoDbDeleteDebeziumData(final JsonNode before, + final JsonNode debeziumEventKey, + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final Set configuredFields, + final boolean isEnforceSchema) { + final String eventJson; + + /* + * The change events produced by MongoDB differ based on the server version. For version BEFORE 6.x, + * the event does not contain the before document. Therefore, the only data that can be extracted is + * the object ID of the deleted document, which is stored in the event key. Otherwise, if the server + * is version 6.+ AND the pre-image support has been enabled on the collection, we can use the + * "before" document from the event to represent the deleted document. + * + * See + * https://www.mongodb.com/docs/manual/reference/change-events/delete/#document-pre--and-post-images + * for more details. + */ + if (!before.isNull()) { + eventJson = before.asText(); + } else { + eventJson = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + } + + return DebeziumEventConverter.addCdcMetadata( + isEnforceSchema + ? MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields) + : MongoDbCdcEventUtils.transformDataTypesNoSchema(eventJson), + source, cdcMetadataInjector, true); + } + + private static Set getConfiguredMongoDbCollectionFields(final JsonNode source, + final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final CdcMetadataInjector cdcMetadataInjector) { + final String streamNamespace = cdcMetadataInjector.namespace(source); + final String streamName = cdcMetadataInjector.name(source); + return configuredAirbyteCatalog.getStreams().stream() + .filter(s -> streamName.equals(s.getStream().getName()) && streamNamespace.equals(s.getStream().getNamespace())) + .map(CatalogHelpers::getTopLevelFieldNames) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java similarity index 80% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java index bd894b2ff708..c715be6080cd 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java @@ -2,22 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.stream.Collectors; @@ -33,6 +31,7 @@ public class MongoDbDebeziumPropertiesManager extends DebeziumPropertiesManager static final String COLLECTION_INCLUDE_LIST_KEY = "collection.include.list"; static final String DATABASE_INCLUDE_LIST_KEY = "database.include.list"; + static final String CAPTURE_TARGET_KEY = "capture.target"; static final String DOUBLE_QUOTES_PATTERN = "\""; static final String MONGODB_AUTHSOURCE_KEY = "mongodb.authsource"; static final String MONGODB_CONNECTION_MODE_KEY = "mongodb.connection.mode"; @@ -45,9 +44,8 @@ public class MongoDbDebeziumPropertiesManager extends DebeziumPropertiesManager public MongoDbDebeziumPropertiesManager(final Properties properties, final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager) { - super(properties, config, catalog, offsetManager, Optional.empty()); + final ConfiguredAirbyteCatalog catalog) { + super(properties, config, catalog); } @Override @@ -82,6 +80,7 @@ protected Properties getIncludeConfiguration(final ConfiguredAirbyteCatalog cata // Database/collection selection properties.setProperty(COLLECTION_INCLUDE_LIST_KEY, createCollectionIncludeString(catalog.getStreams())); properties.setProperty(DATABASE_INCLUDE_LIST_KEY, config.get(DATABASE_CONFIGURATION_KEY).asText()); + properties.setProperty(CAPTURE_TARGET_KEY, config.get(DATABASE_CONFIGURATION_KEY).asText()); return properties; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtil.java similarity index 91% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtil.java index ae940528ecd6..e835de192f1e 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import com.fasterxml.jackson.databind.JsonNode; import com.mongodb.MongoChangeStreamException; @@ -56,6 +56,8 @@ public class MongoDbDebeziumStateUtil implements DebeziumStateUtil { */ public JsonNode constructInitialDebeziumState(final BsonDocument resumeToken, final MongoClient mongoClient, final String serverId) { final String replicaSet = getReplicaSetName(mongoClient); + LOGGER.info("Initial resume token '{}' constructed, corresponding to timestamp (seconds after epoch) {}", + ResumeTokens.getData(resumeToken).asString().getValue(), ResumeTokens.getTimestamp(resumeToken).getTime()); final JsonNode state = formatState(serverId, replicaSet, ((BsonString) ResumeTokens.getData(resumeToken)).getValue()); LOGGER.info("Initial Debezium state constructed: {}", state); return state; @@ -112,12 +114,14 @@ public boolean isValidResumeToken(final BsonDocument savedOffset, final MongoCli final ChangeStreamIterable stream = mongoClient.watch(BsonDocument.class); stream.resumeAfter(savedOffset); try (final var ignored = stream.cursor()) { - LOGGER.info("Valid resume token '{}' present. Incremental sync will be performed for up-to-date streams.", - ResumeTokens.getData(savedOffset).asString().getValue()); + LOGGER.info("Valid resume token '{}' present, corresponding to timestamp (seconds after epoch) : {}. Incremental sync will be performed for " + + "up-to-date streams.", + ResumeTokens.getData(savedOffset).asString().getValue(), ResumeTokens.getTimestamp(savedOffset).getTime()); return true; } catch (final MongoCommandException | MongoChangeStreamException e) { - LOGGER.info("Invalid resume token '{}' present. Initial snapshot will be performed for all streams.", - ResumeTokens.getData(savedOffset).asString().getValue()); + LOGGER.info("Invalid resume token '{}' present, corresponding to timestamp (seconds after epoch) : {}. Initial snapshot will be performed for " + + "all streams.", + ResumeTokens.getData(savedOffset).asString().getValue(), ResumeTokens.getTimestamp(savedOffset).getTime()); return false; } } @@ -140,10 +144,9 @@ public Optional savedOffset(final Properties baseProperties, final JsonNode config, final MongoClient mongoClient) { LOGGER.debug("Initializing file offset backing store with state '{}'...", cdcState); - final DebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(baseProperties, - config, catalog, - AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty())); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + final var offsetManager = AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty()); + final DebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(baseProperties, config, catalog); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); return parseSavedOffset(debeziumProperties, mongoClient); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java similarity index 76% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java index 48dbc5967e9b..0493efa11cda 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java @@ -2,17 +2,24 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import com.fasterxml.jackson.databind.JsonNode; import com.mongodb.client.ChangeStreamIterable; import com.mongodb.client.MongoChangeStreamCursor; import com.mongodb.client.MongoClient; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,8 +36,16 @@ public class MongoDbResumeTokenHelper { * @param mongoClient The {@link MongoClient} used to query the MongoDB server. * @return The most recent resume token value. */ - public static BsonDocument getMostRecentResumeToken(final MongoClient mongoClient) { - final ChangeStreamIterable eventStream = mongoClient.watch(BsonDocument.class); + public static BsonDocument getMostRecentResumeToken(final MongoClient mongoClient, + final String databaseName, + final ConfiguredAirbyteCatalog catalog) { + final List collectionsList = catalog.getStreams().stream() + .map(s -> s.getStream().getName()) + .toList(); + LOGGER.info("Resume token for db {} with collection filter {}", databaseName, Arrays.toString(collectionsList.toArray())); + final List pipeline = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", collectionsList))); + final ChangeStreamIterable eventStream = mongoClient.getDatabase(databaseName).watch(pipeline, BsonDocument.class); try (final MongoChangeStreamCursor> eventStreamCursor = eventStream.cursor()) { /* * Must call tryNext before attempting to get the resume token from the cursor directly. Otherwise, diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index d3fbb130dfaf..07a7268b7158 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -67,6 +67,14 @@ "default": "admin", "examples": ["admin"], "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } }, @@ -117,6 +125,14 @@ "default": "admin", "examples": ["admin"], "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } } @@ -127,7 +143,7 @@ "title": "Initial Waiting Time in Seconds (Advanced)", "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds.", "default": 300, - "order": 7, + "order": 8, "min": 120, "max": 1200, "group": "advanced" @@ -137,7 +153,7 @@ "title": "Size of the queue (Advanced)", "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", "default": 10000, - "order": 8, + "order": 9, "min": 1000, "max": 10000, "group": "advanced" @@ -147,10 +163,19 @@ "title": "Document discovery sample size (Advanced)", "description": "The maximum number of documents to sample when attempting to discover the unique fields for a collection.", "default": 10000, - "order": 9, - "minimum": 1000, + "order": 10, + "minimum": 10, "maximum": 100000, "group": "advanced" + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 11, + "group": "advanced" } }, "groups": [ diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java index cef1eeb91f10..5b7703b85c81 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -22,15 +23,15 @@ import com.mongodb.client.model.Updates; import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbResumeTokenHelper; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcState; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcTargetPosition; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumStateUtil; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbResumeTokenHelper; import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; import io.airbyte.protocol.models.Field; @@ -508,21 +509,16 @@ void testSyncShouldHandlePurgedLogsGracefully() throws Exception { stateMessage.getGlobal().setSharedState(Jsons.jsonNode(cdcState)); final JsonNode state = Jsons.jsonNode(List.of(stateMessage)); - // Re-run the sync to prove that an initial snapshot is initiated due to invalid resume token - final List messages2 = runRead(configuredCatalog, state); - - final List recordMessages2 = filterRecords(messages2); - final List stateMessages2 = filterStateMessages(messages2); - - assertEquals(recordCount, recordMessages2.size()); - assertEquals(recordCount + 1, stateMessages2.size()); + // Re-run the sync to prove that a config error is thrown due to invalid resume token + assertThrows(Exception.class, () -> runRead(configuredCatalog, state)); } @Test void testReachedTargetPosition() { final long eventTimestamp = Long.MAX_VALUE; final Integer order = 0; - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog())); final ChangeEventWithMetadata changeEventWithMetadata = mock(ChangeEventWithMetadata.class); when(changeEventWithMetadata.isSnapshotEvent()).thenReturn(true); @@ -549,8 +545,9 @@ void testReachedTargetPosition() { @Test void testIsSameOffset() { - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog())); + final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog()); final String resumeTokenString = resumeToken.get("_data").asString().getValue(); final String replicaSet = MongoDbDebeziumStateUtil.getReplicaSetName(mongoClient); final Map emptyOffsetA = Map.of(); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt index d80a179a5947..fd2b7f612930 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt @@ -1,7 +1,10 @@ package io.airbyte.integrations.source.mongodb +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.github.javafaker.Faker -import io.airbyte.commons.json.Jsons import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.cli.ArgParser import kotlinx.cli.ArgType @@ -31,7 +34,7 @@ object MongoDbInsertClient { println("Enter password: ") val password = readln() - var config = mapOf(MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY to + val config = mapOf(MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY to mapOf( MongoConstants.DATABASE_CONFIGURATION_KEY to databaseName, MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY to connectionString, @@ -42,7 +45,12 @@ object MongoDbInsertClient { val faker = Faker(); - MongoConnectionUtils.createMongoClient(MongoDbSourceConfig(Jsons.deserialize(Jsons.serialize(config)))).use { mongoClient -> + val objectMapper = ObjectMapper().registerModule(JavaTimeModule()) + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + objectMapper.configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true) + val roundTrippedConfig = objectMapper.readTree(objectMapper.writeValueAsBytes(config)) + + MongoConnectionUtils.createMongoClient(MongoDbSourceConfig(roundTrippedConfig)).use { mongoClient -> val documents = mutableListOf() val batches = if (numberOfDocuments > BATCH_SIZE) numberOfDocuments / BATCH_SIZE else 1; val batchSize = if (numberOfDocuments > BATCH_SIZE) BATCH_SIZE else numberOfDocuments; diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java index 891a57a8d556..9ece697ec8fa 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java @@ -56,9 +56,11 @@ class InitialSnapshotHandlerTest { private static final String COLLECTION3 = "collection3"; private static final String OBJECT_ID1_STRING = "64c0029d95ad260d69ef28a1"; + private static final String OBJECT_ID2_STRING = "64c0029d95ad260d69ef28a2"; + private static final String OBJECT_ID3_STRING = "64c0029d95ad260d69ef28a3"; private static final ObjectId OBJECT_ID1 = new ObjectId(OBJECT_ID1_STRING); - private static final ObjectId OBJECT_ID2 = new ObjectId("64c0029d95ad260d69ef28a2"); - private static final ObjectId OBJECT_ID3 = new ObjectId("64c0029d95ad260d69ef28a3"); + private static final ObjectId OBJECT_ID2 = new ObjectId(OBJECT_ID2_STRING); + private static final ObjectId OBJECT_ID3 = new ObjectId(OBJECT_ID3_STRING); private static final ObjectId OBJECT_ID4 = new ObjectId("64c0029d95ad260d69ef28a4"); private static final ObjectId OBJECT_ID5 = new ObjectId("64c0029d95ad260d69ef28a5"); private static final ObjectId OBJECT_ID6 = new ObjectId("64c0029d95ad260d69ef28a6"); @@ -144,7 +146,7 @@ void testGetIteratorsEmptyInitialState() { final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); final List> iterators = initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), - MongoConstants.CHECKPOINT_INTERVAL); + MongoConstants.CHECKPOINT_INTERVAL, true); assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); @@ -218,7 +220,7 @@ void testGetIteratorsNonEmptyInitialState() { .thenReturn(Optional.of(new MongoDbStreamState(OBJECT_ID1_STRING, null, IdType.OBJECT_ID))); final List> iterators = initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), - MongoConstants.CHECKPOINT_INTERVAL); + MongoConstants.CHECKPOINT_INTERVAL, true); assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); @@ -266,7 +268,7 @@ void testGetIteratorsThrowsExceptionWhenThereAreDifferentIdTypes() { final var thrown = assertThrows(ConfigErrorException.class, () -> initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), - MongoConstants.CHECKPOINT_INTERVAL)); + MongoConstants.CHECKPOINT_INTERVAL, true)); assertTrue(thrown.getMessage().contains("must be consistently typed")); } @@ -282,7 +284,7 @@ void testGetIteratorsThrowsExceptionWhenThereAreUnsupportedIdTypes() { final var thrown = assertThrows(ConfigErrorException.class, () -> initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), - MongoConstants.CHECKPOINT_INTERVAL)); + MongoConstants.CHECKPOINT_INTERVAL, true)); assertTrue(thrown.getMessage().contains("_id fields with the following types are currently supported")); } @@ -308,7 +310,7 @@ void testGetIteratorsWithOneEmptyCollection() { final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); final List> iterators = initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), - MongoConstants.CHECKPOINT_INTERVAL); + MongoConstants.CHECKPOINT_INTERVAL, true); assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); @@ -332,4 +334,58 @@ void testGetIteratorsWithOneEmptyCollection() { assertFalse(collection2.hasNext()); } + @Test + void testGetIteratorsWithInitialStateNonDefaultIdType() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1_STRING, + NAME_FIELD, NAME1)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID2_STRING, + NAME_FIELD, NAME2)))); + + insertDocuments(COLLECTION2, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID3_STRING, + NAME_FIELD, NAME3)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + when(stateManager.getStreamState(COLLECTION1, NAMESPACE)) + .thenReturn(Optional.of(new MongoDbStreamState(OBJECT_ID1_STRING, null, IdType.STRING))); + final List> iterators = + initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true); + + assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); + + final AutoCloseableIterator collection1 = iterators.get(0); + final AutoCloseableIterator collection2 = iterators.get(1); + + // collection1, first document should be skipped + final AirbyteMessage collection1StreamMessage1 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage1.getType()); + assertEquals(COLLECTION1, collection1StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID2.toString(), collection1StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME2, collection1StreamMessage1.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection1SateMessage = collection1.next(); + assertEquals(Type.STATE, collection1SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection1.hasNext()); + + // collection2, no documents should be skipped + final AirbyteMessage collection2StreamMessage1 = collection2.next(); + assertEquals(Type.RECORD, collection2StreamMessage1.getType()); + assertEquals(COLLECTION2, collection2StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID3.toString(), collection2StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD), collection2StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection2SateMessage = collection2.next(); + assertEquals(Type.STATE, collection2SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection2.hasNext()); + } + } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java index 15e9a9919d1c..069bf036f483 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java @@ -7,11 +7,13 @@ import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_PRIMARY_KEY; import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.SUPPORTED_SYNC_MODES; +import static io.airbyte.integrations.source.mongodb.MongoConstants.ID_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMALESS_MODE_DATA_FIELD; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStream; @@ -42,12 +44,49 @@ void testBuildingAirbyteStream() { assertTrue(airbyteStream.getJsonSchema().get("properties").has(DEFAULT_CURSOR_FIELD)); assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), airbyteStream.getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); - assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_DELETED_AT)); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventConverter.CDC_DELETED_AT)); assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); - assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_UPDATED_AT)); + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_DELETED_AT).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventConverter.CDC_UPDATED_AT)); assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_UPDATED_AT).get("type").asText()); + + } + + @Test + void testSchemalessModeAirbyteStream() { + final String streamName = "name"; + final String streamNamespace = "namespace"; + final List discoveredFields = List.of(new Field("_id", JsonSchemaType.STRING), new Field("field1", JsonSchemaType.STRING), + new Field("field2", JsonSchemaType.NUMBER)); + + final AirbyteStream airbyteStream = MongoCatalogHelper.buildSchemalessAirbyteStream(streamName, streamNamespace, discoveredFields); + + assertNotNull(airbyteStream); + assertEquals(streamNamespace, airbyteStream.getNamespace()); + assertEquals(streamName, airbyteStream.getName()); + assertEquals(List.of(DEFAULT_CURSOR_FIELD), airbyteStream.getDefaultCursorField()); + assertEquals(true, airbyteStream.getSourceDefinedCursor()); + assertEquals(List.of(List.of(DEFAULT_PRIMARY_KEY)), airbyteStream.getSourceDefinedPrimaryKey()); + assertEquals(SUPPORTED_SYNC_MODES, airbyteStream.getSupportedSyncModes()); + assertEquals(5, airbyteStream.getJsonSchema().get("properties").size()); + + // All discovered fields that are not the _id field should be discarded + assertTrue(airbyteStream.getJsonSchema().get("properties").has(SCHEMALESS_MODE_DATA_FIELD)); + assertEquals(JsonSchemaType.OBJECT.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(SCHEMALESS_MODE_DATA_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(ID_FIELD)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(ID_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DEFAULT_CURSOR_FIELD)); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventConverter.CDC_DELETED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_DELETED_AT).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventConverter.CDC_UPDATED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_UPDATED_AT).get("type").asText()); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java index 0acb28756ae9..3ca4825b8dfd 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java @@ -14,6 +14,7 @@ import static io.airbyte.integrations.source.mongodb.MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.PASSWORD_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.USERNAME_CONFIGURATION_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -36,15 +37,18 @@ void testCreatingMongoDbSourceConfig() { final String password = "password"; final Integer sampleSize = 5000; final String username = "username"; + final boolean isSchemaEnforced = false; final JsonNode rawConfig = Jsons.jsonNode( - Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of( - AUTH_SOURCE_CONFIGURATION_KEY, authSource, - CHECKPOINT_INTERVAL_CONFIGURATION_KEY, checkpointInterval, - DATABASE_CONFIGURATION_KEY, database, + Map.of( DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY, sampleSize, - PASSWORD_CONFIGURATION_KEY, password, QUEUE_SIZE_CONFIGURATION_KEY, queueSize, - USERNAME_CONFIGURATION_KEY, username))); + DATABASE_CONFIG_CONFIGURATION_KEY, Map.of( + AUTH_SOURCE_CONFIGURATION_KEY, authSource, + CHECKPOINT_INTERVAL_CONFIGURATION_KEY, checkpointInterval, + DATABASE_CONFIGURATION_KEY, database, + PASSWORD_CONFIGURATION_KEY, password, + USERNAME_CONFIGURATION_KEY, username, + SCHEMA_ENFORCED_CONFIGURATION_KEY, isSchemaEnforced))); final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(rawConfig); assertNotNull(sourceConfig); assertEquals(authSource, sourceConfig.getAuthSource()); @@ -52,9 +56,10 @@ void testCreatingMongoDbSourceConfig() { assertEquals(database, sourceConfig.getDatabaseName()); assertEquals(password, sourceConfig.getPassword()); assertEquals(OptionalInt.of(queueSize), sourceConfig.getQueueSize()); - assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.rawConfig()); + assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.getDatabaseConfig()); assertEquals(sampleSize, sourceConfig.getSampleSize()); assertEquals(username, sourceConfig.getUsername()); + assertEquals(isSchemaEnforced, sourceConfig.getEnforceSchema()); } @Test @@ -72,7 +77,7 @@ void testDefaultValues() { assertEquals(null, sourceConfig.getDatabaseName()); assertEquals(null, sourceConfig.getPassword()); assertEquals(OptionalInt.empty(), sourceConfig.getQueueSize()); - assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.rawConfig()); + assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.getDatabaseConfig()); assertEquals(DEFAULT_DISCOVER_SAMPLE_SIZE, sourceConfig.getSampleSize()); assertEquals(null, sourceConfig.getUsername()); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java index 2ead329da9b7..6b6f661ebb78 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java @@ -35,7 +35,7 @@ import com.mongodb.client.MongoIterable; import com.mongodb.connection.ClusterDescription; import com.mongodb.connection.ClusterType; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcInitializer; @@ -67,7 +67,7 @@ class MongoDbSourceTest { @BeforeEach void setup() { - airbyteSourceConfig = createConfiguration(Optional.empty(), Optional.empty()); + airbyteSourceConfig = createConfiguration(Optional.empty(), Optional.empty(), true); sourceConfig = new MongoDbSourceConfig(airbyteSourceConfig); mongoClient = mock(MongoClient.class); cdcInitializer = mock(MongoDbCdcInitializer.class); @@ -240,9 +240,9 @@ void testDiscoverOperation() throws IOException { assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), stream.get().getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + stream.get().getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_DELETED_AT).get("type").asText()); assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + stream.get().getJsonSchema().get("properties").get(DebeziumEventConverter.CDC_UPDATED_AT).get("type").asText()); assertEquals(true, stream.get().getSourceDefinedCursor()); assertEquals(List.of(DEFAULT_CURSOR_FIELD), stream.get().getDefaultCursorField()); assertEquals(List.of(List.of(MongoCatalogHelper.DEFAULT_PRIMARY_KEY)), stream.get().getSourceDefinedPrimaryKey()); @@ -306,12 +306,13 @@ void testReadKeepsMongoClientOpen() { verify(mongoClient, never()).close(); } - private static JsonNode createConfiguration(final Optional username, final Optional password) { + private static JsonNode createConfiguration(final Optional username, final Optional password, final boolean isSchemaEnforced) { final Map baseConfig = Map.of( MongoConstants.DATABASE_CONFIGURATION_KEY, DB_NAME, MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://localhost:27017/", MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, "admin", - MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY, DEFAULT_DISCOVER_SAMPLE_SIZE); + MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY, DEFAULT_DISCOVER_SAMPLE_SIZE, + MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY, isSchemaEnforced); final Map config = new HashMap<>(baseConfig); username.ifPresent(u -> config.put(MongoConstants.USERNAME_CONFIGURATION_KEY, u)); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java index c41f20ba7520..66bf277dddbc 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java @@ -95,7 +95,7 @@ public Document answer(final InvocationOnMock invocation) { final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), - CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION); + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); // with a batch size of 2, the MongoDbStateIterator should return the following after each // `hasNext`/`next` call: @@ -162,7 +162,7 @@ void treatHasNextExceptionAsFalse() { final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), - CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION); + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); // with a batch size of 2, the MongoDbStateIterator should return the following after each // `hasNext`/`next` call: @@ -187,7 +187,7 @@ void treatHasNextExceptionAsFalse() { message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), "state status should be in_progress"); - assertFalse(iter.hasNext(), "should have no more records"); + assertThrows(RuntimeException.class, iter::hasNext, "next iteration should throw exception to fail the sync"); } @Test @@ -203,7 +203,7 @@ void anInvalidIdFieldThrowsAnException() { final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), - CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION); + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); assertTrue(iter.hasNext(), "air force blue should be next"); // first next call should return the document @@ -225,7 +225,7 @@ void initialStateIsReturnedIfUnderlyingIteratorIsEmpty() { new MongoDbStreamState(objectId, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID)); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), - CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION); + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); // the MongoDbStateIterator should return the following after each // `hasNext`/`next` call: @@ -263,7 +263,7 @@ void stateEmittedAfterDuration() throws InterruptedException { new MongoDbStreamState(objectId, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID)); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), 1000000, - Duration.of(1, SECONDS)); + Duration.of(1, SECONDS), true); // with a batch size of 1,000,000 and a 1.5s sleep between hasNext calls, the expected results // should be @@ -333,7 +333,7 @@ void hasNextNoInitialStateAndNoMoreRecordsInCursor() { when(mongoCursor.hasNext()).thenReturn(false); final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), 1000000, - Duration.of(1, SECONDS)); + Duration.of(1, SECONDS), true); assertFalse(iter.hasNext()); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java index 76873c216b1a..832c9c7af936 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java @@ -4,13 +4,16 @@ package io.airbyte.integrations.source.mongodb; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.AIRBYTE_STREAM_PROPERTIES; import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; import static io.airbyte.integrations.source.mongodb.MongoUtil.MAX_QUEUE_SIZE; import static io.airbyte.integrations.source.mongodb.MongoUtil.MIN_QUEUE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoUtil.checkSchemaModeMismatch; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -33,17 +36,20 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.MongoIterable; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.bson.BsonDocument; import org.bson.Document; import org.junit.jupiter.api.Test; @@ -88,12 +94,54 @@ void testGetAirbyteStreams() throws IOException { when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); assertNotNull(streams); assertEquals(1, streams.size()); assertEquals(12, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); } + @Test + void testGetAirbyteStreamsSchemalessMode() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_schemaless.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, false); + assertNotNull(streams); + assertEquals(1, streams.size()); + // In schemaless mode, only the 3 CDC fields + id and data fields should exist. + assertEquals(5, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaLessCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaLessCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(true, false, true)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(false, false, schemaLessCatalog)); + assertThat(throwable).isNull(); + } + @Test void testGetAirbyteStreamsEmptyCollection() throws IOException { final AggregateIterable aggregateIterable = mock(AggregateIterable.class); @@ -112,7 +160,7 @@ void testGetAirbyteStreamsEmptyCollection() throws IOException { when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); assertNotNull(streams); assertEquals(0, streams.size()); } @@ -139,7 +187,7 @@ void testGetAirbyteStreamsDifferentDataTypes() throws IOException { when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); assertNotNull(streams); assertEquals(1, streams.size()); assertEquals(11, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); @@ -152,6 +200,69 @@ void testGetAirbyteStreamsDifferentDataTypes() throws IOException { assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR) .get(JSON_TYPE_PROPERTY_NAME).asText()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaEnforcedCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(false, false, schemaEnforcedCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(false, true, false)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaEnforcedCatalog)); + assertThat(throwable).isNull(); + } + + @Test + void testonlyStateMismatchError() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_different_datatypes.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(11, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get("total").get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_UPDATED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_DELETED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR) + .get(JSON_TYPE_PROPERTY_NAME).asText()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaEnforcedCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(true, false, schemaEnforcedCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(true, true, false)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaEnforcedCatalog)); + assertThat(throwable).isNull(); } @Test @@ -201,18 +312,18 @@ void testGetAuthorizedCollectionsMongoSecurityException() { void testGetDebeziumEventQueueSize() { final int queueSize = 5000; final MongoDbSourceConfig validQueueSizeConfiguration = new MongoDbSourceConfig( - Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, queueSize)))); + Jsons.jsonNode(Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, queueSize, DATABASE_CONFIG_CONFIGURATION_KEY, Map.of()))); final MongoDbSourceConfig tooSmallQueueSizeConfiguration = new MongoDbSourceConfig( - Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MIN_VALUE)))); + Jsons.jsonNode(Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MIN_VALUE, DATABASE_CONFIG_CONFIGURATION_KEY, Map.of()))); final MongoDbSourceConfig tooLargeQueueSizeConfiguration = new MongoDbSourceConfig( - Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MAX_VALUE)))); + Jsons.jsonNode(Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MAX_VALUE, DATABASE_CONFIG_CONFIGURATION_KEY, Map.of()))); final MongoDbSourceConfig missingQueueSizeConfiguration = new MongoDbSourceConfig(Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of()))); - assertEquals(queueSize, MongoUtil.getDebeziumEventQueueSize(validQueueSizeConfiguration).getAsInt()); - assertEquals(MIN_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooSmallQueueSizeConfiguration).getAsInt()); - assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooLargeQueueSizeConfiguration).getAsInt()); - assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(missingQueueSizeConfiguration).getAsInt()); + assertEquals(queueSize, MongoUtil.getDebeziumEventQueueSize(validQueueSizeConfiguration)); + assertEquals(MIN_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooSmallQueueSizeConfiguration)); + assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooLargeQueueSizeConfiguration)); + assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(missingQueueSizeConfiguration)); } @Test @@ -309,6 +420,18 @@ void testGetCollectionStatisticsException() { final Optional statistics = MongoUtil.getCollectionStatistics(mongoClient, configuredAirbyteStream); assertFalse(statistics.isPresent()); + + } + + private static String formatMismatchException(final boolean isConfigSchemaEnforced, + final boolean isCatalogSchemaEnforcing, + final boolean isStateSchemaEnforced) { + final String remedy = isConfigSchemaEnforced == isCatalogSchemaEnforcing + ? "Please reset your data." + : "Please refresh source schema and reset streams."; + return "Mismatch between schema enforcing mode in sync configuration (%b), catalog (%b) and saved state (%b). " + .formatted(isConfigSchemaEnforced, isCatalogSchemaEnforcing, isStateSchemaEnforced) + + remedy; } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java index 78be3791db63..372fea5c766e 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java @@ -4,12 +4,11 @@ package io.airbyte.integrations.source.mongodb.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; import io.airbyte.commons.json.Jsons; import java.lang.reflect.Field; import java.time.Instant; diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtilsTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtilsTest.java new file mode 100644 index 000000000000..3068668bb972 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcEventUtilsTest.java @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.DOCUMENT_OBJECT_ID_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.OBJECT_ID_FIELD_PATTERN; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.SCHEMALESS_MODE_DATA_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.commons.json.Jsons; +import java.nio.charset.Charset; +import java.util.Base64; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.bson.BsonBinary; +import org.bson.BsonBoolean; +import org.bson.BsonDateTime; +import org.bson.BsonDecimal128; +import org.bson.BsonDocument; +import org.bson.BsonDouble; +import org.bson.BsonInt32; +import org.bson.BsonInt64; +import org.bson.BsonJavaScript; +import org.bson.BsonJavaScriptWithScope; +import org.bson.BsonNull; +import org.bson.BsonObjectId; +import org.bson.BsonRegularExpression; +import org.bson.BsonString; +import org.bson.BsonSymbol; +import org.bson.BsonTimestamp; +import org.bson.Document; +import org.bson.UuidRepresentation; +import org.bson.types.Decimal128; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Test; + +class MongoDbCdcEventUtilsTest { + + private static final String OBJECT_ID = "64f24244f95155351c4185b1"; + + @Test + void testGenerateObjectIdDocument() { + final String key = "{\"" + OBJECT_ID_FIELD + "\": \"" + OBJECT_ID + "\"}"; + JsonNode debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, key)); + + String updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + + assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); + assertEquals(key.replaceAll(OBJECT_ID_FIELD_PATTERN, DOCUMENT_OBJECT_ID_FIELD), updated); + + debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, "\"" + OBJECT_ID + "\"")); + updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); + assertEquals(Jsons.serialize(Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, OBJECT_ID))), updated); + } + + @Test + void testNormalizeObjectId() { + final JsonNode data = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( + Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID)))); + assertEquals(OBJECT_ID, data.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutObjectId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( + Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of()))); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode(Map.of())); + assertNull(dataWithoutId.get(DOCUMENT_OBJECT_ID_FIELD)); + + final JsonNode stringId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, "abcd"))); + assertEquals("abcd", stringId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + } + + @Test + void testNormalizeObjectIdNoSchema() { + var objectNode = (ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID))); + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, + Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID)))); + + final JsonNode data = MongoDbCdcEventUtils.normalizeObjectIdNoSchema(objectNode); + assertEquals(OBJECT_ID, data.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + assertEquals(OBJECT_ID, data.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + objectNode = (ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of())); + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of()))); + final JsonNode dataWithoutObjectId = MongoDbCdcEventUtils.normalizeObjectIdNoSchema(objectNode); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutId = MongoDbCdcEventUtils.normalizeObjectIdNoSchema((ObjectNode) Jsons.jsonNode(Map.of())); + assertNull(dataWithoutId.get(DOCUMENT_OBJECT_ID_FIELD)); + } + + @Test + void testTransformDataTypes() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + final UUID standardUuid = UUID.randomUUID(); + final UUID legacyUuid = UUID.randomUUID(); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")) + .append("field17", new BsonBinary(standardUuid, UuidRepresentation.STANDARD)) + .append("field18", new BsonBinary(legacyUuid, UuidRepresentation.JAVA_LEGACY)); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, document.keySet()); + + assertNotNull(transformed); + assertNotEquals(documentAsJson, Jsons.serialize(transformed)); + assertEquals(true, transformed.get("field1").asBoolean()); + assertEquals(1, transformed.get("field2").asInt()); + assertEquals(2, transformed.get("field3").asInt()); + assertEquals(3.0, transformed.get("field4").asDouble()); + assertEquals(4.0, transformed.get("field5").asDouble()); + assertEquals(expectedTimestamp, transformed.get("field6").asText()); + assertEquals(expectedTimestamp, transformed.get("field7").asText()); + assertEquals(Base64.getEncoder().encodeToString("test".getBytes(Charset.defaultCharset())), transformed.get("field8").asText()); + assertEquals("test2", transformed.get("field9").asText()); + assertEquals("test3", transformed.get("field10").asText()); + assertEquals(OBJECT_ID, transformed.get("field11").asText()); + assertEquals("code", transformed.get("field12").asText()); + assertEquals("code2", transformed.get("field13").get("code").asText()); + assertEquals("scope", transformed.get("field13").get("scope").get("scope").asText()); + assertEquals("pattern", transformed.get("field14").asText()); + assertFalse(transformed.has("field15")); + assertEquals("value", transformed.get("field16").get("key").asText()); + // Assert that UUIDs can be serialized. Currently, they will be represented as base 64 encoded + // strings. Since the original mongo source + // may have these UUIDs written by a variety of sources, each with different encodings - we cannot + // decode these back to the original UUID. + assertTrue(transformed.has("field17")); + assertTrue(transformed.has("field18")); + } + + @Test + void testTransformDataTypesWithFilteredFields() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, Set.of("field1", "field2", "field3")); + + assertNotNull(transformed); + assertNotEquals(documentAsJson, Jsons.serialize(transformed)); + assertEquals(true, transformed.get("field1").asBoolean()); + assertEquals(1, transformed.get("field2").asInt()); + assertEquals(2, transformed.get("field3").asInt()); + assertFalse(transformed.has("field4")); + assertFalse(transformed.has("field5")); + assertFalse(transformed.has("field6")); + assertFalse(transformed.has("field7")); + assertFalse(transformed.has("field8")); + assertFalse(transformed.has("field9")); + assertFalse(transformed.has("field10")); + assertFalse(transformed.has("field11")); + assertFalse(transformed.has("field12")); + assertFalse(transformed.has("field13")); + assertFalse(transformed.has("field14")); + assertFalse(transformed.has("field15")); + assertFalse(transformed.has("field16")); + } + + @Test + void testTransformDataTypesNoSchema() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypesNoSchema(documentAsJson); + + assertNotNull(transformed); + final var abDataNode = transformed.get(SCHEMALESS_MODE_DATA_FIELD); + assertNotEquals(documentAsJson, Jsons.serialize(abDataNode)); + assertEquals(true, abDataNode.get("field1").asBoolean()); + assertEquals(1, abDataNode.get("field2").asInt()); + assertEquals(2, abDataNode.get("field3").asInt()); + assertEquals(3.0, abDataNode.get("field4").asDouble()); + assertEquals(4.0, abDataNode.get("field5").asDouble()); + assertTrue(abDataNode.has("field6")); + assertTrue(abDataNode.has("field7")); + assertTrue(abDataNode.has("field8")); + assertTrue(abDataNode.has("field9")); + assertTrue(abDataNode.has("field10")); + assertTrue(abDataNode.has("field11")); + assertTrue(abDataNode.has("field12")); + assertTrue(abDataNode.has("field13")); + assertTrue(abDataNode.has("field14")); + assertFalse(abDataNode.has("field15")); + assertTrue(abDataNode.has("field16")); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java index f65d0882adf4..7e0ea6eaa222 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java @@ -5,6 +5,8 @@ package io.airbyte.integrations.source.mongodb.cdc; import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.RESYNC_DATA_OPTION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -20,6 +22,7 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; import com.mongodb.MongoCommandException; import com.mongodb.ServerAddress; import com.mongodb.client.AggregateIterable; @@ -30,12 +33,12 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; import com.mongodb.connection.ClusterDescription; import com.mongodb.connection.ClusterType; import com.mongodb.connection.ServerDescription; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; @@ -60,6 +63,7 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -67,6 +71,7 @@ import org.bson.BsonDocument; import org.bson.BsonString; import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -108,6 +113,8 @@ class MongoDbCdcInitializerTest { private MongoCursor findCursor; private ChangeStreamIterable changeStreamIterable; private MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector; + private static final List PIPELINE = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", List.of(COLLECTION)))); @BeforeEach void setUp() { @@ -134,6 +141,7 @@ void setUp() { when(clusterDescription.getServerDescriptions()).thenReturn(List.of(serverDescription)); when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); when(mongoClient.getDatabase(DATABASE)).thenReturn(mongoDatabase); when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); when(mongoDatabase.getCollection(COLLECTION)).thenReturn(mongoCollection); @@ -200,20 +208,53 @@ void testCreateCdcIteratorsFromInitialStateWithCompletedInitialSnapshot() { } @Test - void testCreateCdcIteratorsWithCompletedInitialSnapshotSavedOffsetInvalid() { + void testCreateCdcIteratorsWithCompletedInitialSnapshotSavedOffsetInvalidDefaultBehavior() { + when(changeStreamIterable.cursor()) + .thenReturn(mongoChangeStreamCursor) + .thenThrow(new MongoCommandException(new BsonDocument(), new ServerAddress())) + .thenReturn(mongoChangeStreamCursor); + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); + assertThrows(ConfigErrorException.class, () -> cdcInitializer.createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, + stateManager, EMITTED_AT, CONFIG)); + } + + @Test + void testCreateCdcIteratorsWithCompletedInitialSnapshotSavedOffsetFailOption() { + when(changeStreamIterable.cursor()) + .thenReturn(mongoChangeStreamCursor) + .thenThrow(new MongoCommandException(new BsonDocument(), new ServerAddress())) + .thenReturn(mongoChangeStreamCursor); + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); + assertThrows(ConfigErrorException.class, () -> cdcInitializer.createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, + stateManager, EMITTED_AT, CONFIG)); + } + + @Test + void testCreateCdcIteratorsWithCompletedInitialSnapshotSavedOffsetInvalidResyncOption() { + MongoDbSourceConfig resyncConfig = new MongoDbSourceConfig(createConfig(RESYNC_DATA_OPTION)); when(changeStreamIterable.cursor()) .thenReturn(mongoChangeStreamCursor) .thenThrow(new MongoCommandException(new BsonDocument(), new ServerAddress())) .thenReturn(mongoChangeStreamCursor); final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); final List> iterators = cdcInitializer - .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, resyncConfig); assertNotNull(iterators); assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); assertTrue(iterators.get(0).hasNext(), "Initial snapshot iterator should at least have one message if its snapshot state is set as complete but needs to start over due to invalid saved offset"); } + JsonNode createConfig(String cdcCursorFailBehaviour) { + return Jsons.jsonNode(ImmutableMap.builder() + .put(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of( + MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://host:12345/", + MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY, DATABASE)) + .put(INVALID_CDC_CURSOR_POSITION_PROPERTY, cdcCursorFailBehaviour) + .build()); + } + @Test void testUnableToExtractOffsetFromStateException() { final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java index 0bb2bfba05f3..f6ea739be583 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java @@ -6,6 +6,8 @@ import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_KEY; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_SCOPE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_SCOPE_VALUE; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_KEY; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_VALUE; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.HEARTBEAT_FREQUENCY_MS; @@ -24,12 +26,13 @@ class MongoDbCdcPropertiesTest { @Test void testDebeziumProperties() { final Properties debeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); - assertEquals(5, debeziumProperties.size()); + assertEquals(6, debeziumProperties.size()); assertEquals(CONNECTOR_CLASS_VALUE, debeziumProperties.get(CONNECTOR_CLASS_KEY)); assertEquals(SNAPSHOT_MODE_VALUE, debeziumProperties.get(SNAPSHOT_MODE_KEY)); assertEquals(CAPTURE_MODE_VALUE, debeziumProperties.get(CAPTURE_MODE_KEY)); assertEquals(HEARTBEAT_FREQUENCY_MS, debeziumProperties.get(HEARTBEAT_INTERVAL_KEY)); assertEquals(TOMBSTONE_ON_DELETE_VALUE, debeziumProperties.get(TOMBSTONE_ON_DELETE_KEY)); + assertEquals(CAPTURE_SCOPE_VALUE, debeziumProperties.get(CAPTURE_SCOPE_KEY)); } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java index 515903207ce2..935051f3b229 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; import org.junit.jupiter.api.Test; class MongoDbCdcSavedInfoFetcherTest { diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java index a4a253654cf1..d3bfed395b0e 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.core.type.TypeReference; -import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; import io.airbyte.protocol.models.Jsons; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java new file mode 100644 index 000000000000..20cf0b1ef9cb --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static com.mongodb.assertions.Assertions.assertNotNull; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.connector.mongodb.ResumeTokens; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; +import org.junit.jupiter.api.Test; + +class MongoDbCdcTargetPositionTest { + + private static final String OBJECT_ID = "64f24244f95155351c4185b1"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + private static final String OTHER_RESUME_TOKEN = "8264BEB9F3000000012B0229296E05"; + private static final ConfiguredAirbyteCatalog CATALOG = new ConfiguredAirbyteCatalog(); + private static final String DATABASE = "test-database"; + private static final List PIPELINE = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", Collections.emptyList()))); + + @Test + void testCreateTargetPosition() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + assertNotNull(targetPosition); + assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); + } + + @Test + void testReachedTargetPosition() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\": \\d+,", "\"ts_ms\": 1590221043000,")); + final ChangeEventWithMetadata changeEventWithMetadata2 = new ChangeEventWithMetadata(changeEvent); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata2)); + } + + @Test + void testReachedTargetPositionSnapshotEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionSnapshotLastEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot_last.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionFromHeartbeat() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( + Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), + 0); + + assertTrue(targetPosition.reachedTargetPosition(heartbeatTimestamp)); + assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); + } + + @Test + void testIsHeartbeatSupported() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + + assertTrue(targetPosition.isHeartbeatSupported()); + } + + @Test + void testExtractPositionFromHeartbeatOffset() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final BsonTimestamp resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeTokenDocument); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + + final Map sourceOffset = Map.of( + MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, resumeTokenTimestamp.getInc(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN, RESUME_TOKEN); + + final BsonTimestamp timestamp = targetPosition.extractPositionFromHeartbeatOffset(sourceOffset); + assertEquals(resumeTokenTimestamp, timestamp); + } + + @Test + void testIsEventAheadOfOffset() throws IOException { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final Map offset = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + final boolean result = targetPosition.isEventAheadOffset(offset, changeEventWithMetadata); + assertTrue(result); + } + + @Test + void testIsSameOffset() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); + + final Map offsetA = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + final Map offsetB = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + final Map offsetC = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, OTHER_RESUME_TOKEN), new TypeReference<>() {}); + + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); + + assertTrue(targetPosition.isSameOffset(offsetA, offsetA)); + assertTrue(targetPosition.isSameOffset(offsetA, offsetB)); + assertTrue(targetPosition.isSameOffset(offsetB, offsetA)); + assertFalse(targetPosition.isSameOffset(offsetA, offsetC)); + assertFalse(targetPosition.isSameOffset(offsetB, offsetC)); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoaderTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoaderTest.java new file mode 100644 index 000000000000..016c675f9735 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCustomLoaderTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import com.mongodb.ConnectionString; +import io.debezium.connector.mongodb.MongoDbConnectorConfig; +import io.debezium.connector.mongodb.MongoDbOffsetContext; +import io.debezium.connector.mongodb.ReplicaSets; +import io.debezium.connector.mongodb.ResumeTokens; +import io.debezium.connector.mongodb.connection.ReplicaSet; +import java.util.HashMap; +import java.util.Map; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbCustomLoaderTest { + + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + @Test + void testLoadOffsets() { + final String replicaSet = "replica-set"; + final BsonDocument resumeToken = ResumeTokens.fromData(RESUME_TOKEN); + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeToken); + final Map key = Map.of(KEY_REPLICA_SET, replicaSet); + final Map value = new HashMap<>(); + value.put(SOURCE_SECONDS, timestamp.getTime()); + value.put(SOURCE_ORDER, timestamp.getInc()); + value.put(SOURCE_RESUME_TOKEN, RESUME_TOKEN); + value.put(VALUE_TRANSACTION_ID, null); + final Map, Map> offsets = Map.of(key, value); + final MongoDbConnectorConfig mongoDbConnectorConfig = mock(MongoDbConnectorConfig.class); + final ReplicaSets replicaSets = ReplicaSets.of( + new ReplicaSet(new ConnectionString("mongodb://localhost:1234/?replicaSet=" + replicaSet))); + final MongoDbCustomLoader loader = new MongoDbCustomLoader(mongoDbConnectorConfig, replicaSets); + + final MongoDbOffsetContext context = loader.loadOffsets(offsets); + final Map offset = context.getReplicaSetOffsetContext(replicaSets.all().get(0)).getOffset(); + + assertNotNull(offset); + assertEquals(value, offset); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverterTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverterTest.java new file mode 100644 index 000000000000..83f3363ae97d --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumEventConverterTest.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumEventConverter; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class MongoDbDebeziumEventConverterTest { + + @Test + void testConvertRelationalDbChangeEvent() throws IOException { + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("insert_change_event.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("update_change_event.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("delete_change_event.json", ""); + final var eventConverter = new RelationalDbDebeziumEventConverter(cdcMetadataInjector, emittedAt); + + final AirbyteMessage actualInsert = eventConverter.toAirbyteMessage(insertChangeEvent); + final AirbyteMessage actualUpdate = eventConverter.toAirbyteMessage(updateChangeEvent); + final AirbyteMessage actualDelete = eventConverter.toAirbyteMessage(deleteChangeEvent); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "insert_message.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "update_message.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "delete_message.json"); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + } + + @Test + void testConvertMongoDbChangeEvent() throws IOException { + final String objectId = "64f24244f95155351c4185b1"; + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("mongodb/change_event_insert.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("mongodb/change_event_update.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("mongodb/change_event_delete.json", ""); + final ChangeEventWithMetadata deleteChangeEventNoBefore = mockChangeEvent("mongodb/change_event_delete_no_before.json", + "{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + objectId + "\\\"}"); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "mongodb/insert_airbyte_message.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "mongodb/update_airbyte_message.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "mongodb/delete_airbyte_message.json"); + final AirbyteMessage expectedDeleteNoBefore = createAirbyteMessage(stream, emittedAt, "mongodb/delete_no_before_airbyte_message.json"); + + final AirbyteMessage actualInsert = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedInsert), emittedAt, Jsons.emptyObject()) + .toAirbyteMessage(insertChangeEvent); + final AirbyteMessage actualUpdate = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedUpdate), emittedAt, Jsons.emptyObject()) + .toAirbyteMessage(updateChangeEvent); + final AirbyteMessage actualDelete = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedDelete), emittedAt, Jsons.emptyObject()) + .toAirbyteMessage(deleteChangeEvent); + final AirbyteMessage actualDeleteNoBefore = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedDeleteNoBefore), emittedAt, Jsons.emptyObject()) + .toAirbyteMessage(deleteChangeEventNoBefore); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + deepCompare(expectedDeleteNoBefore, actualDeleteNoBefore); + } + + @Test + void testConvertMongoDbChangeEventNoSchema() throws IOException { + final String objectId = "64f24244f95155351c4185b1"; + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("mongodb/change_event_insert.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("mongodb/change_event_update.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("mongodb/change_event_delete.json", ""); + final ChangeEventWithMetadata deleteChangeEventNoBefore = mockChangeEvent("mongodb/change_event_delete_no_before.json", + "{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + objectId + "\\\"}"); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "mongodb/insert_airbyte_message_no_schema.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "mongodb/update_airbyte_message_no_schema.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "mongodb/delete_airbyte_message_no_schema.json"); + final AirbyteMessage expectedDeleteNoBefore = createAirbyteMessage(stream, emittedAt, "mongodb/delete_no_before_airbyte_message_no_schema.json"); + + final ConfiguredAirbyteCatalog insertConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedInsert); + final ConfiguredAirbyteCatalog updateConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedUpdate); + final ConfiguredAirbyteCatalog deleteConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDelete); + final ConfiguredAirbyteCatalog deleteNoBeforeConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDeleteNoBefore); + + final JsonNode noSchemaConfig = + Jsons.jsonNode(Map.of(MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY, false)); + final AirbyteMessage actualInsert = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedInsert), emittedAt, noSchemaConfig) + .toAirbyteMessage(insertChangeEvent); + final AirbyteMessage actualUpdate = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedUpdate), emittedAt, noSchemaConfig) + .toAirbyteMessage(updateChangeEvent); + final AirbyteMessage actualDelete = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedDelete), emittedAt, noSchemaConfig) + .toAirbyteMessage(deleteChangeEvent); + final AirbyteMessage actualDeleteNoBefore = new MongoDbDebeziumEventConverter( + cdcMetadataInjector, buildFromAirbyteMessage(expectedDeleteNoBefore), emittedAt, noSchemaConfig) + .toAirbyteMessage(deleteChangeEventNoBefore); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + deepCompare(expectedDeleteNoBefore, actualDeleteNoBefore); + } + + @Test + void testConvertMongoDbChangeEventUnsupportedOperation() throws IOException { + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata unsupportedOperationEvent = mockChangeEvent("mongodb/change_event_unsupported.json", ""); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = mock(ConfiguredAirbyteCatalog.class); + final var eventConverter = new MongoDbDebeziumEventConverter(cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, Jsons.emptyObject()); + + assertThrows(IllegalArgumentException.class, () -> eventConverter.toAirbyteMessage(unsupportedOperationEvent)); + } + + private ConfiguredAirbyteCatalog buildFromAirbyteMessage(final AirbyteMessage airbyteMessage) { + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = new ConfiguredAirbyteCatalog(); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream(); + final AirbyteStream airbyteStream = new AirbyteStream(); + airbyteStream.setName(airbyteMessage.getRecord().getStream()); + airbyteStream.setNamespace(airbyteMessage.getRecord().getNamespace()); + airbyteStream.setJsonSchema(Jsons.jsonNode(Map.of("properties", airbyteMessage.getRecord().getData()))); + configuredAirbyteStream.setStream(airbyteStream); + configuredAirbyteCatalog.setStreams(List.of(configuredAirbyteStream)); + return configuredAirbyteCatalog; + } + + private static ChangeEventWithMetadata mockChangeEvent(final String resourceName, final String idValue) throws IOException { + final ChangeEvent mocked = mock(ChangeEvent.class); + final String resource = MoreResources.readResource(resourceName); + final String key = "{\"" + ID_FIELD + "\":\"" + idValue + "\"}"; + when(mocked.key()).thenReturn(key); + when(mocked.value()).thenReturn(resource); + + return new ChangeEventWithMetadata(mocked); + } + + private static AirbyteMessage createAirbyteMessage(final String stream, final Instant emittedAt, final String resourceName) throws IOException { + final String data = MoreResources.readResource(resourceName); + + final AirbyteRecordMessage recordMessage = new AirbyteRecordMessage() + .withStream(stream) + .withNamespace("public") + .withData(Jsons.deserialize(data)) + .withEmittedAt(emittedAt.toEpochMilli()); + + return new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(recordMessage); + } + + private static void deepCompare(final Object expected, final Object actual) { + assertEquals(Jsons.deserialize(Jsons.serialize(expected)), Jsons.deserialize(Jsons.serialize(actual))); + } + + public static class DummyMetadataInjector implements CdcMetadataInjector { + + @Override + public void addMetaData(final ObjectNode event, final JsonNode source) { + if (source.has("lsn")) { + final long lsn = source.get("lsn").asLong(); + event.put("_ab_cdc_lsn", lsn); + } + } + + @Override + public String namespace(final JsonNode source) { + return source.has("schema") ? source.get("schema").asText() : source.get("db").asText(); + } + + @Override + public String name(final JsonNode source) { + return source.has("table") ? source.get("table").asText() : source.get("collection").asText(); + } + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java similarity index 78% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java index aaf7805b3695..c635a8e37dde 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java @@ -2,26 +2,26 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import static io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.NAME_KEY; import static io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.TOPIC_PREFIX_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.COLLECTION_INCLUDE_LIST_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.DATABASE_INCLUDE_LIST_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_AUTHSOURCE_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_VALUE; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_STRING_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_PASSWORD_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_KEY; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_VALUE; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_USER_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.COLLECTION_INCLUDE_LIST_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.DATABASE_INCLUDE_LIST_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_AUTHSOURCE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_STRING_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_PASSWORD_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumPropertiesManager.MONGODB_USER_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -59,20 +59,15 @@ void testDebeziumProperties() { final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); - when(offsetManager.getOffsetFilePath()).thenReturn(PATH); when(catalog.getStreams()).thenReturn(streams); final Properties cdcProperties = new Properties(); cdcProperties.put("test", "value"); - final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( - cdcProperties, - config, - catalog, - offsetManager); + final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); - assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -94,20 +89,15 @@ void testDebeziumPropertiesConnectionStringCredentialsPlaceholder() { ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, config.get(CONNECTION_STRING_CONFIGURATION_KEY).asText() .replaceAll("mongodb://", "mongodb://" + CREDENTIALS_PLACEHOLDER)); - when(offsetManager.getOffsetFilePath()).thenReturn(PATH); when(catalog.getStreams()).thenReturn(streams); final Properties cdcProperties = new Properties(); cdcProperties.put("test", "value"); - final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( - cdcProperties, - config, - catalog, - offsetManager); + final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); - assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -128,20 +118,15 @@ void testDebeziumPropertiesQuotedConnectionString() { final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, "\"" + config.get(CONNECTION_STRING_CONFIGURATION_KEY) + "\""); - when(offsetManager.getOffsetFilePath()).thenReturn(PATH); when(catalog.getStreams()).thenReturn(streams); final Properties cdcProperties = new Properties(); cdcProperties.put("test", "value"); - final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( - cdcProperties, - config, - catalog, - offsetManager); + final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); - assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -162,20 +147,15 @@ void testDebeziumPropertiesNoCredentials() { final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); final JsonNode config = createConfiguration(Optional.empty(), Optional.empty(), Optional.empty()); - when(offsetManager.getOffsetFilePath()).thenReturn(PATH); when(catalog.getStreams()).thenReturn(streams); final Properties cdcProperties = new Properties(); cdcProperties.put("test", "value"); - final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( - cdcProperties, - config, - catalog, - offsetManager); + final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); - assertEquals(19 + cdcProperties.size(), debeziumProperties.size()); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); + assertEquals(17 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtilTest.java similarity index 99% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtilTest.java index bfb89edc34dd..c37e80ffa1e1 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumStateUtilTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java similarity index 80% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java index 98cde65bf61d..39fd65567eac 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java @@ -2,11 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mongodb; +package io.airbyte.integrations.source.mongodb.cdc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -14,32 +15,45 @@ import com.mongodb.client.ChangeStreamIterable; import com.mongodb.client.MongoChangeStreamCursor; import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.connector.mongodb.ResumeTokens; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; import org.junit.jupiter.api.Test; class MongoDbResumeTokenHelperTest { + private static final String DATABASE = "test-database"; + @Test void testRetrievingResumeToken() { final String resumeToken = "8264BEB9F3000000012B0229296E04"; final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + final List pipeline = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", Collections.emptyList()))); + when(mongoDatabase.watch(pipeline, BsonDocument.class)).thenReturn(changeStreamIterable); - final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, new ConfiguredAirbyteCatalog()); assertEquals(resumeTokenDocument, actualResumeToken); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java index b9d83ace1c47..30003b914a5a 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.source.mongodb.state; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; -import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_change_event.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/delete_change_event.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_change_event.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/delete_change_event.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/delete_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/delete_message.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_change_event.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/insert_change_event.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_change_event.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/insert_change_event.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/insert_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/insert_message.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_delete.json similarity index 76% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_delete.json index b573a78e6d19..89466aa511d8 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_delete.json @@ -1,5 +1,5 @@ { - "before": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0,\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "before": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", "after": null, "source": { "version": "2.2.0.Final", diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete_no_before.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_delete_no_before.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete_no_before.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_delete_no_before.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_insert.json similarity index 76% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_insert.json index d457f2f4250e..c56115e7a381 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_insert.json @@ -1,6 +1,6 @@ { "before": null, - "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0,\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", "source": { "version": "2.2.0.Final", "connector": "mongodb", diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_snapshot.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_snapshot.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot_last.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_snapshot_last.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot_last.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_snapshot_last.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_unsupported.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_unsupported.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_unsupported.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_unsupported.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_update.json similarity index 82% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_update.json index 074b9c8dffaf..1f132a350759 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/change_event_update.json @@ -1,6 +1,6 @@ { "before": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 1\",\"description\": \"This is document #1\",\"doubleField\": 1.0,\"intField\": 1,\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", - "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0,\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", "source": { "version": "2.2.0.Final", "connector": "mongodb", diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_airbyte_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_airbyte_message.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_airbyte_message_no_schema.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_airbyte_message_no_schema.json new file mode 100644 index 000000000000..c3bc65ae2dd0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_no_before_airbyte_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_no_before_airbyte_message.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json new file mode 100644 index 000000000000..89fe4fd4e9ed --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json @@ -0,0 +1,8 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/insert_airbyte_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/insert_airbyte_message.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/insert_airbyte_message_no_schema.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/insert_airbyte_message_no_schema.json new file mode 100644 index 000000000000..4242b47f8c55 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/insert_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/update_airbyte_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/update_airbyte_message.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/update_airbyte_message_no_schema.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/update_airbyte_message_no_schema.json new file mode 100644 index 000000000000..4242b47f8c55 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/mongodb/update_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json new file mode 100644 index 000000000000..5ea895b9847a --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json @@ -0,0 +1,5 @@ +[ + { + "_idType": "object" + } +] diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_change_event.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/update_change_event.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_change_event.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/update_change_event.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_message.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/update_message.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_message.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/update_message.json diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index 398b7ed61e64..034bc22d127a 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -1,38 +1,37 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.6.2' + cdkVersionRequired = '0.19.0' features = ['db-sources'] useLocalCdk = false } -configurations.all { - resolutionStrategy { - force libs.jooq +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-try,-rawtypes" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-this-escape" } } - - application { mainClass = 'io.airbyte.integrations.source.mssql.MssqlSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation libs.postgresql - - implementation libs.debezium.sqlserver implementation 'com.microsoft.sqlserver:mssql-jdbc:10.2.1.jre8' + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'io.debezium:debezium-connector-sqlserver:2.4.0.Final' implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation 'org.awaitility:awaitility:4.2.0' + testFixturesImplementation 'org.testcontainers:mssqlserver:1.19.0' - testImplementation libs.testcontainers.mssqlserver - testFixturesImplementation libs.testcontainers.mssqlserver + testImplementation 'org.awaitility:awaitility:4.2.0' + testImplementation 'org.hamcrest:hamcrest-all:1.3' + testImplementation 'org.testcontainers:mssqlserver:1.19.0' } diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index 4fb0f5a2b412..69ff2c08b161 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 3.3.0 + dockerImageTag: 3.7.3 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql @@ -18,8 +18,12 @@ data: name: Microsoft SQL Server (MSSQL) registries: cloud: + # CI pipeline is broken for mssql + dockerImageTag: 3.6.1 enabled: true oss: + # CI pipeline is broken for mssql + dockerImageTag: 3.6.1 enabled: true releaseStage: alpha supportLevel: community diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java index 050e4be20d37..717d470a8390 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.source.mssql; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_EVENT_SERIAL_NO; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_LSN; @@ -11,10 +13,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil.MssqlDebeziumStateAttributes; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; -public class MssqlCdcConnectorMetadataInjector implements CdcMetadataInjector { +public class MssqlCdcConnectorMetadataInjector implements CdcMetadataInjector { private final long emittedAtConverted; @@ -44,6 +47,17 @@ public void addMetaData(final ObjectNode event, final JsonNode source) { event.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); } + @Override + public void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, + final String transactionTimestamp, + final MssqlDebeziumStateAttributes debeziumStateAttributes) { + record.put(CDC_UPDATED_AT, transactionTimestamp); + record.put(CDC_EVENT_SERIAL_NO, 1); + record.put(CDC_LSN, debeziumStateAttributes.lsn().toString()); + record.put(CDC_DELETED_AT, (String) null); + record.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); + } + @Override public String namespace(final JsonNode source) { return source.get("schema").asText(); diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index 0f5ee9e4e9eb..86ebc7a0537d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -8,7 +8,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.internals.mssql.MSSQLConverter; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; @@ -28,8 +27,6 @@ public class MssqlCdcHelper { private static final String REPLICATION_FIELD = "replication"; private static final String REPLICATION_TYPE_FIELD = "replication_type"; private static final String METHOD_FIELD = "method"; - private static final String CDC_SNAPSHOT_ISOLATION_FIELD = "snapshot_isolation"; - private static final String CDC_DATA_TO_SYNC_FIELD = "data_to_sync"; private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10L); @@ -41,69 +38,6 @@ public enum ReplicationMethod { CDC } - /** - * The default "SNAPSHOT" mode can prevent other (non-Airbyte) transactions from updating table rows - * while we snapshot. References: - * https://docs.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver15 - * https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-property-snapshot-isolation-mode - */ - public enum SnapshotIsolation { - - SNAPSHOT("Snapshot", "snapshot"), - READ_COMMITTED("Read Committed", "read_committed"); - - private final String snapshotIsolationLevel; - private final String debeziumIsolationMode; - - SnapshotIsolation(final String snapshotIsolationLevel, final String debeziumIsolationMode) { - this.snapshotIsolationLevel = snapshotIsolationLevel; - this.debeziumIsolationMode = debeziumIsolationMode; - } - - public String getDebeziumIsolationMode() { - return debeziumIsolationMode; - } - - public static SnapshotIsolation from(final String jsonValue) { - for (final SnapshotIsolation value : values()) { - if (value.snapshotIsolationLevel.equalsIgnoreCase(jsonValue)) { - return value; - } - } - throw new IllegalArgumentException("Unexpected snapshot isolation level: " + jsonValue); - } - - } - - // https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-property-snapshot-mode - public enum DataToSync { - - EXISTING_AND_NEW("Existing and New", "initial"), - NEW_CHANGES_ONLY("New Changes Only", "schema_only"); - - private final String dataToSyncConfig; - private final String debeziumSnapshotMode; - - DataToSync(final String value, final String debeziumSnapshotMode) { - this.dataToSyncConfig = value; - this.debeziumSnapshotMode = debeziumSnapshotMode; - } - - public String getDebeziumSnapshotMode() { - return debeziumSnapshotMode; - } - - public static DataToSync from(final String value) { - for (final DataToSync s : values()) { - if (s.dataToSyncConfig.equalsIgnoreCase(value)) { - return s; - } - } - throw new IllegalArgumentException("Unexpected data to sync setting: " + value); - } - - } - @VisibleForTesting static boolean isCdc(final JsonNode config) { // new replication method config since version 0.4.0 @@ -123,29 +57,7 @@ static boolean isCdc(final JsonNode config) { return false; } - @VisibleForTesting - static SnapshotIsolation getSnapshotIsolationConfig(final JsonNode config) { - // new replication method config since version 0.4.0 - if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { - final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); - final JsonNode snapshotIsolation = replicationConfig.get(CDC_SNAPSHOT_ISOLATION_FIELD); - return SnapshotIsolation.from(snapshotIsolation.asText()); - } - return SnapshotIsolation.SNAPSHOT; - } - - @VisibleForTesting - static DataToSync getDataToSyncConfig(final JsonNode config) { - // new replication method config since version 0.4.0 - if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { - final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); - final JsonNode dataToSync = replicationConfig.get(CDC_DATA_TO_SYNC_FIELD); - return DataToSync.from(dataToSync.asText()); - } - return DataToSync.EXISTING_AND_NEW; - } - - static Properties getDebeziumProperties(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final boolean isSnapshot) { + public static Properties getDebeziumProperties(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final boolean isSnapshot) { final JsonNode config = database.getSourceConfig(); final JsonNode dbConfig = database.getDatabaseConfig(); @@ -158,7 +70,8 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("provide.transaction.metadata", "false"); props.setProperty("converters", "mssql_converter"); - props.setProperty("mssql_converter.type", MSSQLConverter.class.getName()); + + props.setProperty("mssql_converter.type", MssqlDebeziumConverter.class.getName()); // If new stream(s) are added after a previously successful sync, // the snapshot.mode needs to be initial_only since we don't want to continue streaming changes @@ -166,10 +79,12 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi if (isSnapshot) { props.setProperty("snapshot.mode", "initial_only"); } else { - props.setProperty("snapshot.mode", getDataToSyncConfig(config).getDebeziumSnapshotMode()); + // If not in snapshot mode, initial will make sure that a snapshot is taken if the transaction log + // is rotated out. This will also end up read streaming changes from the transaction_log. + props.setProperty("snapshot.mode", "initial"); } - props.setProperty("snapshot.isolation.mode", getSnapshotIsolationConfig(config).getDebeziumIsolationMode()); + props.setProperty("snapshot.isolation.mode", "read_committed"); props.setProperty("schema.include.list", getSchema(catalog)); props.setProperty("database.names", config.get(JdbcUtils.DATABASE_KEY).asText()); @@ -179,8 +94,6 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi ? HEARTBEAT_INTERVAL_IN_TESTS : HEARTBEAT_INTERVAL; props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis())); - // TODO: enable heartbeats in MS SQL Server. - props.setProperty("heartbeat.interval.ms", "0"); if (config.has("ssl_method")) { final JsonNode sslConfig = config.get("ssl_method"); @@ -192,16 +105,17 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("driver.trustServerCertificate", "true"); } else if ("encrypted_verify_certificate".equals(sslMethod)) { props.setProperty("driver.encrypt", "true"); + props.setProperty("driver.trustServerCertificate", "false"); if (dbConfig.has("trustStore") && !dbConfig.get("trustStore").asText().isEmpty()) { - props.setProperty("database.ssl.truststore", dbConfig.get("trustStore").asText()); + props.setProperty("database.trustStore", dbConfig.get("trustStore").asText()); } if (dbConfig.has("trustStorePassword") && !dbConfig.get("trustStorePassword").asText().isEmpty()) { - props.setProperty("database.ssl.truststore.password", dbConfig.get("trustStorePassword").asText()); + props.setProperty("database.trustStorePassword", dbConfig.get("trustStorePassword").asText()); } if (dbConfig.has("hostNameInCertificate") && !dbConfig.get("hostNameInCertificate").asText().isEmpty()) { - props.setProperty("driver.hostNameInCertificate", dbConfig.get("hostNameInCertificate").asText()); + props.setProperty("database.hostNameInCertificate", dbConfig.get("hostNameInCertificate").asText()); } } } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java index 921a0178e185..6ad000250d32 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java @@ -20,7 +20,7 @@ public class MssqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedSchemaHistory; private final boolean isSavedSchemaHistoryCompressed; - protected MssqlCdcSavedInfoFetcher(final CdcState savedState) { + public MssqlCdcSavedInfoFetcher(final CdcState savedState) { final boolean savedStatePresent = savedState != null && savedState.getState() != null; this.savedOffset = savedStatePresent ? savedState.getState().get(MSSQL_CDC_OFFSET) : null; this.savedSchemaHistory = savedStatePresent ? savedState.getState().get(MSSQL_DB_HISTORY) : null; diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java index 7b733b3d284a..709c1bc12690 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java @@ -32,6 +32,11 @@ public MssqlCdcStateHandler(final StateManager stateManager) { this.stateManager = stateManager; } + @Override + public boolean isCdcCheckpointEnabled() { + return true; + } + @Override public AirbyteMessage saveState(final Map offset, final SchemaHistory dbHistory) { final Map state = new HashMap<>(); diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java new file mode 100644 index 000000000000..123459f386da --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; +import io.airbyte.commons.json.Jsons; +import io.debezium.connector.sqlserver.Lsn; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlCdcTargetPosition implements CdcTargetPosition { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlCdcTargetPosition.class); + + public static final Duration MAX_LSN_QUERY_DELAY = Duration.ZERO; + public static final Duration MAX_LSN_QUERY_DELAY_TEST = Duration.ofSeconds(1); + public final Lsn targetLsn; + + public MssqlCdcTargetPosition(final Lsn targetLsn) { + this.targetLsn = targetLsn; + } + + @Override + public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { + if (changeEventWithMetadata.isSnapshotEvent()) { + return false; + } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { + LOGGER.info("Signalling close because Snapshot is complete"); + return true; + } else { + final Lsn recordLsn = extractLsn(changeEventWithMetadata.eventValueAsJson()); + final boolean isEventLSNAfter = targetLsn.compareTo(recordLsn) <= 0; + if (isEventLSNAfter) { + LOGGER.info("Signalling close because record's LSN : " + recordLsn + " is after target LSN : " + targetLsn); + } + return isEventLSNAfter; + } + } + + @Override + public Lsn extractPositionFromHeartbeatOffset(final Map sourceOffset) { + final Object commitLsnValue = sourceOffset.get("commit_lsn"); + return (commitLsnValue == null) ? Lsn.NULL : Lsn.valueOf(commitLsnValue.toString()); + } + + private Lsn extractLsn(final JsonNode valueAsJson) { + return Optional.ofNullable(valueAsJson.get("source")) + .flatMap(source -> Optional.ofNullable(source.get("commit_lsn").asText())) + .map(Lsn::valueOf) + .orElseThrow(() -> new IllegalStateException("Could not find LSN")); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final MssqlCdcTargetPosition that = (MssqlCdcTargetPosition) o; + return targetLsn.equals(that.targetLsn); + } + + @Override + public int hashCode() { + return targetLsn.hashCode(); + } + + public static MssqlCdcTargetPosition getTargetPosition(final JdbcDatabase database, final String dbName) { + try { + // We might have to wait a bit before querying the max_lsn to give the CDC capture job + // a chance to catch up. This is important in tests, where reads might occur in quick succession + // which might leave the CT tables (which Debezium consumes) in a stale state. + final JsonNode sourceConfig = database.getSourceConfig(); + final Duration delay = (sourceConfig != null && sourceConfig.has("is_test") && sourceConfig.get("is_test").asBoolean()) + ? MAX_LSN_QUERY_DELAY_TEST + : MAX_LSN_QUERY_DELAY; + final String maxLsnQuery = """ + USE [%s]; + WAITFOR DELAY '%02d:%02d:%02d'; + SELECT sys.fn_cdc_get_max_lsn() AS max_lsn; + """.formatted(dbName, delay.toHours(), delay.toMinutesPart(), delay.toSecondsPart()); + // Query the high-water mark. + final List jsonNodes = database.bufferedResultSetQuery( + connection -> connection.createStatement().executeQuery(maxLsnQuery), + JdbcUtils.getDefaultSourceOperations()::rowToJson); + Preconditions.checkState(jsonNodes.size() == 1); + if (jsonNodes.get(0).get("max_lsn") != null) { + final Lsn maxLsn = Lsn.valueOf(jsonNodes.get(0).get("max_lsn").binaryValue()); + LOGGER.info("identified target lsn: " + maxLsn); + return new MssqlCdcTargetPosition(maxLsn); + } else { + throw new RuntimeException("SQL returned max LSN as null, this might be because the SQL Server Agent is not running. " + + "Please enable the Agent and try again (https://docs.microsoft.com/en-us/sql/ssms/agent/start-stop-or-pause-the-sql-server-agent-service)"); + } + } catch (final SQLException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isHeartbeatSupported() { + return true; + } + + @Override + public boolean reachedTargetPosition(Lsn positionFromHeartbeat) { + return positionFromHeartbeat.compareTo(targetLsn) >= 0; + } + + @Override + public boolean isEventAheadOffset(Map offset, ChangeEventWithMetadata event) { + if (offset == null || offset.size() != 1) { + return false; + } + final Lsn eventLsn = extractLsn(event.eventValueAsJson()); + final Lsn offsetLsn = offsetToLsn(offset); + return eventLsn.compareTo(offsetLsn) > 0; + } + + @Override + public boolean isSameOffset(Map offsetA, Map offsetB) { + if ((offsetA == null || offsetA.size() != 1) || (offsetB == null || offsetB.size() != 1)) { + return false; + } + return offsetToLsn(offsetA).equals(offsetToLsn(offsetB)); + } + + private Lsn offsetToLsn(Map offset) { + final JsonNode offsetJson = Jsons.deserialize((String) offset.values().toArray()[0]); + final JsonNode commitLsnJson = offsetJson.get("commit_lsn"); + return (commitLsnJson == null || commitLsnJson.isNull()) ? Lsn.NULL : Lsn.valueOf(commitLsnJson.asText()); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java new file mode 100644 index 000000000000..afa721f609dd --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.microsoft.sqlserver.jdbc.Geography; +import com.microsoft.sqlserver.jdbc.Geometry; +import com.microsoft.sqlserver.jdbc.SQLServerException; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumConverterUtils; +import io.debezium.spi.converter.CustomConverter; +import io.debezium.spi.converter.RelationalColumn; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Base64; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import microsoft.sql.DateTimeOffset; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlDebeziumConverter implements CustomConverter { + + private final Logger LOGGER = LoggerFactory.getLogger(MssqlDebeziumConverter.class); + + private final Set BINARY = Set.of("VARBINARY", "BINARY"); + private final Set DATETIME_TYPES = Set.of("DATETIME", "DATETIME2", "SMALLDATETIME"); + private final String DATE = "DATE"; + private static final String DATETIMEOFFSET = "DATETIMEOFFSET"; + private static final String TIME_TYPE = "TIME"; + private static final String SMALLMONEY_TYPE = "SMALLMONEY"; + private static final String GEOMETRY = "GEOMETRY"; + private static final String GEOGRAPHY = "GEOGRAPHY"; + private static final String DEBEZIUM_DATETIMEOFFSET_FORMAT = "yyyy-MM-dd HH:mm:ss[.][SSSSSSS] XXX"; + + private static final String DATETIME_FORMAT_MICROSECONDS = "yyyy-MM-dd'T'HH:mm:ss[.][SSSSSS]"; + + @Override + public void configure(Properties props) {} + + @Override + public void converterFor(final RelationalColumn field, + final ConverterRegistration registration) { + if (DATE.equalsIgnoreCase(field.typeName())) { + registerDate(field, registration); + } else if (DATETIME_TYPES.contains(field.typeName().toUpperCase())) { + registerDatetime(field, registration); + } else if (SMALLMONEY_TYPE.equalsIgnoreCase(field.typeName())) { + registerMoney(field, registration); + } else if (BINARY.contains(field.typeName().toUpperCase())) { + registerBinary(field, registration); + } else if (GEOMETRY.equalsIgnoreCase(field.typeName())) { + registerGeometry(field, registration); + } else if (GEOGRAPHY.equalsIgnoreCase(field.typeName())) { + registerGeography(field, registration); + } else if (TIME_TYPE.equalsIgnoreCase(field.typeName())) { + registerTime(field, registration); + } else if (DATETIMEOFFSET.equalsIgnoreCase(field.typeName())) { + registerDateTimeOffSet(field, registration); + } + } + + private void registerGeometry(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + try { + return Geometry.deserialize((byte[]) input).toString(); + } catch (SQLServerException e) { + LOGGER.error(e.getMessage()); + } + } + + LOGGER.warn("Uncovered Geometry class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerGeography(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + try { + return Geography.deserialize((byte[]) input).toString(); + } catch (SQLServerException e) { + LOGGER.error(e.getMessage()); + } + } + + LOGGER.warn("Uncovered Geography class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerDate(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + if (field.typeName().equalsIgnoreCase("DATE")) { + return DateTimeConverter.convertToDate(input); + } + return DateTimeConverter.convertToTimestamp(input); + }); + } + + private void registerDatetime(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), + input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + final LocalDateTime localDateTime = ((Timestamp) input).toLocalDateTime(); + return localDateTime.format(DateTimeFormatter.ofPattern(DATETIME_FORMAT_MICROSECONDS)); + }); + + } + + private void registerDateTimeOffSet(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof DateTimeOffset) { + return DataTypeUtils.toISO8601String( + OffsetDateTime.parse(input.toString(), + DateTimeFormatter.ofPattern(DEBEZIUM_DATETIMEOFFSET_FORMAT))); + } + + LOGGER.warn("Uncovered DateTimeOffSet class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerTime(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof Timestamp) { + return DataTypeUtils.toISOTimeString(((Timestamp) input).toLocalDateTime()); + } + + LOGGER.warn("Uncovered time class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerMoney(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.float64(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof BigDecimal) { + return ((BigDecimal) input).doubleValue(); + } + + LOGGER.warn("Uncovered money class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerBinary(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + return Base64.getEncoder().encodeToString((byte[]) input); + } + + LOGGER.warn("Uncovered binary class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlQueryUtils.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlQueryUtils.java new file mode 100644 index 000000000000..fd69a8bcc73b --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlQueryUtils.java @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.integrations.source.mssql; + +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getIdentifierWithQuoting; +import static io.airbyte.integrations.source.mssql.MssqlSource.HIERARCHYID; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CursorBasedStatus; +import io.airbyte.cdk.integrations.source.relationaldb.models.InternalModels.StateType; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.math.BigDecimal; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class to define constants related to querying mssql + */ +public class MssqlQueryUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlQueryUtils.class); + private static final String MAX_OC_VALUE_QUERY = + """ + SELECT MAX(%s) as %s FROM %s; + """; + + public record TableSizeInfo(Long tableSize, Long avgRowLength) {} + + private static final String MAX_CURSOR_VALUE_QUERY = + """ + SELECT %s FROM %s WHERE %s = (SELECT MAX(%s) FROM %s); + """; + public static final String INDEX_QUERY = "EXEC sp_helpindex N'%s'"; + + public record Index( + @JsonProperty("index_name") String name, + @JsonProperty("index_description") String description, + @JsonProperty("index_keys") String keys) {} + + public static final String TABLE_ESTIMATE_QUERY = + """ + EXEC sp_spaceused N'"%s"."%s"' + """; + + public static final String MAX_OC_COL = "max_oc"; + public static final String DATA_SIZE_HUMAN_READABLE = "data"; + public static final String NUM_ROWS = "rows"; + + public static void getIndexInfoForStreams(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final String quoteString) { + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + final String streamName = stream.getStream().getName(); + final String schemaName = stream.getStream().getNamespace(); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString); + LOGGER.info("Discovering indexes for table {}", fullTableName); + try { + final String query = INDEX_QUERY.formatted(fullTableName); + LOGGER.debug("Index lookup query: {}", query); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(query).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + if (jsonNodes != null) { + jsonNodes.stream().map(node -> Jsons.convertValue(node, Index.class)) + .forEach(i -> LOGGER.info("Index {}", i)); + } + } catch (final Exception ex) { + LOGGER.info("Failed to get index for {}", fullTableName); + } + } + + } + + public static String getMaxOcValueForStream(final JdbcDatabase database, + final ConfiguredAirbyteStream stream, + final String ocFieldName, + final String quoteString) { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); + final String maxOcQuery = String.format(MAX_OC_VALUE_QUERY, + getIdentifierWithQuoting(ocFieldName, quoteString), + MAX_OC_COL, + fullTableName); + LOGGER.info("Querying for max oc value: {}", maxOcQuery); + try { + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(maxOcQuery).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + if (jsonNodes.get(0).get(MAX_OC_COL) == null) { + LOGGER.info("Max PK is null for table {} - this could indicate an empty table", fullTableName); + return null; + } + return jsonNodes.get(0).get(MAX_OC_COL).asText(); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + private static long toBytes(final String filesize) { + long returnValue = -1; + final Pattern patt = Pattern.compile("([\\d.]+)[\s+]*([GMK]B)", Pattern.CASE_INSENSITIVE); + final Matcher matcher = patt.matcher(filesize); + Map powerMap = new HashMap(); + powerMap.put("GB", 3); + powerMap.put("MB", 2); + powerMap.put("KB", 1); + if (matcher.find()) { + String number = matcher.group(1).trim(); + int pow = powerMap.get(matcher.group(2).toUpperCase()); + BigDecimal bytes = new BigDecimal(number); + bytes = bytes.multiply(BigDecimal.valueOf(1024).pow(pow)); + returnValue = bytes.longValue(); + } + return returnValue; + } + + public static Map getTableSizeInfoForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final Map tableSizeInfoMap = new HashMap<>(); + streams.forEach(stream -> { + try { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(name, namespace, quoteString); + final List tableEstimateResult = getTableEstimate(database, namespace, name); + + if (tableEstimateResult != null + && tableEstimateResult.size() == 1 + && tableEstimateResult.get(0).get(DATA_SIZE_HUMAN_READABLE) != null + && tableEstimateResult.get(0).get(NUM_ROWS) != null) { + final long tableEstimateBytes = toBytes(tableEstimateResult.get(0).get(DATA_SIZE_HUMAN_READABLE).asText()); + final long numRows = tableEstimateResult.get(0).get(NUM_ROWS).asLong(); + final long avgTableRowSizeBytes = numRows > 0 ? tableEstimateBytes / numRows : 0; + LOGGER.info("Stream {} size estimate is {}, average row size estimate is {}", fullTableName, tableEstimateBytes, avgTableRowSizeBytes); + final TableSizeInfo tableSizeInfo = new TableSizeInfo(tableEstimateBytes, avgTableRowSizeBytes); + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + tableSizeInfoMap.put(namespacePair, tableSizeInfo); + } + } catch (final Exception e) { + LOGGER.warn("Error occurred while attempting to estimate sync size", e); + } + }); + return tableSizeInfoMap; + } + + /** + * Iterates through each stream and find the max cursor value and the record count which has that + * value based on each cursor field provided by the customer per stream This information is saved in + * a Hashmap with the mapping being the AirbyteStreamNameNamespacepair -> CursorBasedStatus + * + * @param database the source db + * @param streams streams to be synced + * @param stateManager stream stateManager + * @return Map of streams to statuses + */ + public static Map getCursorBasedSyncStatusForStreams(final JdbcDatabase database, + final List streams, + final StateManager stateManager, + final String quoteString) { + + final Map cursorBasedStatusMap = new HashMap<>(); + streams.forEach(stream -> { + try { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); + + final Optional cursorInfoOptional = + stateManager.getCursorInfo(new io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair(name, namespace)); + if (cursorInfoOptional.isEmpty()) { + throw new RuntimeException(String.format("Stream %s was not provided with an appropriate cursor", stream.getStream().getName())); + } + + LOGGER.info("Querying max cursor value for {}.{}", namespace, name); + final String cursorField = cursorInfoOptional.get().getCursorField(); + final String quotedCursorField = getIdentifierWithQuoting(cursorField, quoteString); + final String cursorBasedSyncStatusQuery = String.format(MAX_CURSOR_VALUE_QUERY, + quotedCursorField, + fullTableName, + quotedCursorField, + quotedCursorField, + fullTableName); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(cursorBasedSyncStatusQuery).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + final CursorBasedStatus cursorBasedStatus = new CursorBasedStatus(); + cursorBasedStatus.setStateType(StateType.CURSOR_BASED); + cursorBasedStatus.setVersion(2L); + cursorBasedStatus.setStreamName(name); + cursorBasedStatus.setStreamNamespace(namespace); + cursorBasedStatus.setCursorField(ImmutableList.of(cursorField)); + + if (!jsonNodes.isEmpty()) { + final JsonNode result = jsonNodes.get(0); + cursorBasedStatus.setCursor(result.get(cursorField).asText()); + cursorBasedStatus.setCursorRecordCount((long) jsonNodes.size()); + } + + cursorBasedStatusMap.put(new io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair(name, namespace), cursorBasedStatus); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + }); + + return cursorBasedStatusMap; + } + + private static List getTableEstimate(final JdbcDatabase database, final String namespace, final String name) + throws SQLException { + // Construct the table estimate query. + final String tableEstimateQuery = + String.format(TABLE_ESTIMATE_QUERY, namespace, name); + LOGGER.info("Querying for table estimate size: {}", tableEstimateQuery); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(tableEstimateQuery), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + LOGGER.debug("Estimate: {}", jsonNodes); + return jsonNodes; + } + + public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { + return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); + } + + /** + * There is no support for hierarchyid even in the native SQL Server JDBC driver. Its value can be + * converted to a nvarchar(4000) data type by calling the ToString() method. So we make a separate + * query to get Table's MetaData, check is there any hierarchyid columns, and wrap required fields + * with the ToString() function in the final Select query. Reference: + * https://docs.microsoft.com/en-us/sql/t-sql/data-types/hierarchyid-data-type-method-reference?view=sql-server-ver15#data-type-conversion + * Note: This is where the main logic for the same method in MssqlSource. Extracted logic in order + * to be used in MssqlInitialLoadRecordIterator + * + * @return the list with Column names updated to handle functions (if nay) properly + */ + public static String getWrappedColumnNames( + final JdbcDatabase database, + final String quoteString, + final List columnNames, + final String schemaName, + final String tableName) { + final List hierarchyIdColumns = new ArrayList<>(); + try { + final String identifierQuoteString = database.getMetaData().getIdentifierQuoteString(); + final SQLServerResultSetMetaData sqlServerResultSetMetaData = (SQLServerResultSetMetaData) database + .queryMetadata(String + .format("SELECT TOP 1 %s FROM %s", // only first row is enough to get field's type + enquoteIdentifierList(columnNames, quoteString), + getFullyQualifiedTableNameWithQuoting(schemaName, tableName, quoteString))); + + // metadata will be null if table doesn't contain records + if (sqlServerResultSetMetaData != null) { + for (int i = 1; i <= sqlServerResultSetMetaData.getColumnCount(); i++) { + if (HIERARCHYID.equals(sqlServerResultSetMetaData.getColumnTypeName(i))) { + hierarchyIdColumns.add(sqlServerResultSetMetaData.getColumnName(i)); + } + } + } + + // iterate through names and replace Hierarchyid field for query is with toString() function + // Eventually would get columns like this: testColumn.toString as "testColumn" + // toString function in SQL server is the only way to get human-readable value, but not mssql + // specific HEX value + return String.join(", ", columnNames.stream() + .map( + el -> hierarchyIdColumns.contains(el) ? String.format("%s.ToString() as %s%s%s", el, identifierQuoteString, el, identifierQuoteString) + : getIdentifierWithQuoting(el, quoteString)) + .toList()); + } catch (final SQLException e) { + LOGGER.error("Failed to fetch metadata to prepare a proper request.", e); + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 21ee7eb1b7bc..f7de282d5e8c 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -5,54 +5,77 @@ package io.airbyte.integrations.source.mssql; import static io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; -import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getIdentifierWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.logStreamSyncStatus; import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbReadUtil.convertNameNamespacePairFromV0; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbReadUtil.identifyStreamsForCursorBased; +import static io.airbyte.integrations.source.mssql.MssqlQueryUtils.getCursorBasedSyncStatusForStreams; +import static io.airbyte.integrations.source.mssql.MssqlQueryUtils.getTableSizeInfoForStreams; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.initPairToOrderedColumnInfoMap; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.streamsForInitialOrderedColumnLoad; import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.db.util.SSLCertificateUtils; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; -import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; -import io.airbyte.cdk.integrations.debezium.internals.mssql.MssqlCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordIterator; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordPublisher; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumShutdownProcedure; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumEventConverter; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CursorBasedStatus; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.integrations.source.mssql.cursor_based.MssqlCursorBasedStateManager; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadHandler; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStreamStateManager; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.CursorBasedStreams; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.InitialLoadStreams; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; import io.debezium.connector.sqlserver.Lsn; -import java.io.File; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.net.URI; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.sql.Connection; import java.sql.JDBCType; import java.sql.PreparedStatement; @@ -62,12 +85,17 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; +import java.util.Properties; import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,7 +108,7 @@ public class MssqlSource extends AbstractJdbcSource implements Source """; public static final String NULL_CURSOR_VALUE_WITH_SCHEMA_QUERY = """ - SELECT CAST(IIF(EXISTS(SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL), 1, 0) AS BIT) AS %s + SELECT CASE WHEN (SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL)=1 then 1 else 0 end as %s """; public static final String DRIVER_CLASS = DatabaseDriver.MSSQLSERVER.getDriverClassName(); public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset"; @@ -88,16 +116,18 @@ SELECT CAST(IIF(EXISTS(SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL), 1, 0) public static final String IS_COMPRESSED = "is_compressed"; public static final String CDC_LSN = "_ab_cdc_lsn"; public static final String CDC_EVENT_SERIAL_NO = "_ab_cdc_event_serial_no"; - private static final String HIERARCHYID = "hierarchyid"; + public static final String HIERARCHYID = "hierarchyid"; private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; public static final String TUNNEL_METHOD = "tunnel_method"; public static final String NO_TUNNEL = "NO_TUNNEL"; public static final String SSL_METHOD = "ssl_method"; public static final String SSL_METHOD_UNENCRYPTED = "unencrypted"; + + public static final String JDBC_DELIMITER = ";"; private List schemas; - public static Source sshWrappedSource(MssqlSource source) { + public static Source sshWrappedSource(final MssqlSource source) { return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); } @@ -107,10 +137,6 @@ public MssqlSource() { @Override protected AirbyteStateType getSupportedStateType(final JsonNode config) { - if (!featureFlags.useStreamCapableState()) { - return AirbyteStateType.LEGACY; - } - return MssqlCdcHelper.isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; } @@ -168,13 +194,7 @@ public AutoCloseableIterator queryTableFullRefresh(final JdbcDatabase } /** - * There is no support for hierarchyid even in the native SQL Server JDBC driver. Its value can be - * converted to a nvarchar(4000) data type by calling the ToString() method. So we make a separate - * query to get Table's MetaData, check is there any hierarchyid columns, and wrap required fields - * with the ToString() function in the final Select query. Reference: - * https://docs.microsoft.com/en-us/sql/t-sql/data-types/hierarchyid-data-type-method-reference?view=sql-server-ver15#data-type-conversion - * - * @return the list with Column names updated to handle functions (if nay) properly + * See {@link MssqlQueryUtils#getWrappedColumnNames} */ @Override protected String getWrappedColumnNames(final JdbcDatabase database, @@ -182,38 +202,7 @@ protected String getWrappedColumnNames(final JdbcDatabase database, final List columnNames, final String schemaName, final String tableName) { - final List hierarchyIdColumns = new ArrayList<>(); - try { - final String identifierQuoteString = database.getMetaData().getIdentifierQuoteString(); - final SQLServerResultSetMetaData sqlServerResultSetMetaData = (SQLServerResultSetMetaData) database - .queryMetadata(String - .format("SELECT TOP 1 %s FROM %s", // only first row is enough to get field's type - enquoteIdentifierList(columnNames, getQuoteString()), - getFullyQualifiedTableNameWithQuoting(schemaName, tableName, getQuoteString()))); - - // metadata will be null if table doesn't contain records - if (sqlServerResultSetMetaData != null) { - for (int i = 1; i <= sqlServerResultSetMetaData.getColumnCount(); i++) { - if (HIERARCHYID.equals(sqlServerResultSetMetaData.getColumnTypeName(i))) { - hierarchyIdColumns.add(sqlServerResultSetMetaData.getColumnName(i)); - } - } - } - - // iterate through names and replace Hierarchyid field for query is with toString() function - // Eventually would get columns like this: testColumn.toString as "testColumn" - // toString function in SQL server is the only way to get human readable value, but not mssql - // specific HEX value - return String.join(", ", columnNames.stream() - .map( - el -> hierarchyIdColumns.contains(el) ? String - .format("%s.ToString() as %s%s%s", el, identifierQuoteString, el, identifierQuoteString) - : getIdentifierWithQuoting(el, getQuoteString())) - .toList()); - } catch (final SQLException e) { - LOGGER.error("Failed to fetch metadata to prepare a proper request.", e); - throw new RuntimeException(e); - } + return MssqlQueryUtils.getWrappedColumnNames(database, quoteString, columnNames, schemaName, tableName); } @Override @@ -369,7 +358,6 @@ public List> getCheckOperations(final J checkOperations.add(database -> assertCdcEnabledInDb(config, database)); checkOperations.add(database -> assertCdcSchemaQueryable(config, database)); checkOperations.add(database -> assertSqlServerAgentRunning(database)); - checkOperations.add(database -> assertSnapshotIsolationAllowed(config, database)); } return checkOperations; @@ -429,18 +417,27 @@ protected void assertCdcSchemaQueryable(final JsonNode config, final JdbcDatabas // todo: ensure this works for Azure managed SQL (since it uses different sql server agent) protected void assertSqlServerAgentRunning(final JdbcDatabase database) throws SQLException { try { - final List queryResponse = database.queryJsons(connection -> { - final String sql = - "SELECT status_desc FROM sys.dm_server_services WHERE [servicename] LIKE 'SQL Server Agent%' OR [servicename] LIKE 'SQL Server 代理%' "; - final PreparedStatement ps = connection.prepareStatement(sql); - LOGGER.info(String.format("Checking that the SQL Server Agent is running using the query: '%s'", sql)); - return ps; - }, sourceOperations::rowToJson); - - if (!(queryResponse.get(0).get("status_desc").toString().contains("Running"))) { - throw new RuntimeException(String.format( - "The SQL Server Agent is not running. Current state: '%s'. Please check the documentation on ensuring SQL Server Agent is running.", - queryResponse.get(0).get("status_desc").toString())); + // EngineEdition property values can be found at + // https://learn.microsoft.com/en-us/sql/t-sql/functions/serverproperty-transact-sql?view=sql-server-ver16 + // SQL Server Agent is always running on SQL Managed Instance: + // https://learn.microsoft.com/en-us/azure/azure-sql/managed-instance/transact-sql-tsql-differences-sql-server?view=azuresql#sql-server-agent + final Integer engineEdition = database.queryInt("SELECT ServerProperty('EngineEdition')"); + if (engineEdition == 8) { + LOGGER.info(String.format("SQL Server Agent is assumed to be running when EngineEdition == '%s'", engineEdition)); + } else { + final List queryResponse = database.queryJsons(connection -> { + final String sql = + "SELECT status_desc FROM sys.dm_server_services WHERE [servicename] LIKE 'SQL Server Agent%' OR [servicename] LIKE 'SQL Server 代理%' "; + final PreparedStatement ps = connection.prepareStatement(sql); + LOGGER.info(String.format("Checking that the SQL Server Agent is running using the query: '%s'", sql)); + return ps; + }, sourceOperations::rowToJson); + + if (!(queryResponse.get(0).get("status_desc").toString().contains("Running"))) { + throw new RuntimeException(String.format( + "The SQL Server Agent is not running. Current state: '%s'. Please check the documentation on ensuring SQL Server Agent is running.", + queryResponse.get(0).get("status_desc").toString())); + } } } catch (final Exception e) { if (e.getCause() != null && e.getCause().getClass().equals(com.microsoft.sqlserver.jdbc.SQLServerException.class)) { @@ -453,35 +450,6 @@ protected void assertSqlServerAgentRunning(final JdbcDatabase database) throws S } } - protected void assertSnapshotIsolationAllowed(final JsonNode config, final JdbcDatabase database) - throws SQLException { - if (MssqlCdcHelper.getSnapshotIsolationConfig(config) != SnapshotIsolation.SNAPSHOT) { - return; - } - - final List queryResponse = database.queryJsons(connection -> { - final String sql = "SELECT name, snapshot_isolation_state FROM sys.databases WHERE name = ?"; - final PreparedStatement ps = connection.prepareStatement(sql); - ps.setString(1, config.get(JdbcUtils.DATABASE_KEY).asText()); - LOGGER.info(String.format( - "Checking that snapshot isolation is enabled on database '%s' using the query: '%s'", - config.get(JdbcUtils.DATABASE_KEY).asText(), sql)); - return ps; - }, sourceOperations::rowToJson); - - if (queryResponse.size() < 1) { - throw new RuntimeException(String.format( - "Couldn't find '%s' in sys.databases table. Please check the spelling and that the user has relevant permissions (see docs).", - config.get(JdbcUtils.DATABASE_KEY).asText())); - } - if (queryResponse.get(0).get("snapshot_isolation_state").asInt() != 1) { - throw new RuntimeException(String.format( - "Detected that snapshot isolation is not enabled for database '%s'. MSSQL CDC relies on snapshot isolation. " - + "Please check the documentation on how to enable snapshot isolation on MS SQL Server.", - config.get(JdbcUtils.DATABASE_KEY).asText())); - } - } - @Override public List> getIncrementalIterators(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, @@ -490,60 +458,85 @@ public List> getIncrementalIterators(final final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); if (MssqlCdcHelper.isCdc(sourceConfig) && isAnyStreamIncrementalSyncMode(catalog)) { - LOGGER.info("using CDC: {}", true); - final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); - final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(sourceConfig); - final var targetPosition = MssqlCdcTargetPosition.getTargetPosition(database, sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()); - final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>( - sourceConfig, - targetPosition, - true, - firstRecordWaitTime, - subsequentRecordWaitTime, - OptionalInt.empty()); - final MssqlCdcConnectorMetadataInjector mssqlCdcConnectorMetadataInjector = MssqlCdcConnectorMetadataInjector.getInstance(emittedAt); - - // Determine if new stream(s) have been added to the catalog after initial sync of existing streams - final List streamsToSnapshot = identifyStreamsToSnapshot(catalog, stateManager); - final ConfiguredAirbyteCatalog streamsToSnapshotCatalog = new ConfiguredAirbyteCatalog().withStreams(streamsToSnapshot); - - final Supplier> incrementalIteratorsSupplier = () -> handler.getIncrementalIterators( - catalog, - new MssqlCdcSavedInfoFetcher(stateManager.getCdcStateManager().getCdcState()), - new MssqlCdcStateHandler(stateManager), - mssqlCdcConnectorMetadataInjector, - MssqlCdcHelper.getDebeziumProperties(database, catalog, false), - DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, - emittedAt, - true); - - /* - * If the CDC state is null or there is no streams to snapshot, that means no stream has gone - * through the initial sync, so we return the list of incremental iterators - */ - if ((stateManager.getCdcStateManager().getCdcState() == null || - stateManager.getCdcStateManager().getCdcState().getState() == null || - streamsToSnapshot.isEmpty())) { - return List.of(incrementalIteratorsSupplier.get()); - } - - // Otherwise, we build the snapshot iterators for the newly added streams(s) - final AutoCloseableIterator snapshotIterators = - handler.getSnapshotIterators(streamsToSnapshotCatalog, - mssqlCdcConnectorMetadataInjector, - MssqlCdcHelper.getDebeziumProperties(database, catalog, true), - new MssqlCdcStateHandler(stateManager), - DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, - emittedAt); - /* - * The incremental iterators needs to be wrapped in a lazy iterator since only 1 Debezium engine for - * the DB can be running at a time - */ - return List.of(snapshotIterators, AutoCloseableIterators.lazyIterator(incrementalIteratorsSupplier, null)); + LOGGER.info("using OC + CDC"); + return MssqlInitialReadUtil.getCdcReadIterators(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString()); } else { - LOGGER.info("using CDC: {}", false); - return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); + if (isAnyStreamIncrementalSyncMode(catalog)) { + LOGGER.info("Syncing via Primary Key"); + final MssqlCursorBasedStateManager cursorBasedStateManager = new MssqlCursorBasedStateManager(stateManager.getRawStateMessages(), catalog); + final InitialLoadStreams initialLoadStreams = streamsForInitialOrderedColumnLoad(cursorBasedStateManager, catalog); + final Map pairToCursorBasedStatus = + getCursorBasedSyncStatusForStreams(database, initialLoadStreams.streamsForInitialLoad(), stateManager, quoteString); + final CursorBasedStreams cursorBasedStreams = + new CursorBasedStreams(identifyStreamsForCursorBased(catalog, initialLoadStreams.streamsForInitialLoad()), pairToCursorBasedStatus); + + logStreamSyncStatus(initialLoadStreams.streamsForInitialLoad(), "Primary Key"); + logStreamSyncStatus(cursorBasedStreams.streamsForCursorBased(), "Cursor"); + + final MssqlInitialLoadStreamStateManager mssqlInitialLoadStreamStateManager = new MssqlInitialLoadStreamStateManager(catalog, + initialLoadStreams, initPairToOrderedColumnInfoMap(database, initialLoadStreams, tableNameToTable, quoteString)); + final MssqlInitialLoadHandler initialLoadHandler = + new MssqlInitialLoadHandler(sourceConfig, database, new MssqlSourceOperations(), quoteString, mssqlInitialLoadStreamStateManager, + namespacePair -> Jsons.jsonNode(pairToCursorBasedStatus.get(convertNameNamespacePairFromV0(namespacePair))), + getTableSizeInfoForStreams(database, initialLoadStreams.streamsForInitialLoad(), quoteString)); + + final List> initialLoadIterator = new ArrayList<>(initialLoadHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(initialLoadStreams.streamsForInitialLoad()), + tableNameToTable, + emittedAt)); + + // Build Cursor based iterator + final List> cursorBasedIterator = + new ArrayList<>(super.getIncrementalIterators(database, + new ConfiguredAirbyteCatalog().withStreams( + cursorBasedStreams.streamsForCursorBased()), + tableNameToTable, + cursorBasedStateManager, emittedAt)); + + return Stream.of(initialLoadIterator, cursorBasedIterator).flatMap(Collection::stream).collect(Collectors.toList()); + + } } + + LOGGER.info("using CDC: {}", false); + return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); + } + + public AutoCloseableIterator getDebeziumSnapshotIterators( + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final CdcTargetPosition targetPosition, + final Duration firstRecordWaitTime, + final Duration subsequentRecordWaitTime, + final MssqlCdcConnectorMetadataInjector cdcMetadataInjector, + final Properties properties, + final CdcStateHandler cdcStateHandler, + final Instant emittedAt) { + + LOGGER.info("Running snapshot for " + catalog.getStreams().size() + " new tables"); + final var queue = new LinkedBlockingQueue>(AirbyteDebeziumHandler.QUEUE_CAPACITY); + + final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeDummyStateForSnapshotPurpose(); + final var emptyHistory = new AirbyteSchemaHistoryStorage.SchemaHistory>(Optional.empty(), false); + final var schemaHistoryManager = AirbyteSchemaHistoryStorage.initializeDBHistory(emptyHistory, cdcStateHandler.compressSchemaHistoryForState()); + final var propertiesManager = new RelationalDbDebeziumPropertiesManager(properties, config, catalog); + final DebeziumRecordPublisher tableSnapshotPublisher = new DebeziumRecordPublisher(propertiesManager); + tableSnapshotPublisher.start(queue, offsetManager, Optional.of(schemaHistoryManager)); + + final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( + queue, + targetPosition, + tableSnapshotPublisher::hasClosed, + new DebeziumShutdownProcedure<>(queue, tableSnapshotPublisher::close, tableSnapshotPublisher::hasClosed), + firstRecordWaitTime, + subsequentRecordWaitTime); + + final var eventConverter = new RelationalDbDebeziumEventConverter(cdcMetadataInjector, emittedAt); + return AutoCloseableIterators.concatWithEagerClose( + AutoCloseableIterators.transform(eventIterator, eventConverter::toAirbyteMessage), + AutoCloseableIterators.fromIterator( + MoreIterators.singletonIteratorFromSupplier( + cdcStateHandler::saveStateAfterCompletionOfSnapshotOfNewStreams))); } @Override @@ -604,30 +597,33 @@ private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { private void readSsl(final JsonNode sslMethod, final List additionalParameters) { final JsonNode config = sslMethod.get("ssl_method"); switch (config.get("ssl_method").asText()) { - case "unencrypted" -> additionalParameters.add("encrypt=false"); + case "unencrypted" -> { + additionalParameters.add("encrypt=false"); + additionalParameters.add("trustServerCertificate=true"); + } case "encrypted_trust_server_certificate" -> { additionalParameters.add("encrypt=true"); additionalParameters.add("trustServerCertificate=true"); } case "encrypted_verify_certificate" -> { additionalParameters.add("encrypt=true"); - - // trust store location code found at https://stackoverflow.com/a/56570588 - final String trustStoreLocation = Optional - .ofNullable(System.getProperty("javax.net.ssl.trustStore")) - .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); - final File trustStoreFile = new File(trustStoreLocation); - if (!trustStoreFile.exists()) { - throw new RuntimeException( - "Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " - + trustStoreLocation + " does not exist."); - } - final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - additionalParameters.add("trustStore=" + trustStoreLocation); - if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + additionalParameters.add("trustServerCertificate=false"); + + if (config.has("certificate")) { + String certificate = config.get("certificate").asText(); + String password = RandomStringUtils.randomAlphanumeric(100); + final URI keyStoreUri; + try { + keyStoreUri = SSLCertificateUtils.keyStoreFromCertificate(certificate, password, null, null); + } catch (IOException | KeyStoreException | NoSuchAlgorithmException | CertificateException e) { + throw new RuntimeException(e); + } additionalParameters - .add("trustStorePassword=" + config.get("trustStorePassword").asText()); + .add("trustStore=" + keyStoreUri.getPath()); + additionalParameters + .add("trustStorePassword=" + password); } + if (config.has("hostNameInCertificate")) { additionalParameters .add("hostNameInCertificate=" + config.get("hostNameInCertificate").asText()); @@ -636,10 +632,27 @@ private void readSsl(final JsonNode sslMethod, final List additionalPara } } + @Override + public Collection> readStreams(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) + throws Exception { + final JdbcDatabase database = createDatabase(config); + logPreSyncDebugData(database, catalog); + return super.readStreams(config, catalog, state); + } + private boolean cloudDeploymentMode() { return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); } + public Duration getConnectionTimeoutMssql(final Map connectionProperties) { + return getConnectionTimeout(connectionProperties); + } + + @Override + public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { + return createDatabase(sourceConfig, JDBC_DELIMITER); + } + public static void main(final String[] args) throws Exception { final Source source = MssqlSource.sshWrappedSource(new MssqlSource()); LOGGER.info("starting source: {}", MssqlSource.class); @@ -647,4 +660,10 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed source: {}", MssqlSource.class); } + @Override + protected void logPreSyncDebugData(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog) throws SQLException { + super.logPreSyncDebugData(database, catalog); + MssqlQueryUtils.getIndexInfoForStreams(database, catalog, getQuoteString()); + } + } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java index 7fb984c6d7d4..1e891a16a16d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -18,7 +18,6 @@ import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; -import java.nio.charset.Charset; import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -28,6 +27,7 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.Base64; import microsoft.sql.DateTimeOffset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +128,7 @@ protected void putBinary(final ObjectNode node, final int index) throws SQLException { final byte[] bytes = resultSet.getBytes(index); - final String value = new String(bytes, Charset.defaultCharset()); + final String value = Base64.getEncoder().encodeToString(bytes); node.put(columnName, value); } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlCdcStateConstants.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlCdcStateConstants.java new file mode 100644 index 000000000000..5236d102a567 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlCdcStateConstants.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.cdc; + +public class MssqlCdcStateConstants { + + public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset"; + public static final String MSSQL_DB_HISTORY = "mssql_db_history"; + public static final String IS_COMPRESSED = "is_compressed"; + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlDebeziumStateUtil.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlDebeziumStateUtil.java new file mode 100644 index 000000000000..f998fb5113bf --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cdc/MssqlDebeziumStateUtil.java @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.cdc; + +import static io.debezium.relational.RelationalDatabaseConnectorConfig.DATABASE_NAME; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordPublisher; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.config.Configuration; +import io.debezium.connector.common.OffsetReader; +import io.debezium.connector.sqlserver.Lsn; +import io.debezium.connector.sqlserver.SqlServerConnectorConfig; +import io.debezium.connector.sqlserver.SqlServerOffsetContext; +import io.debezium.connector.sqlserver.SqlServerOffsetContext.Loader; +import io.debezium.connector.sqlserver.SqlServerPartition; +import io.debezium.engine.ChangeEvent; +import io.debezium.pipeline.spi.Offsets; +import io.debezium.pipeline.spi.Partition; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; +import org.codehaus.plexus.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlDebeziumStateUtil implements DebeziumStateUtil { + + final static String LSN_OFFSET_INCLUDED_QUERY = """ + DECLARE @saved_lsn BINARY(10), @min_lsn BINARY(10), @max_lsn BINARY(10), @res BIT + -- Set @saved_lsn = 0x0000DF7C000006A80006 + Set @saved_lsn = ? + SELECT @min_lsn = MIN(start_lsn) FROM cdc.change_tables + SELECT @max_lsn = sys.fn_cdc_get_max_lsn() + IF (@saved_lsn >= @min_lsn) + Set @res = 1 + ELSE + Set @res = 0 + select @res as [included], @MIN_LSN as [min], @MAX_LSN as [max] + """; + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlDebeziumStateUtil.class); + + /** + * Generate initial state for debezium state. + */ + public JsonNode constructInitialDebeziumState(final Properties properties, + final ConfiguredAirbyteCatalog catalog, + final JdbcDatabase database) { + properties.setProperty("heartbeat.interval.ms", "0"); + final JsonNode highWaterMark = constructLsnSnapshotState(database, database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText()); + final AirbyteFileOffsetBackingStore emptyOffsetManager = AirbyteFileOffsetBackingStore.initializeState(null, + Optional.empty()); + final AirbyteSchemaHistoryStorage schemaHistoryStorage = + AirbyteSchemaHistoryStorage.initializeDBHistory(new SchemaHistory<>(Optional.empty(), false), false); + final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(); + final Instant engineStartTime = Instant.now(); + boolean schemaHistoryRead = false; + SchemaHistory schemaHistory = null; + final var debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(properties, database.getSourceConfig(), catalog); + try { + final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(debeziumPropertiesManager); + publisher.start(queue, emptyOffsetManager, Optional.of(schemaHistoryStorage)); + while (!publisher.hasClosed()) { + final ChangeEvent event = queue.poll(10, TimeUnit.SECONDS); + + // If no event such as an empty table, generating schema history may take a few cycles + // depending on the size of history. + schemaHistory = schemaHistoryStorage.read(); + schemaHistoryRead = Objects.nonNull(schemaHistory) && StringUtils.isNotBlank(schemaHistory.schema()); + + if (event != null || schemaHistoryRead) { + publisher.close(); + break; + } + + if (Duration.between(engineStartTime, Instant.now()).compareTo(Duration.ofMinutes(5)) > 0) { + LOGGER.error("No record is returned even after {} seconds of waiting, closing the engine", 300); + publisher.close(); + throw new RuntimeException( + "Building schema history has timed out. Please consider increasing the debezium wait time in advanced options."); + } + } + } catch (final InterruptedException ine) { + LOGGER.info("*** interrupted"); + } catch (final Exception e) { + throw new RuntimeException(e); + } + + final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState(highWaterMark, + Optional.empty()); + + final Map offset = offsetManager.read(); + if (!schemaHistoryRead) { + schemaHistory = schemaHistoryStorage.read(); + } + + assert !offset.isEmpty(); + assert Objects.nonNull(schemaHistory); + assert Objects.nonNull(schemaHistory.schema()); + + final JsonNode asJson = serialize(offset, schemaHistory); + LOGGER.info("Initial Debezium state constructed: {}", asJson); + + if (asJson.get(MssqlCdcStateConstants.MSSQL_DB_HISTORY).asText().isBlank()) { + throw new RuntimeException("Schema history snapshot returned empty history."); + } + return asJson; + + } + + private static JsonNode serialize(final Map offset, final SchemaHistory dbHistory) { + final Map state = new HashMap<>(); + state.put(MssqlCdcStateConstants.MSSQL_CDC_OFFSET, offset); + state.put(MssqlCdcStateConstants.MSSQL_DB_HISTORY, dbHistory.schema()); + state.put(MssqlCdcStateConstants.IS_COMPRESSED, dbHistory.isCompressed()); + + return Jsons.jsonNode(state); + } + + public static MssqlDebeziumStateAttributes getStateAttributesFromDB(final JdbcDatabase database) { + try (final Stream stream = database.unsafeResultSetQuery( + connection -> connection.createStatement().executeQuery("select sys.fn_cdc_get_max_lsn()"), + resultSet -> { + final byte[] lsnBinary = resultSet.getBytes(1); + Lsn lsn = Lsn.valueOf(lsnBinary); + return new MssqlDebeziumStateAttributes(lsn); + })) { + final List stateAttributes = stream.toList(); + assert stateAttributes.size() == 1; + return stateAttributes.get(0); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + public record MssqlDebeziumStateAttributes(Lsn lsn) {} + + /** + * Method to construct initial Debezium state which can be passed onto Debezium engine to make it + * process binlogs from a specific file and position and skip snapshot phase Example: + * ["test",{"server":"test","database":"test"}]" : + * "{"transaction_id":null,"event_serial_no":1,"commit_lsn":"00000644:00002ff8:0099","change_lsn":"0000062d:00017ff0:016d"}" + */ + JsonNode constructLsnSnapshotState(final JdbcDatabase database, final String dbName) { + return format(getStateAttributesFromDB(database), dbName); + } + + @VisibleForTesting + public JsonNode format(final MssqlDebeziumStateAttributes attributes, final String dbName) { + final String key = "[\"" + dbName + "\",{\"server\":\"" + dbName + "\",\"database\":\"" + dbName + "\"}]"; + final String value = + "{\"commit_lsn\":\"" + attributes.lsn.toString() + "\",\"snapshot\":true,\"snapshot_completed\":true" + + "}"; + + final Map result = new HashMap<>(); + result.put(key, value); + + final JsonNode jsonNode = Jsons.jsonNode(result); + LOGGER.info("Initial Debezium state offset constructed: {}", jsonNode); + + return jsonNode; + } + + public Optional savedOffset(final Properties baseProperties, + final ConfiguredAirbyteCatalog catalog, + final JsonNode cdcOffset, + final JsonNode config) { + if (Objects.isNull(cdcOffset)) { + return Optional.empty(); + } + + final var offsetManager = AirbyteFileOffsetBackingStore.initializeState(cdcOffset, Optional.empty()); + final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); + return parseSavedOffset(debeziumProperties); + } + + private Optional parseSavedOffset(final Properties properties) { + FileOffsetBackingStore fileOffsetBackingStore = null; + OffsetStorageReaderImpl offsetStorageReader = null; + + try { + fileOffsetBackingStore = getFileOffsetBackingStore(properties); + offsetStorageReader = getOffsetStorageReader(fileOffsetBackingStore, properties); + + final SqlServerConnectorConfig connectorConfig = new SqlServerConnectorConfig(Configuration.from(properties)); + final SqlServerOffsetContext.Loader loader = new Loader(connectorConfig); + final Set partitions = + Collections.singleton(new SqlServerPartition(connectorConfig.getLogicalName(), properties.getProperty(DATABASE_NAME.name()))); + final OffsetReader offsetReader = new OffsetReader<>(offsetStorageReader, loader); + final Map offsets = offsetReader.offsets(partitions); + return extractStateAttributes(partitions, offsets); + } finally { + LOGGER.info("Closing offsetStorageReader and fileOffsetBackingStore"); + if (offsetStorageReader != null) { + offsetStorageReader.close(); + } + + if (fileOffsetBackingStore != null) { + fileOffsetBackingStore.stop(); + } + + } + } + + private Optional extractStateAttributes(final Set partitions, + final Map offsets) { + boolean found = false; + for (final Partition partition : partitions) { + final SqlServerOffsetContext mssqlOffsetContext = offsets.get(partition); + + if (mssqlOffsetContext != null) { + found = true; + LOGGER.info("Found previous partition offset {}: {}", partition, mssqlOffsetContext.getOffset()); + } + } + + if (!found) { + LOGGER.info("No previous offsets found"); + return Optional.empty(); + } + + final Offsets of = Offsets.of(offsets); + final SqlServerOffsetContext previousOffset = of.getTheOnlyOffset(); + return Optional.of(new MssqlDebeziumStateAttributes(previousOffset.getChangePosition().getCommitLsn())); + } + + public boolean savedOffsetStillPresentOnServer(final JdbcDatabase database, final MssqlDebeziumStateAttributes savedState) { + final Lsn savedLsn = savedState.lsn(); + try (final Stream stream = database.unsafeResultSetQuery( + connection -> { + PreparedStatement stmt = connection.prepareStatement(LSN_OFFSET_INCLUDED_QUERY); + stmt.setBytes(1, savedLsn.getBinary()); + return stmt.executeQuery(); + }, + resultSet -> { + final byte[] minLsnBinary = resultSet.getBytes(2); + Lsn min_lsn = Lsn.valueOf(minLsnBinary); + final byte[] maxLsnBinary = resultSet.getBytes(3); + Lsn max_lsn = Lsn.valueOf(maxLsnBinary); + final Boolean included = resultSet.getBoolean(1); + LOGGER.info("{} lsn exists on server: [{}]. (min server lsn: {} max server lsn: {})", savedLsn.toString(), included, min_lsn.toString(), + max_lsn.toString()); + return included; + })) { + final List reses = stream.toList(); + assert reses.size() == 1; + + return reses.get(0); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cursor_based/MssqlCursorBasedStateManager.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cursor_based/MssqlCursorBasedStateManager.java new file mode 100644 index 000000000000..843947efe5ab --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/cursor_based/MssqlCursorBasedStateManager.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.cursor_based; + +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.MSSQL_STATE_VERSION; + +import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CursorBasedStatus; +import io.airbyte.cdk.integrations.source.relationaldb.models.InternalModels.StateType; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlCursorBasedStateManager extends StreamStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlCursorBasedStateManager.class); + + public MssqlCursorBasedStateManager(final List airbyteStateMessages, final ConfiguredAirbyteCatalog catalog) { + super(airbyteStateMessages, catalog); + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + if (pair.isPresent()) { + final Map pairToCursorInfoMap = getPairToCursorInfoMap(); + final Optional cursorInfo = Optional.ofNullable(pairToCursorInfoMap.get(pair.get())); + + if (cursorInfo.isPresent()) { + LOGGER.debug("Generating state message for {}...", pair); + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + // Temporarily include legacy state for backwards compatibility with the platform + .withStream(generateStreamState(pair.get(), cursorInfo.get())); + } else { + LOGGER.warn("Cursor information could not be located in state for stream {}. Returning a new, empty state message...", pair); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } else { + LOGGER.warn("Stream not provided. Returning a new, empty state message..."); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } + + /** + * Generates the stream state for the given stream and cursor information. + * + * @param airbyteStreamNameNamespacePair The stream. + * @param cursorInfo The current cursor. + * @return The {@link AirbyteStreamState} representing the current state of the stream. + */ + private AirbyteStreamState generateStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(airbyteStreamNameNamespacePair.getName()).withNamespace(airbyteStreamNameNamespacePair.getNamespace())) + .withStreamState(Jsons.jsonNode(generateDbStreamState(airbyteStreamNameNamespacePair, cursorInfo))); + } + + private CursorBasedStatus generateDbStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + final CursorBasedStatus state = new CursorBasedStatus(); + state.setStateType(StateType.CURSOR_BASED); + state.setVersion(MSSQL_STATE_VERSION); + state.setStreamName(airbyteStreamNameNamespacePair.getName()); + state.setStreamNamespace(airbyteStreamNameNamespacePair.getNamespace()); + state.setCursorField(cursorInfo.getCursorField() == null ? Collections.emptyList() : Lists.newArrayList(cursorInfo.getCursorField())); + state.setCursor(cursorInfo.getCursor()); + if (cursorInfo.getCursorRecordCount() > 0L) { + state.setCursorRecordCount(cursorInfo.getCursorRecordCount()); + } + return state; + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadGlobalStateManager.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadGlobalStateManager.java new file mode 100644 index 000000000000..e947e4052f41 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadGlobalStateManager.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.InitialLoadStreams; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.OrderedColumnInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class MssqlInitialLoadGlobalStateManager implements MssqlInitialLoadStateManager { + + private final Map pairToOrderedColLoadStatus; + private final Map pairToOrderedColInfo; + private final CdcState cdcState; + + // Only one global state is emitted, which is fanned out into many entries in the DB by platform. As + // a result, we need to keep track of streams that have completed the snapshot. + private final Set streamsThatHaveCompletedSnapshot; + + public MssqlInitialLoadGlobalStateManager(final InitialLoadStreams initialLoadStreams, + final Map pairToOrderedColInfo, + final CdcState cdcState, + final ConfiguredAirbyteCatalog catalog) { + this.cdcState = cdcState; + this.pairToOrderedColLoadStatus = MssqlInitialLoadStateManager.initPairToOrderedColumnLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); + this.pairToOrderedColInfo = pairToOrderedColInfo; + this.streamsThatHaveCompletedSnapshot = initStreamsCompletedSnapshot(initialLoadStreams, catalog); + } + + private static Set initStreamsCompletedSnapshot(final InitialLoadStreams initialLoadStreams, + final ConfiguredAirbyteCatalog catalog) { + + return catalog.getStreams().stream() + .filter(s -> !initialLoadStreams.streamsForInitialLoad().contains(s)) + .filter(s -> s.getSyncMode() == SyncMode.INCREMENTAL) + .map(s -> new AirbyteStreamNameNamespacePair(s.getStream().getName(), s.getStream().getNamespace())) + .collect(Collectors.toSet()); + } + + @Override + public AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus) { + final List streamStates = streamsThatHaveCompletedSnapshot.stream() + .map(s -> getAirbyteStreamState(s, Jsons.jsonNode(getFinalState(s)))) + .collect(Collectors.toList()); + + streamStates.add(getAirbyteStreamState(pair, (Jsons.jsonNode(ocLoadStatus)))); + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + private AirbyteStreamState getAirbyteStreamState(final AirbyteStreamNameNamespacePair pair, final JsonNode stateData) { + Preconditions.checkNotNull(pair); + Preconditions.checkNotNull(pair.getName()); + Preconditions.checkNotNull(pair.getNamespace()); + + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(pair.getName()).withNamespace(pair.getNamespace())) + .withStreamState(stateData); + } + + @Override + public void updateOrderedColumnLoadState(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus) { + pairToOrderedColLoadStatus.put(pair, ocLoadStatus); + } + + @Override + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + streamsThatHaveCompletedSnapshot.add(pair); + + final List streamStates = streamsThatHaveCompletedSnapshot.stream() + .map(s -> getAirbyteStreamState(s, Jsons.jsonNode(getFinalState(s)))) + .collect(Collectors.toList()); + + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + @Override + public OrderedColumnLoadStatus getOrderedColumnLoadStatus(final AirbyteStreamNameNamespacePair pair) { + return pairToOrderedColLoadStatus.get(pair); + } + + @Override + public OrderedColumnInfo getOrderedColumnInfo(final AirbyteStreamNameNamespacePair pair) { + return pairToOrderedColInfo.get(pair); + } + + private DbStreamState getFinalState(final AirbyteStreamNameNamespacePair pair) { + Preconditions.checkNotNull(pair); + Preconditions.checkNotNull(pair.getName()); + Preconditions.checkNotNull(pair.getNamespace()); + + return new DbStreamState() + .withStreamName(pair.getName()) + .withStreamNamespace(pair.getNamespace()) + .withCursorField(Collections.emptyList()) + .withCursor(null); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java new file mode 100644 index 000000000000..2000e363e87c --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_DATABASE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TYPE; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.SqlDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.commons.stream.AirbyteStreamUtils; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mssql.MssqlQueryUtils.TableSizeInfo; +import io.airbyte.integrations.source.mssql.MssqlSourceOperations; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.sql.DatabaseMetaData; +import java.sql.JDBCType; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlInitialLoadHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlInitialLoadHandler.class); + private static final long RECORD_LOGGING_SAMPLE_RATE = 1_000_000; + private final JsonNode config; + private final JdbcDatabase database; + private final MssqlSourceOperations sourceOperations; + private final String quoteString; + private final MssqlInitialLoadStateManager initialLoadStateManager; + private static final long QUERY_TARGET_SIZE_GB = 1_073_741_824; + private static final long DEFAULT_CHUNK_SIZE = 1_000_000; + private final Function streamStateForIncrementalRunSupplier; + final Map tableSizeInfoMap; + + public MssqlInitialLoadHandler( + final JsonNode config, + final JdbcDatabase database, + final MssqlSourceOperations sourceOperations, + final String quoteString, + final MssqlInitialLoadStateManager initialLoadStateManager, + final Function streamStateForIncrementalRunSupplier, + final Map tableSizeInfoMap) { + this.config = config; + this.database = database; + this.sourceOperations = sourceOperations; + this.quoteString = quoteString; + this.initialLoadStateManager = initialLoadStateManager; + this.streamStateForIncrementalRunSupplier = streamStateForIncrementalRunSupplier; + this.tableSizeInfoMap = tableSizeInfoMap; + } + + private static String getCatalog(final SqlDatabase database) { + return (database.getSourceConfig().has(JdbcUtils.DATABASE_KEY) ? database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText() : null); + } + + public static String discoverClusteredIndexForStream(final JdbcDatabase database, + final AirbyteStream stream) { + Map clusteredIndexes = new HashMap<>(); + try { + // Get all clustered index names without specifying a table name + clusteredIndexes = aggregateClusteredIndexes(database.bufferedResultSetQuery( + connection -> connection.getMetaData().getIndexInfo(getCatalog(database), stream.getNamespace(), stream.getName(), false, false), + r -> { + if (r.getShort(JDBC_COLUMN_TYPE) == DatabaseMetaData.tableIndexClustered) { + final String schemaName = + r.getObject(JDBC_COLUMN_SCHEMA_NAME) != null ? r.getString(JDBC_COLUMN_SCHEMA_NAME) : r.getString(JDBC_COLUMN_DATABASE_NAME); + final String streamName = JdbcUtils.getFullyQualifiedTableName(schemaName, r.getString(JDBC_COLUMN_TABLE_NAME)); + final String columnName = r.getString(JDBC_COLUMN_COLUMN_NAME); + return new ClusteredIndexAttributesFromDb(streamName, columnName); + } else { + return null; + } + })); + } catch (final SQLException e) { + LOGGER.debug(String.format("Could not retrieve clustered indexes without a table name (%s), not blocking, fall back to use pk.", e)); + } + return clusteredIndexes.getOrDefault(stream.getName(), null); + } + + @VisibleForTesting + public record ClusteredIndexAttributesFromDb(String streamName, + String columnName) {} + + /** + * Aggregate list of @param entries of StreamName and clustered index column name + * + * @return a map by StreamName to associated columns in clustered index. If clustered index has + * multiple columns, we always use the first column. + */ + @VisibleForTesting + static Map aggregateClusteredIndexes(final List entries) { + final Map result = new HashMap<>(); + entries.forEach(entry -> { + if (entry == null) { + return; + } + if (result.containsKey(entry.streamName())) { + return; + } + result.put(entry.streamName, entry.columnName()); + }); + return result; + } + + public List> getIncrementalIterators( + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final Instant emittedAt) { + final List> iteratorList = new ArrayList<>(); + for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { + final AirbyteStream stream = airbyteStream.getStream(); + final String streamName = stream.getName(); + final String namespace = stream.getNamespace(); + // TODO: need to select column according to indexing status of table. may not be primary key + List keys = new ArrayList<>(); + final String clusteredFirstColumn = discoverClusteredIndexForStream(database, stream); + if (clusteredFirstColumn == null) { + keys = stream.getSourceDefinedPrimaryKey().stream().flatMap(pk -> Stream.of(pk.get(0))).toList(); + } else { + keys.add(clusteredFirstColumn); + } + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamName, namespace); + final String fullyQualifiedTableName = DbSourceDiscoverUtil.getFullyQualifiedTableName(namespace, streamName); + if (!tableNameToTable.containsKey(fullyQualifiedTableName)) { + LOGGER.info("Skipping stream {} because it is not in the source", fullyQualifiedTableName); + continue; + } + if (airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) { + // Grab the selected fields to sync + final TableInfo> table = tableNameToTable.get(fullyQualifiedTableName); + final List selectedDatabaseFields = table.getFields() + .stream() + .map(CommonField::getName) + .filter(CatalogHelpers.getTopLevelFieldNames(airbyteStream)::contains) + .toList(); + keys.forEach(key -> { + if (!selectedDatabaseFields.contains(key)) { + selectedDatabaseFields.add(0, key); + } + }); + + final AutoCloseableIterator queryStream = + new MssqlInitialLoadRecordIterator(database, sourceOperations, quoteString, initialLoadStateManager, selectedDatabaseFields, pair, + calculateChunkSize(tableSizeInfoMap.get(pair), pair), isCompositePrimaryKey(airbyteStream)); + final AutoCloseableIterator recordIterator = + getRecordIterator(queryStream, streamName, namespace, emittedAt.toEpochMilli()); + final AutoCloseableIterator recordAndMessageIterator = augmentWithState(recordIterator, pair); + iteratorList.add(augmentWithLogs(recordAndMessageIterator, pair, streamName)); + } + } + return iteratorList; + } + + // Transforms the given iterator to create an {@link AirbyteRecordMessage} + private AutoCloseableIterator getRecordIterator( + final AutoCloseableIterator recordIterator, + final String streamName, + final String namespace, + final long emittedAt) { + return AutoCloseableIterators.transform(recordIterator, r -> new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(streamName) + .withNamespace(namespace) + .withEmittedAt(emittedAt) + .withData(r))); + } + + // Augments the given iterator with record count logs. + private AutoCloseableIterator augmentWithLogs(final AutoCloseableIterator iterator, + final AirbyteStreamNameNamespacePair pair, + final String streamName) { + final AtomicLong recordCount = new AtomicLong(); + return AutoCloseableIterators.transform(iterator, + AirbyteStreamUtils.convertFromNameAndNamespace(pair.getName(), pair.getNamespace()), + r -> { + final long count = recordCount.incrementAndGet(); + if (count % RECORD_LOGGING_SAMPLE_RATE == 0) { + LOGGER.info("Reading stream {}. Records read: {}", streamName, count); + } + return r; + }); + } + + private AutoCloseableIterator augmentWithState(final AutoCloseableIterator recordIterator, + final AirbyteStreamNameNamespacePair pair) { + final OrderedColumnLoadStatus currentOcLoadStatus = initialLoadStateManager.getOrderedColumnLoadStatus(pair); + final JsonNode incrementalState = + (currentOcLoadStatus == null || currentOcLoadStatus.getIncrementalState() == null) + ? streamStateForIncrementalRunSupplier.apply(pair) + : currentOcLoadStatus.getIncrementalState(); + + final Duration syncCheckpointDuration = + config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null + ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) + : MssqlInitialSyncStateIterator.SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() + : MssqlInitialSyncStateIterator.SYNC_CHECKPOINT_RECORDS; + + return AutoCloseableIterators.transformIterator( + r -> new MssqlInitialSyncStateIterator(r, pair, initialLoadStateManager, incrementalState, syncCheckpointDuration, syncCheckpointRecords), + recordIterator, pair); + } + + private static boolean isCompositePrimaryKey(final ConfiguredAirbyteStream stream) { + return stream.getStream().getSourceDefinedPrimaryKey().size() > 1; + } + + public static long calculateChunkSize(final TableSizeInfo tableSizeInfo, final AirbyteStreamNameNamespacePair pair) { + // If table size info could not be calculated, a default chunk size will be provided. + if (tableSizeInfo == null || tableSizeInfo.tableSize() == 0 || tableSizeInfo.avgRowLength() == 0) { + LOGGER.info("Chunk size could not be determined for pair: {}, defaulting to {} rows", pair, DEFAULT_CHUNK_SIZE); + return DEFAULT_CHUNK_SIZE; + } + final long avgRowLength = tableSizeInfo.avgRowLength(); + final long chunkSize = QUERY_TARGET_SIZE_GB / avgRowLength; + LOGGER.info("Chunk size determined for pair: {}, is {}", pair, chunkSize); + return chunkSize; + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadRecordIterator.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadRecordIterator.java new file mode 100644 index 000000000000..54c94aff8707 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadRecordIterator.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.AbstractIterator; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mssql.MssqlQueryUtils; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.OrderedColumnInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("try") +public class MssqlInitialLoadRecordIterator extends AbstractIterator + implements AutoCloseableIterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlInitialLoadRecordIterator.class); + + private AutoCloseableIterator currentIterator; + private final JdbcDatabase database; + private int numSubqueries = 0; + private final String quoteString; + private final JdbcCompatibleSourceOperations sourceOperations; + private final List columnNames; + private final AirbyteStreamNameNamespacePair pair; + private final MssqlInitialLoadStateManager initialLoadStateManager; + private final long chunkSize; + private final OrderedColumnInfo ocInfo; + private final boolean isCompositeKeyLoad; + + MssqlInitialLoadRecordIterator( + final JdbcDatabase database, + final JdbcCompatibleSourceOperations sourceOperations, + final String quoteString, + final MssqlInitialLoadStateManager initialLoadStateManager, + final List columnNames, + final AirbyteStreamNameNamespacePair pair, + final long chunkSize, + final boolean isCompositeKeyLoad) { + this.database = database; + this.sourceOperations = sourceOperations; + this.quoteString = quoteString; + this.initialLoadStateManager = initialLoadStateManager; + this.columnNames = columnNames; + this.pair = pair; + this.chunkSize = chunkSize; + this.ocInfo = initialLoadStateManager.getOrderedColumnInfo(pair); + this.isCompositeKeyLoad = isCompositeKeyLoad; + } + + @CheckForNull + @Override + protected JsonNode computeNext() { + if (shouldBuildNextSubquery()) { + try { + // We will only issue one query for a composite key load. If we have already processed all the data + // associated with this + // query, we should indicate that we are done processing for the given stream. + if (isCompositeKeyLoad && numSubqueries >= 1) { + return endOfData(); + } + // Previous stream (and connection) must be manually closed in this iterator. + if (currentIterator != null) { + currentIterator.close(); + } + + LOGGER.info("Subquery number : {}", numSubqueries); + final Stream stream = database.unsafeQuery( + this::getOcPreparedStatement, sourceOperations::rowToJson); + currentIterator = AutoCloseableIterators.fromStream(stream, pair); + numSubqueries++; + // If the current subquery has no records associated with it, the entire stream has been read. + if (!currentIterator.hasNext()) { + return endOfData(); + } + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + return currentIterator.next(); + } + + private boolean shouldBuildNextSubquery() { + // The next sub-query should be built if (i) it is the first subquery in the sequence. (ii) the + // previous subquery has finished. + return (currentIterator == null || !currentIterator.hasNext()); + } + + private PreparedStatement getOcPreparedStatement(final Connection connection) { + try { + final String tableName = pair.getName(); + final String schemaName = pair.getNamespace(); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, + quoteString); + LOGGER.info("Preparing query for table: {}", fullTableName); + final String wrappedColumnNames = MssqlQueryUtils.getWrappedColumnNames(database, quoteString, columnNames, schemaName, tableName); + final OrderedColumnLoadStatus ocLoadStatus = initialLoadStateManager.getOrderedColumnLoadStatus(pair); + if (ocLoadStatus == null) { + final String quotedCursorField = enquoteIdentifier(ocInfo.ocFieldName(), quoteString); + final String sql; + if (isCompositeKeyLoad) { + sql = "SELECT %s FROM %s ORDER BY %s".formatted(wrappedColumnNames, fullTableName, quotedCursorField); + } else { + sql = "SELECT TOP %s %s FROM %s ORDER BY %s".formatted(chunkSize, wrappedColumnNames, fullTableName, quotedCursorField); + } + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + LOGGER.info("Executing query for table {}: {}", tableName, sql); + return preparedStatement; + } else { + LOGGER.info("ocLoadStatus value is : {}", ocLoadStatus.getOrderedColVal()); + final String quotedCursorField = enquoteIdentifier(ocInfo.ocFieldName(), quoteString); + final String sql; + if (isCompositeKeyLoad) { + sql = "SELECT %s FROM %s WHERE %s >= ? ORDER BY %s".formatted(wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField); + } else { + // The ordered column max value could be null - this can happen in the case of empty tables. In this + // case, + // we can just issue a query without any chunking. + if (ocInfo.ocMaxValue() != null) { + sql = "SELECT TOP %s %s FROM %s WHERE %s > ? AND %s <= ? ORDER BY %s".formatted(chunkSize, wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField, quotedCursorField); + } else { + sql = "SELECT %s FROM %s WHERE %s > ? ORDER BY %s".formatted(wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField); + } + } + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + final JDBCType cursorFieldType = ocInfo.fieldType(); + sourceOperations.setCursorField(preparedStatement, 1, cursorFieldType, ocLoadStatus.getOrderedColVal()); + if (!isCompositeKeyLoad && ocInfo.ocMaxValue() != null) { + sourceOperations.setCursorField(preparedStatement, 2, cursorFieldType, ocInfo.ocMaxValue()); + } + LOGGER.info("Executing query for table {}: {}", tableName, sql); + return preparedStatement; + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + if (currentIterator != null) { + currentIterator.close(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadSourceOperations.java new file mode 100644 index 000000000000..506f1c26c257 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadSourceOperations.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mssql.MssqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mssql.MssqlSourceOperations; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil.MssqlDebeziumStateAttributes; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Optional; + +public class MssqlInitialLoadSourceOperations extends MssqlSourceOperations { + + private final Optional metadataInjector; + + public MssqlInitialLoadSourceOperations(final Optional metadataInjector) { + super(); + this.metadataInjector = metadataInjector; + } + + @Override + public JsonNode rowToJson(final ResultSet queryContext) throws SQLException { + if (metadataInjector.isPresent()) { + // the first call communicates with the database. after that the result is cached. + final ResultSetMetaData metadata = queryContext.getMetaData(); + final int columnCount = metadata.getColumnCount(); + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + for (int i = 1; i <= columnCount; i++) { + // attempt to access the column. this allows us to know if it is null before we do type-specific + // parsing. if it is null, we can move on. while awkward, this seems to be the agreed upon way of + // checking for null values with jdbc. + queryContext.getObject(i); + if (queryContext.wasNull()) { + continue; + } + + // convert to java types that will convert into reasonable json. + copyToJsonField(queryContext, i, jsonNode); + } + + metadataInjector.get().inject(jsonNode); + return jsonNode; + } else { + return super.rowToJson(queryContext); + } + } + + public static class CdcMetadataInjector { + + private final String transactionTimestamp; + private final MssqlDebeziumStateAttributes stateAttributes; + private final MssqlCdcConnectorMetadataInjector metadataInjector; + + public CdcMetadataInjector(final String transactionTimestamp, + final MssqlDebeziumStateAttributes stateAttributes, + final MssqlCdcConnectorMetadataInjector metadataInjector) { + this.transactionTimestamp = transactionTimestamp; + this.stateAttributes = stateAttributes; + this.metadataInjector = metadataInjector; + } + + private void inject(final ObjectNode record) { + metadataInjector.addMetaDataToRowsFetchedOutsideDebezium(record, transactionTimestamp, stateAttributes); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStateManager.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStateManager.java new file mode 100644 index 000000000000..c253ba765dfe --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStateManager.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.OrderedColumnInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +public interface MssqlInitialLoadStateManager { + + public static long MSSQL_STATE_VERSION = 2; + String STATE_TYPE_KEY = "state_type"; + String ORDERED_COL_STATE_TYPE = "ordered_column"; + + /** + * Returns an intermediate state message for the initial sync. + * + * @param pair pair + * @param ocLoadStatus ordered column load status + * @return state message + */ + AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus); + + /** + * Updates the {@link OrderedColumnLoadStatus} for the state associated with the given pair. + * + * @param pair pair + * @param ocLoadStatus updated status + */ + void updateOrderedColumnLoadState(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus); + + /** + * Returns the final state message for the initial sync.. + * + * @param pair pair + * @param streamStateForIncrementalRun incremental status + * @return state message + */ + AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun); + + /** + * Returns the previous state emitted. Represented as a {@link OrderedColumnLoadStatus} associated + * with the stream. + * + * @param pair pair + * @return load status + */ + OrderedColumnLoadStatus getOrderedColumnLoadStatus(final AirbyteStreamNameNamespacePair pair); + + /** + * Returns the current {@OrderedColumnInfo}, associated with the stream. This includes the data type + * and the column name associated with the stream. + * + * @param pair pair + * @return load status + */ + OrderedColumnInfo getOrderedColumnInfo(final AirbyteStreamNameNamespacePair pair); + + static Map initPairToOrderedColumnLoadStatusMap( + final Map pairToOcStatus) { + return pairToOcStatus.entrySet().stream() + .collect(Collectors.toMap( + e -> new AirbyteStreamNameNamespacePair(e.getKey().getName(), e.getKey().getNamespace()), + Entry::getValue)); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStreamStateManager.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStreamStateManager.java new file mode 100644 index 000000000000..4348e75e0348 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadStreamStateManager.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.InitialLoadStreams; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialReadUtil.OrderedColumnInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This state manager extends the StreamStateManager to enable writing the state_type and version + * keys to the stream state when they're going through the iterator Once we have verified that + * expanding StreamStateManager itself to include this functionality, this class will be removed + */ +public class MssqlInitialLoadStreamStateManager implements MssqlInitialLoadStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlInitialLoadStateManager.class); + private final Map pairToOrderedColLoadStatus; + + private final Map pairToOrderedColInfo; + + public MssqlInitialLoadStreamStateManager(final ConfiguredAirbyteCatalog catalog, + final InitialLoadStreams initialLoadStreams, + final Map pairToOrderedColInfo) { + this.pairToOrderedColInfo = pairToOrderedColInfo; + this.pairToOrderedColLoadStatus = MssqlInitialLoadStateManager.initPairToOrderedColumnLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); + } + + @Override + public void updateOrderedColumnLoadState(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus) { + pairToOrderedColLoadStatus.put(pair, ocLoadStatus); + } + + @Override + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(getAirbyteStreamState(pair, streamStateForIncrementalRun)); + } + + @Override + public OrderedColumnLoadStatus getOrderedColumnLoadStatus(final AirbyteStreamNameNamespacePair pair) { + return pairToOrderedColLoadStatus.get(pair); + } + + @Override + public OrderedColumnInfo getOrderedColumnInfo(final AirbyteStreamNameNamespacePair pair) { + return pairToOrderedColInfo.get(pair); + } + + @Override + public AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final OrderedColumnLoadStatus ocLoadStatus) { + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(getAirbyteStreamState(pair, Jsons.jsonNode(ocLoadStatus))); + } + + private AirbyteStreamState getAirbyteStreamState(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, final JsonNode stateData) { + Preconditions.checkNotNull(pair); + Preconditions.checkNotNull(pair.getName()); + Preconditions.checkNotNull(pair.getNamespace()); + LOGGER.info("State data for {}: {}", pair.getNamespace().concat("_").concat(pair.getName()), stateData); + + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(pair.getName()).withNamespace(pair.getNamespace())) + .withStreamState(stateData); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialReadUtil.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialReadUtil.java new file mode 100644 index 000000000000..24945d25430e --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialReadUtil.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import static io.airbyte.integrations.source.mssql.MssqlCdcHelper.getDebeziumProperties; +import static io.airbyte.integrations.source.mssql.MssqlQueryUtils.getTableSizeInfoForStreams; +import static io.airbyte.integrations.source.mssql.MssqlQueryUtils.prettyPrintConfiguredAirbyteStreamList; +import static io.airbyte.integrations.source.mssql.cdc.MssqlCdcStateConstants.MSSQL_CDC_OFFSET; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadHandler.discoverClusteredIndexForStream; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.ORDERED_COL_STATE_TYPE; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.STATE_TYPE_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Sets; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumEventConverter; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.CursorBasedStatus; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mssql.MssqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mssql.MssqlCdcSavedInfoFetcher; +import io.airbyte.integrations.source.mssql.MssqlCdcStateHandler; +import io.airbyte.integrations.source.mssql.MssqlCdcTargetPosition; +import io.airbyte.integrations.source.mssql.MssqlQueryUtils; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil.MssqlDebeziumStateAttributes; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadSourceOperations.CdcMetadataInjector; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import io.debezium.connector.sqlserver.Lsn; +import java.sql.JDBCType; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlInitialReadUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlInitialReadUtil.class); + + public record InitialLoadStreams(List streamsForInitialLoad, + Map pairToInitialLoadStatus) { + + } + + public record CursorBasedStreams(List streamsForCursorBased, + Map pairToCursorBasedStatus) { + + } + + public record OrderedColumnInfo(String ocFieldName, JDBCType fieldType, String ocMaxValue) {} + + public static List> getCdcReadIterators(final JdbcDatabase database, + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final StateManager stateManager, + final Instant emittedAt, + final String quoteString) { + final JsonNode sourceConfig = database.getSourceConfig(); + final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); + final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(sourceConfig); + LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); + // Determine the streams that need to be loaded via primary key sync. + final List> initialLoadIterator = new ArrayList<>(); + // Construct the initial state for Mssql. If there is already existing state, we use that instead + // since that is associated with the debezium state associated with the initial sync. + final MssqlDebeziumStateUtil mssqlDebeziumStateUtil = new MssqlDebeziumStateUtil(); + final JsonNode initialDebeziumState = mssqlDebeziumStateUtil.constructInitialDebeziumState( + getDebeziumProperties(database, catalog, false), catalog, database); + + final JsonNode state = + (stateManager.getCdcStateManager().getCdcState() == null || stateManager.getCdcStateManager().getCdcState().getState() == null) + ? initialDebeziumState + : Jsons.clone(stateManager.getCdcStateManager().getCdcState().getState()); + + final Optional savedOffset = mssqlDebeziumStateUtil.savedOffset( + getDebeziumProperties(database, catalog, true), catalog, state.get(MSSQL_CDC_OFFSET), sourceConfig); + final boolean savedOffsetStillPresentOnServer = + savedOffset.isPresent() && mssqlDebeziumStateUtil.savedOffsetStillPresentOnServer(database, savedOffset.get()); + + if (!savedOffsetStillPresentOnServer) { + LOGGER.warn("Saved offset no longer present on the server, Airbyte is going to trigger a sync from scratch"); + } + + final InitialLoadStreams initialLoadStreams = + cdcStreamsForInitialOrderedCoumnLoad(stateManager.getCdcStateManager(), catalog, savedOffsetStillPresentOnServer); + final CdcState stateToBeUsed = (!savedOffsetStillPresentOnServer || (stateManager.getCdcStateManager().getCdcState() == null + || stateManager.getCdcStateManager().getCdcState().getState() == null)) + ? new CdcState().withState(initialDebeziumState) + : stateManager.getCdcStateManager().getCdcState(); + + final MssqlCdcConnectorMetadataInjector metadataInjector = MssqlCdcConnectorMetadataInjector.getInstance(emittedAt); + // If there are streams to sync via ordered column load, build the relevant iterators. + if (!initialLoadStreams.streamsForInitialLoad().isEmpty()) { + LOGGER.info("Streams to be synced via ordered column : {}", initialLoadStreams.streamsForInitialLoad().size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(initialLoadStreams.streamsForInitialLoad())); + final MssqlInitialLoadStateManager initialLoadStateManager = + new MssqlInitialLoadGlobalStateManager(initialLoadStreams, + initPairToOrderedColumnInfoMap(database, initialLoadStreams, tableNameToTable, quoteString), + stateToBeUsed, catalog); + + final MssqlDebeziumStateAttributes stateAttributes = MssqlDebeziumStateUtil.getStateAttributesFromDB(database); + final MssqlInitialLoadSourceOperations sourceOperations = + new MssqlInitialLoadSourceOperations(Optional.of(new CdcMetadataInjector(emittedAt.toString(), stateAttributes, metadataInjector))); + + final MssqlInitialLoadHandler initialLoadHandler = new MssqlInitialLoadHandler(sourceConfig, database, + sourceOperations, quoteString, initialLoadStateManager, + namespacePair -> Jsons.emptyObject(), + getTableSizeInfoForStreams(database, initialLoadStreams.streamsForInitialLoad(), quoteString)); + + initialLoadIterator.addAll(initialLoadHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(initialLoadStreams.streamsForInitialLoad()), + tableNameToTable, + emittedAt)); + } else { + LOGGER.info("No streams will be synced via ordered column"); + } + + // Build the incremental CDC iterators. + final var targetPosition = MssqlCdcTargetPosition.getTargetPosition(database, sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()); + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>( + sourceConfig, + targetPosition, + true, + firstRecordWaitTime, + subsequentRecordWaitTime, + AirbyteDebeziumHandler.QUEUE_CAPACITY, + false); + + final var propertiesManager = new RelationalDbDebeziumPropertiesManager(getDebeziumProperties(database, catalog, false), sourceConfig, catalog); + final var eventConverter = new RelationalDbDebeziumEventConverter(metadataInjector, emittedAt); + final Supplier> incrementalIteratorsSupplier = () -> handler.getIncrementalIterators( + propertiesManager, eventConverter, new MssqlCdcSavedInfoFetcher(stateToBeUsed), new MssqlCdcStateHandler(stateManager)); + + // This starts processing the transaction logs as soon as initial sync is complete, + // this is a bit different from the current cdc syncs. + // We finish the current CDC once the initial snapshot is complete and the next sync starts + // processing the transaction logs + return Collections.singletonList( + AutoCloseableIterators.concatWithEagerClose( + Stream + .of(initialLoadIterator, Collections.singletonList(AutoCloseableIterators.lazyIterator(incrementalIteratorsSupplier, null))) + .flatMap(Collection::stream) + .collect(Collectors.toList()), + AirbyteTraceMessageUtility::emitStreamStatusTrace)); + } + + public static InitialLoadStreams cdcStreamsForInitialOrderedCoumnLoad(final CdcStateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final boolean savedOffsetStillPresentOnServer) { + if (!savedOffsetStillPresentOnServer) { + return new InitialLoadStreams( + fullCatalog.getStreams() + .stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .collect(Collectors.toList()), + new HashMap<>()); + } + final AirbyteStateMessage airbyteStateMessage = stateManager.getRawStateMessage(); + final Set streamsStillInOcSync = new HashSet<>(); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + final Map pairToInitialLoadStatus = new HashMap<>(); + if (airbyteStateMessage != null && airbyteStateMessage.getGlobal() != null && airbyteStateMessage.getGlobal().getStreamStates() != null) { + airbyteStateMessage.getGlobal().getStreamStates().forEach(stateMessage -> { + final JsonNode streamState = stateMessage.getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(ORDERED_COL_STATE_TYPE)) { + final OrderedColumnLoadStatus orderedColumnLoadStatus = Jsons.object(streamState, OrderedColumnLoadStatus.class); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + pairToInitialLoadStatus.put(pair, orderedColumnLoadStatus); + streamsStillInOcSync.add(pair); + } + } + }); + } + + final List streamForOcSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInOcSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamForOcSync::add); + final List newlyAddedStreams = identifyStreamsToSnapshot(fullCatalog, stateManager.getInitialStreamsSynced()); + streamForOcSync.addAll(newlyAddedStreams); + + return new InitialLoadStreams(streamForOcSync, pairToInitialLoadStatus); + } + + public static Map initPairToOrderedColumnInfoMap( + final JdbcDatabase database, + final InitialLoadStreams initialLoadStreams, + final Map>> tableNameToTable, + final String quoteString) { + final Map pairToOcInfoMap = new HashMap<>(); + // For every stream that is in initial ordered column sync, we want to maintain information about + // the current ordered column info associated with the stream + initialLoadStreams.streamsForInitialLoad.forEach(stream -> { + final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair = + new io.airbyte.protocol.models.AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + final OrderedColumnInfo ocInfo = getOrderedColumnInfo(database, stream, tableNameToTable, quoteString); + pairToOcInfoMap.put(pair, ocInfo); + }); + return pairToOcInfoMap; + } + + static OrderedColumnInfo getOrderedColumnInfo(final JdbcDatabase database, + final ConfiguredAirbyteStream stream, + final Map>> tableNameToTable, + final String quoteString) { + // For cursor-based syncs, we cannot always assume a ordered column field exists. We need to handle + // the case where it does not exist when we support cursor-based syncs. + // if (stream.getStream().getSourceDefinedPrimaryKey().size() > 1) { + // LOGGER.info("Composite primary key detected for {namespace, stream} : {}, {}", + // stream.getStream().getNamespace(), stream.getStream().getName()); + // } // TODO: validate the seleted column rather than primary key + final String clusterdIndexField = discoverClusteredIndexForStream(database, stream.getStream()); + final String ocFieldName = clusterdIndexField != null ? clusterdIndexField : stream.getStream().getSourceDefinedPrimaryKey().get(0).get(0); + LOGGER.info("selected ordered column field name: " + ocFieldName); + final String fullyQualifiedTableName = + DbSourceDiscoverUtil.getFullyQualifiedTableName(stream.getStream().getNamespace(), stream.getStream().getName()); + final TableInfo> table = tableNameToTable + .get(fullyQualifiedTableName); + final JDBCType ocFieldType = table.getFields().stream() + .filter(field -> field.getName().equals(ocFieldName)) + .findFirst().get().getType(); + + final String ocMaxValue = MssqlQueryUtils.getMaxOcValueForStream(database, stream, ocFieldName, quoteString); + return new OrderedColumnInfo(ocFieldName, ocFieldType, ocMaxValue); + } + + public static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public static InitialLoadStreams streamsForInitialOrderedColumnLoad(final StateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog) { + + final List rawStateMessages = stateManager.getRawStateMessages(); + final Set streamsStillInOrderedColumnSync = new HashSet<>(); + final Set alreadySeenStreamPairs = new HashSet<>(); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + final Map pairToInitialLoadStatus = new HashMap<>(); + + if (rawStateMessages != null) { + rawStateMessages.forEach(stateMessage -> { + final AirbyteStreamState stream = stateMessage.getStream(); + final JsonNode streamState = stream.getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStream().getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(ORDERED_COL_STATE_TYPE)) { + final OrderedColumnLoadStatus orderedColumnLoadStatus = Jsons.object(streamState, OrderedColumnLoadStatus.class); + pairToInitialLoadStatus.put(pair, orderedColumnLoadStatus); + streamsStillInOrderedColumnSync.add(pair); + } + } + alreadySeenStreamPairs.add(new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace())); + }); + } + final List streamsForPkSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInOrderedColumnSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamsForPkSync::add); + + final List newlyAddedStreams = identifyStreamsToSnapshot(fullCatalog, + Collections.unmodifiableSet(alreadySeenStreamPairs)); + streamsForPkSync.addAll(newlyAddedStreams); + return new InitialLoadStreams(streamsForPkSync.stream().filter((stream) -> !stream.getStream().getSourceDefinedPrimaryKey() + .isEmpty()).collect(Collectors.toList()), + pairToInitialLoadStatus); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialSyncStateIterator.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialSyncStateIterator.java new file mode 100644 index 000000000000..0fe6a872f75b --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialSyncStateIterator.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql.initialsync; + +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.MSSQL_STATE_VERSION; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.AbstractIterator; +import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; +import io.airbyte.cdk.integrations.source.relationaldb.models.InternalModels.StateType; +import io.airbyte.cdk.integrations.source.relationaldb.models.OrderedColumnLoadStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Iterator; +import java.util.Objects; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlInitialSyncStateIterator extends AbstractIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlInitialSyncStateIterator.class); + public static final Duration SYNC_CHECKPOINT_DURATION = DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; + public static final Integer SYNC_CHECKPOINT_RECORDS = DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; + + private final Iterator messageIterator; + private final AirbyteStreamNameNamespacePair pair; + private boolean hasEmittedFinalState = false; + private OrderedColumnLoadStatus ocStatus; + private final JsonNode streamStateForIncrementalRun; + private final MssqlInitialLoadStateManager stateManager; + private long recordCount = 0L; + private Instant lastCheckpoint = Instant.now(); + private final Duration syncCheckpointDuration; + private final Long syncCheckpointRecords; + private final String ocFieldName; + + public MssqlInitialSyncStateIterator(final Iterator messageIterator, + final AirbyteStreamNameNamespacePair pair, + final MssqlInitialLoadStateManager stateManager, + final JsonNode streamStateForIncrementalRun, + final Duration checkpointDuration, + final Long checkpointRecords) { + this.messageIterator = messageIterator; + this.pair = pair; + this.stateManager = stateManager; + this.streamStateForIncrementalRun = streamStateForIncrementalRun; + this.syncCheckpointDuration = checkpointDuration; + this.syncCheckpointRecords = checkpointRecords; + this.ocFieldName = stateManager.getOrderedColumnInfo(pair).ocFieldName(); + this.ocStatus = stateManager.getOrderedColumnLoadStatus(pair); + } + + @CheckForNull + @Override + protected AirbyteMessage computeNext() { + if (messageIterator.hasNext()) { + if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) + && Objects.nonNull(ocStatus)) { + LOGGER.info("Emitting initial sync ordered col state for stream {}, state is {}", pair, ocStatus); + recordCount = 0L; + lastCheckpoint = Instant.now(); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateManager.createIntermediateStateMessage(pair, ocStatus)); + } + // Use try-catch to catch Exception that could occur when connection to the database fails + try { + final AirbyteMessage message = messageIterator.next(); + if (Objects.nonNull(message)) { + final String lastOcVal = message.getRecord().getData().get(ocFieldName).asText(); + ocStatus = new OrderedColumnLoadStatus() + .withVersion(MSSQL_STATE_VERSION) + .withStateType(StateType.ORDERED_COLUMN) + .withOrderedCol(ocFieldName) + .withOrderedColVal(lastOcVal) + .withIncrementalState(streamStateForIncrementalRun); + stateManager.updateOrderedColumnLoadState(pair, ocStatus); + } + recordCount++; + return message; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } else if (!hasEmittedFinalState) { + hasEmittedFinalState = true; + final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); + LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(finalStateMessage); + } else { + return endOfData(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index 4ee4554d009e..005311b9e5a8 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MSSQL Source Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": ["host", "port", "database", "username", "password"], "properties": { "host": { "description": "The hostname of the database.", @@ -90,7 +90,7 @@ { "title": "Encrypted (verify certificate)", "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], + "required": ["ssl_method"], "properties": { "ssl_method": { "type": "string", @@ -100,7 +100,15 @@ "title": "Host Name In Certificate", "type": "string", "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 + "order": 0 + }, + "certificate": { + "title": "Certificate", + "type": "string", + "description": "certificate of the server, or of the CA that signed the server certificate", + "order": 1, + "airbyte_secret": true, + "multiline": true } } } @@ -124,22 +132,6 @@ "const": "CDC", "order": 0 }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index f229caaad68f..32c42ebea52c 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -257,7 +257,7 @@ protected void initTests() { .sourceType("binary") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'A' AS BINARY(1))", "null") - .addExpectedValues("A", null) + .addExpectedValues("QQ==", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -267,7 +267,7 @@ protected void initTests() { .fullSourceDataType("varbinary(3)") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null") - .addExpectedValues("ABC", null) + .addExpectedValues("QUJD", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java index 8f612d0327bb..0ebbcb72c7dc 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java @@ -6,13 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; @@ -27,27 +30,25 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.HashMap; +import java.util.List; +import org.jooq.SQLDialect; public abstract class AbstractSshMssqlSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "dbo.id_and_name"; private static final String STREAM_NAME2 = "dbo.starships"; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - public abstract SshTunnel.TunnelMethod getTunnelMethod(); - protected MsSQLTestDatabase testdb; - protected SshBastionContainer bastion; + private final SshBastionContainer bastion = new SshBastionContainer(); + private MsSQLTestDatabase testdb; @Override protected JsonNode getConfig() { try { return testdb.integrationTestConfigBuilder() - .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .withoutSsl() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), true)) .build(); } catch (IOException e) { throw new UncheckedIOException(e); @@ -56,23 +57,51 @@ protected JsonNode getConfig() { } } + private void populateDatabaseTestData() throws Exception { + final var outerConfig = testdb.integrationTestConfigBuilder() + .withSchemas("public") + .withoutSsl() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .build(); + SshTunnel.sshWrap( + outerConfig, + JdbcUtils.HOST_LIST_KEY, + JdbcUtils.PORT_LIST_KEY, + (CheckedFunction, Exception>) mangledConfig -> getDatabaseFromConfig(mangledConfig) + .query(ctx -> { + ctx.fetch("ALTER DATABASE %s SET AUTO_CLOSE OFF WITH NO_WAIT;", testdb.getDatabaseName()); + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); + ctx.fetch("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1, 'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');"); + return null; + })); + } + + private static Database getDatabaseFromConfig(final JsonNode config) { + return new Database( + DSLContextFactory.create( + config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format(DatabaseDriver.MSSQLSERVER.getUrlFormatString(), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), + config.get(JdbcUtils.DATABASE_KEY).asText()), + SQLDialect.DEFAULT)); + } + @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2017, ContainerModifier.NETWORK); - testdb = testdb - .with("ALTER DATABASE %s SET AUTO_CLOSE OFF WITH NO_WAIT;", testdb.getDatabaseName()) - .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") - .with("INSERT INTO id_and_name (id, name, born) VALUES " + - "(1, 'picard', '2124-03-04T01:01:01Z'), " + - "(2, 'crusher', '2124-03-04T01:01:01Z'), " + - "(3, 'vash', '2124-03-04T01:01:01Z');"); bastion.initAndStartBastion(testdb.getContainer().getNetwork()); + populateDatabaseTestData(); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - bastion.close(); - testdb.close(); + bastion.stopAndClose(); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index cd5f995b9461..671dc8e31634 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -12,8 +12,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; @@ -47,11 +45,6 @@ protected String getImageName() { return "airbyte/source-mssql:dev"; } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected ConnectorSpecification getSpec() throws Exception { return SshHelpers.getSpecAndInjectSsh(); @@ -113,9 +106,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) { \t@role_name = N'%s', \t@supports_net_changes = 0"""; testdb - .withSnapshotIsolation() - .withCdc() .withWaitUntilAgentRunning() + .withCdc() // create tables .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME) .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME2) @@ -125,6 +117,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) { // enable cdc on tables for designated role .with(enableCdcSqlFmt, SCHEMA_NAME, STREAM_NAME, CDC_ROLE_NAME) .with(enableCdcSqlFmt, SCHEMA_NAME, STREAM_NAME2, CDC_ROLE_NAME) + .withShortenedCapturePollingInterval() .withWaitUntilMaxLsnAvailable() // revoke user permissions .with("REVOKE ALL FROM %s CASCADE;", testdb.getUserName()) @@ -152,7 +145,7 @@ void testAddNewStreamToExistingSync() throws Exception { final List streamStates = stateMessages.get(0).getGlobal().getStreamStates(); assertEquals(3, recordMessages.size()); - assertEquals(1, stateMessages.size()); + assertEquals(2, stateMessages.size()); assertEquals(1, streamStates.size()); assertEquals(STREAM_NAME, streamStates.get(0).getStreamDescriptor().getName()); assertEquals(SCHEMA_NAME, streamStates.get(0).getStreamDescriptor().getNamespace()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index 14ed5321a0c4..adfa26005af3 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; @@ -22,15 +20,9 @@ protected JsonNode getConfig() { .build(); } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected Database setupDatabase() { testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022, ContainerModifier.AGENT) - .withSnapshotIsolation() .withCdc(); return testdb.getDatabase(); } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java index 37767d5f99b2..f2a311d6b455 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java @@ -8,7 +8,6 @@ import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.features.FeatureFlagsWrapper; -import java.util.Map; public class CloudDeploymentSslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTest { @@ -38,7 +37,7 @@ protected FeatureFlags featureFlags() { @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .withEncrytedTrustServerCertificate() .build(); } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index bcb433852253..4bdc5cecf61a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -12,8 +12,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; @@ -42,11 +40,6 @@ public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { protected MsSQLTestDatabase testdb; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java index 58fb23ab3023..8b11db5c3e77 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; public class MssqlSourceDatatypeTest extends AbstractMssqlSourceDatatypeTest { @@ -18,11 +16,6 @@ protected Database setupDatabase() { return testdb.getDatabase(); } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java index 276bcc7ee804..4990c606952a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java @@ -5,9 +5,7 @@ package io.airbyte.integrations.source.mssql; import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; -import org.junit.jupiter.api.Disabled; -@Disabled public class SshKeyMssqlSourceAcceptanceTest extends AbstractSshMssqlSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java index 61b015fc538a..35b0b57bf6f8 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java @@ -5,9 +5,7 @@ package io.airbyte.integrations.source.mssql; import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; -import org.junit.jupiter.api.Disabled; -@Disabled public class SshPasswordMssqlSourceAcceptanceTest extends AbstractSshMssqlSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java index 92b0858bf1ac..ccd887c9a4b9 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java @@ -6,24 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; -import java.util.Map; public class SslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTest { @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .withEncrytedTrustServerCertificate() .build(); } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) { final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-latest"); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json index 560e55333378..1f42c042e746 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json @@ -2,5 +2,6 @@ "host": "default", "port": 5555, "database": "default", - "username": "default" + "username": "default", + "password": "default" } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json index db86aeb7f765..c2f000494ee4 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MSSQL Source Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": ["host", "port", "database", "username", "password"], "properties": { "host": { "description": "The hostname of the database.", @@ -90,7 +90,7 @@ { "title": "Encrypted (verify certificate)", "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], + "required": ["ssl_method"], "properties": { "ssl_method": { "type": "string", @@ -100,7 +100,15 @@ "title": "Host Name In Certificate", "type": "string", "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 + "order": 0 + }, + "certificate": { + "title": "Certificate", + "type": "string", + "description": "certificate of the server, or of the CA that signed the server certificate", + "order": 1, + "airbyte_secret": true, + "multiline": true } } } @@ -124,22 +132,6 @@ "const": "CDC", "order": 0 }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java index c309a81c495f..552b64136b29 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java @@ -28,9 +28,7 @@ protected JsonNode getConfig() { } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - } + protected void tearDown(final TestDestinationEnv testEnv) {} @Override protected String getImageName() { diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java index 73ccccee6c35..ad63ac5b8558 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java @@ -4,13 +4,16 @@ package io.airbyte.integrations.source.mssql; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_EVENT_SERIAL_NO; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_LSN; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_DB_HISTORY; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.ORDERED_COL_STATE_TYPE; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.STATE_TYPE_KEY; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -30,29 +33,43 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.debezium.CdcSourceTest; -import io.airbyte.cdk.integrations.debezium.internals.mssql.MssqlCdcTargetPosition; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.SyncMode; import io.debezium.connector.sqlserver.Lsn; import java.time.Duration; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.testcontainers.containers.MSSQLServerContainer; -import org.testcontainers.utility.DockerImageName; +@TestInstance(Lifecycle.PER_CLASS) public class CdcMssqlSourceTest extends CdcSourceTest { static private final String CDC_ROLE_NAME = "cdc_selector"; @@ -61,43 +78,43 @@ public class CdcMssqlSourceTest extends CdcSourceTest UNSHARED_CONTAINER = new MsSQLContainerFactory() - .createNewContainer(DockerImageName.parse("mcr.microsoft.com/mssql/server:2022-latest")); + protected final MSSQLServerContainer privateContainer; private DataSource testDataSource; - @BeforeAll - static public void beforeAll() { - new MsSQLContainerFactory().withAgent(UNSHARED_CONTAINER); - UNSHARED_CONTAINER.start(); + CdcMssqlSourceTest() { + this.privateContainer = createContainer(); + } + + protected MSSQLServerContainer createContainer() { + return new MsSQLContainerFactory().exclusive( + MsSQLTestDatabase.BaseImage.MSSQL_2022.reference, + MsSQLTestDatabase.ContainerModifier.AGENT.methodName); } @AfterAll - static void afterAll() { - UNSHARED_CONTAINER.close(); + void afterAll() { + privateContainer.close(); } - private String testUserName() { + protected final String testUserName() { return testdb.withNamespace(TEST_USER_NAME_PREFIX); } @Override protected MsSQLTestDatabase createTestDatabase() { - final var testdb = new MsSQLTestDatabase(UNSHARED_CONTAINER); + final var testdb = new MsSQLTestDatabase(privateContainer); return testdb .withConnectionProperty("encrypt", "false") .withConnectionProperty("databaseName", testdb.getDatabaseName()) .initialized() - .withSnapshotIsolation() - .withCdc() - .withWaitUntilAgentRunning(); + .withWaitUntilAgentRunning() + .withCdc(); } @Override protected MssqlSource source() { - final var source = new MssqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MssqlSource(); } @Override @@ -110,6 +127,7 @@ protected JsonNode config() { .withSchemas(modelsSchema(), randomSchema()) .withCdcReplication() .withoutSsl() + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) .build(); } @@ -127,7 +145,8 @@ protected void setup() { \t@supports_net_changes = 0"""; testdb .with(enableCdcSqlFmt, modelsSchema(), MODELS_STREAM_NAME, CDC_ROLE_NAME) - .with(enableCdcSqlFmt, randomSchema(), RANDOM_TABLE_NAME, CDC_ROLE_NAME); + .with(enableCdcSqlFmt, randomSchema(), RANDOM_TABLE_NAME, CDC_ROLE_NAME) + .withShortenedCapturePollingInterval(); // Create a test user to be used by the source, with proper permissions. testdb @@ -143,12 +162,17 @@ protected void setup() { .with("USE [%s]", testdb.getDatabaseName()) .with("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testUserName()); - testDataSource = DataSourceFactory.create( + testDataSource = createTestDataSource(); + } + + protected DataSource createTestDataSource() { + return DataSourceFactory.create( testUserName(), testdb.getPassword(), testdb.getDatabaseDriver().getDriverClassName(), testdb.getJdbcUrl(), - Map.of("encrypt", "false")); + Map.of("encrypt", "false"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); } @Override @@ -156,7 +180,7 @@ protected void setup() { protected void tearDown() { try { DataSourceFactory.close(testDataSource); - } catch (Exception e) { + } catch (final Exception e) { throw new RuntimeException(e); } super.tearDown(); @@ -173,6 +197,69 @@ public void newTableSnapshotTest() { // Do nothing } + // Utilize the setup to do test on MssqlDebeziumStateUtil. + @Test + public void testCdcSnapshot() { + MssqlDebeziumStateUtil util = new MssqlDebeziumStateUtil(); + + JdbcDatabase testDatabase = testDatabase(); + testDatabase.setSourceConfig(config()); + testDatabase.setDatabaseConfig(source().toDatabaseConfig(config())); + + JsonNode debeziumState = util.constructInitialDebeziumState(MssqlCdcHelper.getDebeziumProperties(testDatabase, getConfiguredCatalog(), true), + getConfiguredCatalog(), testDatabase); + + Assertions.assertEquals(3, Jsons.object(debeziumState, Map.class).size()); + Assertions.assertTrue(debeziumState.has("is_compressed")); + Assertions.assertFalse(debeziumState.get("is_compressed").asBoolean()); + Assertions.assertTrue(debeziumState.has("mssql_db_history")); + Assertions.assertNotNull(debeziumState.get("mssql_db_history")); + Assertions.assertTrue(debeziumState.has("mssql_cdc_offset")); + } + + // Tests even with consistent inserting operations, CDC snapshot and incremental load will not lose + // data. + @Test + public void testCdcNotLoseDataWithConsistentWriting() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(10); + + // Inserting 50 records in 10 seconds. + // Intention is to insert records while we are running the first snapshot read. And we check with + // the first snapshot read operations + // and a following incremental read operation, we will be able to capture all data. + int numberOfRecordsToInsert = 50; + var insertingProcess = executor.submit(() -> { + for (int i = 0; i < numberOfRecordsToInsert; i++) { + testdb.with("INSERT INTO %s.%s (%s, %s, %s) VALUES (%s, %s, '%s');", + modelsSchema(), MODELS_STREAM_NAME, COL_ID, COL_MAKE_ID, COL_MODEL, 910019 + i, i, "car description"); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + + final AutoCloseableIterator read1 = source() + .read(config(), getConfiguredCatalog(), null); + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + final Set recordMessages = extractRecordMessages(actualRecords1); + final List stateMessagesFromFirstSync = extractStateMessages(actualRecords1); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessagesFromFirstSync.get(stateMessagesFromFirstSync.size() - 1))); + // Make sure we have finished inserting process and read from previous state. + insertingProcess.get(); + + final AutoCloseableIterator read2 = source() + .read(config(), getConfiguredCatalog(), state); + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + + recordMessages.addAll(extractRecordMessages(actualRecords2)); + + final Set ids = recordMessages.stream().map(message -> message.getData().get("id").intValue()).collect(Collectors.toSet()); + // Originally in setup we have inserted 6 records in the table. + assertEquals(ids.size(), numberOfRecordsToInsert + 6); + } + @Override protected String columnClause(final Map columnsWithDataType, final Optional primaryKey) { final StringBuilder columnClause = new StringBuilder(); @@ -222,30 +309,6 @@ void testAssertSqlServerAgentRunning() { assertDoesNotThrow(() -> source().assertSqlServerAgentRunning(testDatabase())); } - @Test - void testAssertSnapshotIsolationAllowed() { - // snapshot isolation enabled by setup so assert check passes - assertDoesNotThrow(() -> source().assertSnapshotIsolationAllowed(config(), testDatabase())); - // now disable snapshot isolation and assert that check fails - testdb.withoutSnapshotIsolation(); - assertThrows(RuntimeException.class, () -> source().assertSnapshotIsolationAllowed(config(), testDatabase())); - } - - @Test - void testAssertSnapshotIsolationDisabled() { - final JsonNode replicationConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("data_to_sync", "New Changes Only") - // set snapshot_isolation level to "Read Committed" to disable snapshot - .put("snapshot_isolation", "Read Committed") - .build()); - final var config = config(); - Jsons.replaceNestedValue(config, List.of("replication_method"), replicationConfig); - assertDoesNotThrow(() -> source().assertSnapshotIsolationAllowed(config, testDatabase())); - testdb.withoutSnapshotIsolation(); - assertDoesNotThrow(() -> source().assertSnapshotIsolationAllowed(config, testDatabase())); - } - // Ensure the CDC check operations are included when CDC is enabled // todo: make this better by checking the returned checkOperations from source.getCheckOperations @Test @@ -267,8 +330,6 @@ void testCdcCheckOperations() throws Exception { status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); testdb.withAgentStarted().withWaitUntilAgentRunning(); - // assertSnapshotIsolationAllowed - testdb.withoutSnapshotIsolation(); status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); } @@ -369,10 +430,52 @@ protected void addCdcDefaultCursorField(final AirbyteStream stream) { @Override protected void assertExpectedStateMessages(final List stateMessages) { + assertEquals(7, stateMessages.size()); + assertStateTypes(stateMessages, 4); + } + + @Override + protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { assertEquals(1, stateMessages.size()); assertNotNull(stateMessages.get(0).getData()); - assertNotNull(stateMessages.get(0).getData().get("cdc_state").get("state").get(MSSQL_CDC_OFFSET)); - assertNotNull(stateMessages.get(0).getData().get("cdc_state").get("state").get(MSSQL_DB_HISTORY)); + for (final AirbyteStateMessage stateMessage : stateMessages) { + assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MSSQL_CDC_OFFSET)); + assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MSSQL_DB_HISTORY)); + } + } + + @Override + protected void assertExpectedStateMessagesForNoData(final List stateMessages) { + assertEquals(2, stateMessages.size()); + } + + @Override + protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { + assertEquals(27, stateAfterFirstBatch.size()); + assertStateTypes(stateAfterFirstBatch, 24); + } + + private void assertStateTypes(final List stateMessages, final int indexTillWhichExpectOcState) { + JsonNode sharedState = null; + for (int i = 0; i < stateMessages.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + if (i <= indexTillWhichExpectOcState) { + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(ORDERED_COL_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else { + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } + } } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java new file mode 100644 index 000000000000..f0f869eb686f --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.CertificateKey; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.MSSQLServerContainer; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class CdcMssqlSslSourceTest extends CdcMssqlSourceTest { + + @Override + protected MSSQLServerContainer createContainer() { + return new MsSQLContainerFactory().exclusive( + MsSQLTestDatabase.BaseImage.MSSQL_2022.reference, + MsSQLTestDatabase.ContainerModifier.AGENT.methodName, + MsSQLTestDatabase.ContainerModifier.WITH_SSL_CERTIFICATES.methodName); + } + + @Override + final protected MsSQLTestDatabase createTestDatabase() { + final var testdb = new MsSQLTestDatabase(privateContainer); + return testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .withConnectionProperty("trustServerCertificate", "true") + .initialized() + .withWaitUntilAgentRunning() + .withCdc(); + } + + @Override + protected DataSource createTestDataSource() { + return DataSourceFactory.create( + testUserName(), + testdb.getPassword(), + testdb.getDatabaseDriver().getDriverClassName(), + testdb.getJdbcUrl(), + Map.of("encrypt", "true", "databaseName", testdb.getDatabaseName(), "trustServerCertificate", "true"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); + } + + @Override + protected JsonNode config() { + final String containerIp; + try { + containerIp = InetAddress.getByName(testdb.getContainer().getHost()) + .getHostAddress(); + } catch (final UnknownHostException e) { + throw new RuntimeException(e); + } + final String certificate = testdb.getCertificate(CertificateKey.SERVER); + return testdb.configBuilder() + .withEncrytedVerifyServerCertificate(certificate, testdb.getContainer().getHost()) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testdb.getContainer().getFirstMappedPort()) + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testUserName()) + .with(JdbcUtils.PASSWORD_KEY, testdb.getPassword()) + .withSchemas(modelsSchema(), randomSchema()) + .withCdcReplication() + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java index 99bf15c74f92..293189b6683a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java @@ -16,8 +16,6 @@ import com.google.common.collect.Lists; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.protocol.models.Field; @@ -42,7 +40,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MSSQLServerContainer; public class CdcStateCompressionTest { @@ -56,21 +53,13 @@ public class CdcStateCompressionTest { static private final int ADDED_COLUMNS = 1000; - static private final MSSQLServerContainer CONTAINER = new MsSQLContainerFactory().shared( - "mcr.microsoft.com/mssql/server:2022-latest", "withAgent"); - private MsSQLTestDatabase testdb; @BeforeEach public void setup() { - testdb = new MsSQLTestDatabase(CONTAINER); - testdb = testdb - .withConnectionProperty("encrypt", "false") - .withConnectionProperty("databaseName", testdb.getDatabaseName()) - .initialized() - .withSnapshotIsolation() - .withCdc() - .withWaitUntilAgentRunning(); + testdb = MsSQLTestDatabase.in(MsSQLTestDatabase.BaseImage.MSSQL_2022, MsSQLTestDatabase.ContainerModifier.AGENT) + .withWaitUntilAgentRunning() + .withCdc(); // Create a test schema and a bunch of test tables with CDC enabled. // Insert one row in each table so that they're not empty. @@ -87,6 +76,7 @@ public void setup() { testdb .with("CREATE TABLE %s.test_table_%d (id INT IDENTITY(1,1) PRIMARY KEY);", TEST_SCHEMA, i) .with(enableCdcSqlFmt, TEST_SCHEMA, i, CDC_ROLE_NAME, i, 1) + .withShortenedCapturePollingInterval() .with("INSERT INTO %s.test_table_%d DEFAULT VALUES", TEST_SCHEMA, i); } @@ -124,12 +114,13 @@ public void setup() { testdb .with(sb.toString()) .with(enableCdcSqlFmt, TEST_SCHEMA, i, CDC_ROLE_NAME, i, 2) - .with(disableCdcSqlFmt, TEST_SCHEMA, i, i, 1); + .with(disableCdcSqlFmt, TEST_SCHEMA, i, i, 1) + .withShortenedCapturePollingInterval(); } } private AirbyteCatalog getCatalog() { - var streams = new ArrayList(); + final var streams = new ArrayList(); for (int i = 0; i < TEST_TABLES; i++) { streams.add(CatalogHelpers.createAirbyteStream( "test_table_%d".formatted(i), @@ -148,9 +139,7 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() { } private MssqlSource source() { - final var source = new MssqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MssqlSource(); } private JsonNode config() { @@ -160,8 +149,13 @@ private JsonNode config() { .with(JdbcUtils.USERNAME_KEY, testUserName()) .with(JdbcUtils.PASSWORD_KEY, testdb.getPassword()) .withSchemas(TEST_SCHEMA) - .withCdcReplication() .withoutSsl() + // Configure for CDC replication but with a higher timeout than usual. + // This is because Debezium requires more time than usual to build the initial snapshot. + .with("is_test", true) + .with("replication_method", Map.of( + "method", "CDC", + "initial_waiting_seconds", 60)) .build(); } @@ -188,7 +182,7 @@ public void testCompressedSchemaHistory() throws Exception { assertTrue(lastSharedStateFromFirstBatch.get(IS_COMPRESSED).asBoolean()); final var recordsFromFirstBatch = extractRecordMessages(dataFromFirstBatch); assertEquals(TEST_TABLES, recordsFromFirstBatch.size()); - for (var record : recordsFromFirstBatch) { + for (final var record : recordsFromFirstBatch) { assertEquals("1", record.getData().get("id").toString()); } @@ -213,7 +207,7 @@ public void testCompressedSchemaHistory() throws Exception { assertTrue(lastSharedStateFromSecondBatch.get(IS_COMPRESSED).asBoolean()); final var recordsFromSecondBatch = extractRecordMessages(dataFromSecondBatch); assertEquals(TEST_TABLES, recordsFromSecondBatch.size()); - for (var record : recordsFromSecondBatch) { + for (final var record : recordsFromSecondBatch) { assertEquals("2", record.getData().get("id").toString()); } } @@ -235,7 +229,7 @@ private Map> extractRecordMessagesStreamWise(f .collect(Collectors.groupingBy(AirbyteRecordMessage::getStream)); final Map> recordsPerStreamWithNoDuplicates = new HashMap<>(); - for (var entry : recordsPerStream.entrySet()) { + for (final var entry : recordsPerStream.entrySet()) { final var set = new HashSet<>(entry.getValue()); recordsPerStreamWithNoDuplicates.put(entry.getKey(), set); assertEquals(entry.getValue().size(), set.size(), "duplicate records in sync for " + entry.getKey()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java index 97508ceae381..1889315bea6f 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java @@ -15,7 +15,6 @@ import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; @@ -61,7 +60,7 @@ void testStrictSSLUnsecuredNoTunnel() throws Exception { void testStrictSSLSecuredNoTunnel() throws Exception { try (final var testdb = createTestDatabase()) { final var config = testdb.testConfigBuilder() - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .withEncrytedTrustServerCertificate() .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) .build(); final AirbyteConnectionStatus actual = source().check(config); @@ -77,7 +76,7 @@ void testStrictSSLSecuredWithTunnel() throws Exception { .withDatabase() .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) .with(JdbcUtils.PASSWORD_KEY, "fake") - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .withEncrytedTrustServerCertificate() .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) .build(); final AirbyteConnectionStatus actual = source().check(config); @@ -94,7 +93,7 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception { .withDatabase() .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) .with(JdbcUtils.PASSWORD_KEY, "fake") - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .withEncrytedTrustServerCertificate() .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) .build(); final AirbyteConnectionStatus actual = source().check(config); diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java index a2f29d5064a7..d1ec53fe1915 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java @@ -4,14 +4,11 @@ package io.airbyte.integrations.source.mssql; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.DataToSync; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; import java.util.Map; import org.junit.jupiter.api.Test; @@ -33,9 +30,7 @@ public void testIsCdc() { final JsonNode newCdc = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); + "method", "CDC")))); assertTrue(MssqlCdcHelper.isCdc(newCdc)); // migration from legacy to new config @@ -46,90 +41,10 @@ public void testIsCdc() { final JsonNode mixCdc = Jsons.jsonNode(Map.of( "replication", Jsons.jsonNode(Map.of( - "replication_type", "Standard", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")), + "replication_type", "Standard")), "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); + "method", "CDC")))); assertTrue(MssqlCdcHelper.isCdc(mixCdc)); } - @Test - public void testGetSnapshotIsolation() { - // legacy replication method config before version 0.4.0 - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(LEGACY_CDC_CONFIG)); - - // new replication method config since version 0.4.0 - final JsonNode newCdcNonSnapshot = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcNonSnapshot)); - - final JsonNode newCdcSnapshot = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcSnapshot)); - - // migration from legacy to new config - final JsonNode mixCdcNonSnapshot = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcNonSnapshot)); - - final JsonNode mixCdcSnapshot = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcSnapshot)); - } - - @Test - public void testGetDataToSyncConfig() { - // legacy replication method config before version 0.4.0 - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(LEGACY_CDC_CONFIG)); - - // new replication method config since version 0.4.0 - final JsonNode newCdcExistingAndNew = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(newCdcExistingAndNew)); - - final JsonNode newCdcNewOnly = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "New Changes Only", - "snapshot_isolation", "Snapshot")))); - assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(newCdcNewOnly)); - - final JsonNode mixCdcExistingAndNew = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(mixCdcExistingAndNew)); - - final JsonNode mixCdcNewOnly = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "New Changes Only", - "snapshot_isolation", "Snapshot")))); - assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(mixCdcNewOnly)); - } - } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java new file mode 100644 index 000000000000..64d2fae404d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; + +public class MssqlDataSourceFactoryTest { + + @Test + protected void testCreatingDataSourceWithConnectionTimeoutSetBelowDefault() { + try (var testdb = MsSQLTestDatabase.in(MsSQLTestDatabase.BaseImage.MSSQL_2022)) { + final Map connectionProperties = Map.of("loginTimeout", String.valueOf(5)); + final DataSource dataSource = DataSourceFactory.create( + testdb.getUserName(), + testdb.getPassword(), + testdb.getDatabaseDriver().getDriverClassName(), + testdb.getJdbcUrl(), + connectionProperties, + new MssqlSource().getConnectionTimeoutMssql(connectionProperties)); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDebeziumStateUtilTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDebeziumStateUtilTest.java new file mode 100644 index 000000000000..ceddd2b9268d --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDebeziumStateUtilTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil; +import io.airbyte.integrations.source.mssql.cdc.MssqlDebeziumStateUtil.MssqlDebeziumStateAttributes; +import io.debezium.connector.sqlserver.Lsn; +import org.junit.jupiter.api.Test; + +public class MssqlDebeziumStateUtilTest { + + private static String DB_NAME = "db_name"; + private static String LSN_STRING = "0000062d:00017ff0:016d"; + private static Lsn LSN = Lsn.valueOf(LSN_STRING); + + @Test + void generateCorrectFormat() { + MssqlDebeziumStateUtil util = new MssqlDebeziumStateUtil(); + MssqlDebeziumStateAttributes attributes = new MssqlDebeziumStateAttributes(LSN); + JsonNode formatResult = util.format(attributes, DB_NAME); + assertEquals("{\"commit_lsn\":\"0000062d:00017ff0:016d\",\"snapshot\":true,\"snapshot_completed\":true}", + formatResult.get("[\"db_name\",{\"server\":\"db_name\",\"database\":\"db_name\"}]").asText()); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlInitialLoadHandlerTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlInitialLoadHandlerTest.java new file mode 100644 index 000000000000..9693bb6cd563 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlInitialLoadHandlerTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.integrations.source.mssql.MssqlQueryUtils.TableSizeInfo; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadHandler; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import org.junit.jupiter.api.Test; + +public class MssqlInitialLoadHandlerTest { + + private static final long ONE_GB = 1_073_741_824; + private static final long ONE_MB = 1_048_576; + + @Test + void testInvalidOrNullTableSizeInfo() { + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair("table_name", "schema_name"); + assertEquals(MssqlInitialLoadHandler.calculateChunkSize(null, pair), 1_000_000L); + final TableSizeInfo invalidRowLengthInfo = new TableSizeInfo(ONE_GB, 0L); + assertEquals(MssqlInitialLoadHandler.calculateChunkSize(invalidRowLengthInfo, pair), 1_000_000L); + final TableSizeInfo invalidTableSizeInfo = new TableSizeInfo(0L, 0L); + assertEquals(MssqlInitialLoadHandler.calculateChunkSize(invalidTableSizeInfo, pair), 1_000_000L); + } + + @Test + void testTableSizeInfo() { + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair("table_name", "schema_name"); + assertEquals(MssqlInitialLoadHandler.calculateChunkSize(new TableSizeInfo(ONE_GB, 2 * ONE_MB), pair), 512L); + assertEquals(MssqlInitialLoadHandler.calculateChunkSize(new TableSizeInfo(ONE_GB, 200L), pair), 5368709L); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index 167843841fef..605737a09c36 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -4,24 +4,50 @@ package io.airbyte.integrations.source.mssql; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager.STATE_TYPE_KEY; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.cdk.integrations.source.relationaldb.models.CursorBasedStatus; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.cdk.integrations.source.relationaldb.models.InternalModels.StateType; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -45,9 +71,7 @@ protected JsonNode config() { @Override protected MssqlSource source() { - final MssqlSource source = new MssqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MssqlSource(); } @Override @@ -127,6 +151,229 @@ public void testUserHasNoPermissionToDataBase() throws Exception { assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;"), status.getMessage()); } + @Test + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { + final var config = config(); + ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1); + final String streamOneName = TABLE_NAME + "one"; + // Create a fresh first table + testdb.with("CREATE TABLE %s (\n" + + " id INT NOT NULL PRIMARY KEY,\n" + + " name VARCHAR(50) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (1, 'picard', '2004-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", + getFullyQualifiedTableName(streamOneName)); + + // Create a fresh second table + final String streamTwoName = TABLE_NAME + "two"; + final String streamTwoFullyQualifiedName = getFullyQualifiedTableName(streamTwoName); + // Insert records into second table + testdb.with("CREATE TABLE %s (\n" + + " id INT NOT NULL PRIMARY KEY,\n" + + " name VARCHAR(50) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoFullyQualifiedName) + .with("INSERT INTO %s (id, name, updated_at) VALUES (40, 'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName) + .with("INSERT INTO %s (id, name, updated_at) VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName) + .with("INSERT INTO %s (id, name, updated_at) VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName); + + final List streamOneExpectedRecords = Arrays.asList( + createRecord(streamOneName, getDefaultNamespace(), Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")), + createRecord(streamOneName, getDefaultNamespace(), Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")), + createRecord(streamOneName, getDefaultNamespace(), Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))); + + // Create records list that we expect to see in the state message + final List streamTwoExpectedRecords = Arrays.asList( + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 40, + COL_NAME, "Jean Luc", + COL_UPDATED_AT, "2006-10-19")), + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 41, + COL_NAME, "Groot", + COL_UPDATED_AT, "2006-10-19")), + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 42, + COL_NAME, "Thanos", + COL_UPDATED_AT, "2006-10-19"))); + + // Prep and create a configured catalog to perform sync + final AirbyteStream streamOne = getAirbyteStream(streamOneName, getDefaultNamespace()); + final AirbyteStream streamTwo = getAirbyteStream(streamTwoName, getDefaultNamespace()); + + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( + new AirbyteCatalog().withStreams(List.of(streamOne, streamTwo))); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + airbyteStream.withPrimaryKey(List.of(List.of(COL_ID))); + }); + + // Perform initial sync + final List messagesFromFirstSync = MoreIterators + .toList(source().read(config, configuredCatalog, null)); + + final List recordsFromFirstSync = filterRecords(messagesFromFirstSync); + + setEmittedAtToNull(messagesFromFirstSync); + // All records in the 2 configured streams should be present + assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( + Stream.concat(streamOneExpectedRecords.stream().parallel(), + streamTwoExpectedRecords.stream().parallel()).collect(toList())); + + final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); + // Since we are emitting a state message after each record, we should have 1 state for each record - + // 3 from stream1 and 3 from stream2 + assertEquals(6, actualFirstSyncState.size()); + + // The expected state type should be 2 ordered_column's and the last one being cursor_based + final List expectedStateTypesFromFirstSync = List.of("ordered_column", "ordered_column", "cursor_based"); + final List stateTypeOfStreamOneStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, STATE_TYPE_KEY); + final List stateTypeOfStreamTwoStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamTwoName, STATE_TYPE_KEY); + // It should be the same for stream1 and stream2 + assertEquals(stateTypeOfStreamOneStatesFromFirstSync, expectedStateTypesFromFirstSync); + assertEquals(stateTypeOfStreamTwoStatesFromFirstSync, expectedStateTypesFromFirstSync); + + // Create the expected ordered_column values that we should see + final List expectedOrderedColumnValueFromFirstSync = List.of("1", "2"); + final List orderedColumnValuesOfStreamOneFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "ordered_col_val"); + final List orderedColumnValuesOfStreamTwoFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "ordered_col_val"); + + // Verifying each element and its index to match. + // Only checking the first 2 elements since we have verified that the last state_type is + // "cursor_based" + assertEquals(expectedOrderedColumnValueFromFirstSync.get(0), orderedColumnValuesOfStreamOneFromFirstSync.get(0)); + assertEquals(expectedOrderedColumnValueFromFirstSync.get(1), orderedColumnValuesOfStreamOneFromFirstSync.get(1)); + assertEquals(expectedOrderedColumnValueFromFirstSync.get(0), orderedColumnValuesOfStreamTwoFromFirstSync.get(0)); + assertEquals(expectedOrderedColumnValueFromFirstSync.get(1), orderedColumnValuesOfStreamTwoFromFirstSync.get(1)); + + // Extract only state messages for each stream + final List streamOneStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamOneName); + final List streamTwoStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamTwoName); + // Extract the incremental states of each stream's first and second state message + final List streamOneIncrementalStatesFromFirstSync = + List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + final List streamTwoIncrementalStatesFromFirstSync = + List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + // The incremental_state of each stream's first and second incremental states is expected + // to be identical to the stream_state of the final state message for each stream + assertEquals(streamOneIncrementalStatesFromFirstSync.get(0), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamOneIncrementalStatesFromFirstSync.get(1), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(0), streamTwoFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(1), streamTwoFinalStreamStateFromFirstSync); + + // Sync should work with a ordered_column state AND a cursor-based state from each stream + // Forcing a sync with + // - stream one state still being the first record read via Ordered column. + // - stream two state being the Ordered Column state before the final emitted state before the + // cursor + // switch + final List messagesFromSecondSyncWithMixedStates = MoreIterators + .toList(source().read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); + + // Extract only state messages for each stream after second sync + final List streamOneStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamOneName); + final List stateTypeOfStreamOneStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamOneName, STATE_TYPE_KEY); + + final List streamTwoStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamTwoName); + final List stateTypeOfStreamTwoStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamTwoName, STATE_TYPE_KEY); + + // Stream One states after the second sync are expected to have 2 stream states + // - 1 with PrimaryKey state_type and 1 state that is of cursorBased state type + assertEquals(2, streamOneStateMessagesFromSecondSync.size()); + assertEquals(List.of("ordered_column", "cursor_based"), stateTypeOfStreamOneStatesFromSecondSync); + + // Stream Two states after the second sync are expected to have 1 stream state + // - The state that is of cursorBased state type + assertEquals(1, streamTwoStateMessagesFromSecondSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromSecondSync); + + // Add some data to each table and perform a third read. + // Expect to see all records be synced via cursorBased method and not ordered_column + testdb.with("INSERT INTO %s (id, name, updated_at) VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s (id, name, updated_at) VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName); + + final List messagesFromThirdSync = MoreIterators + .toList(source().read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); + + // Extract only state messages, state type, and cursor for each stream after second sync + final List streamOneStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamOneName); + final List stateTypeOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, STATE_TYPE_KEY); + final List cursorOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, "cursor"); + + final List streamTwoStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamTwoName); + final List stateTypeOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, STATE_TYPE_KEY); + final List cursorOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, "cursor"); + + // Both streams should now be synced via standard cursor and have updated max cursor values + // cursor: 4 for stream one + // cursor: 43 for stream two + assertEquals(1, streamOneStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamOneStatesFromThirdSync); + assertEquals(List.of("4"), cursorOfStreamOneStatesFromThirdSync); + + assertEquals(1, streamTwoStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromThirdSync); + assertEquals(List.of("43"), cursorOfStreamTwoStatesFromThirdSync); + } + + private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { + return CatalogHelpers.createAirbyteStream( + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); + } + @Override protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( @@ -158,8 +405,64 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { } @Override - protected boolean supportsPerStream() { - return true; + protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, + final String cursorField, + final String cursorValue) { + return new CursorBasedStatus().withStateType(StateType.CURSOR_BASED).withVersion(2L) + .withStreamName(configuredAirbyteStream.getStream().getName()) + .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(cursorValue) + .withCursorRecordCount(1L); + } + + // Override from parent class as we're no longer including the legacy Data field. + @Override + protected List createExpectedTestMessages(final List states, final long numRecords) { + return states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) numRecords)))) + .collect( + Collectors.toList()); + } + + @Override + protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { + final JsonNode streamState = airbyteMessage.getState().getStream().getStreamState(); + if (streamState.get("stream_name").asText().equals(streamName)) { + return streamState; + } + + throw new IllegalArgumentException("Stream not found in state message: " + streamName); + } + + @Override + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { + final List expectedMessages = new ArrayList<>(); + expectedMessages.addAll(List.of(createRecord(streamName(), namespace, ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19")), + createRecord(streamName(), namespace, ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19")))); + final DbStreamState state = new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withVersion(2L) + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L); + + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2L)); + return expectedMessages; } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java index 1819109ee2a4..098ebab0ef41 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java @@ -11,8 +11,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; @@ -42,9 +40,7 @@ class MssqlSourceTest { private MsSQLTestDatabase testdb; private MssqlSource source() { - final MssqlSource source = new MssqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MssqlSource(); } // how to interact with the mssql test container manaully. diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java new file mode 100644 index 000000000000..3b45cb7e8210 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.CertificateKey; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import java.net.InetAddress; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlSslSourceTest { + + private MsSQLTestDatabase testDb; + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlSslSourceTest.class); + + @BeforeEach + void setup() { + testDb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022, ContainerModifier.WITH_SSL_CERTIFICATES); + } + + @AfterEach + public void tearDown() { + testDb.close(); + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateTrustHostnameWithValidCertificates(CertificateKey certificateKey) throws Exception { + if (!certificateKey.isValid) { + return; + } + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.testConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .build(); + AirbyteCatalog catalog = new MssqlSource().discover(config); + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateTrustHostnameWithInvalidCertificates(CertificateKey certificateKey) throws Exception { + if (certificateKey.isValid) { + return; + } + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.testConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .build(); + try { + AirbyteCatalog catalog = new MssqlSource().discover(config); + } catch (ConnectionErrorException e) { + if (!e.getCause().getCause().getMessage().contains("PKIX path validation") && + !e.getCause().getCause().getMessage().contains("PKIX path building failed")) { + throw e; + } + } + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateNoTrustHostnameWrongHostname(CertificateKey certificateKey) throws Throwable { + if (!certificateKey.isValid) { + return; + } + String containerIp = InetAddress.getByName(testDb.getContainer().getHost()).getHostAddress(); + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.configBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testDb.getContainer().getFirstMappedPort()) + .withCredentials() + .withDatabase() + .build(); + try { + AirbyteCatalog catalog = new MssqlSource().discover(config); + fail("discover should have failed!"); + } catch (ConnectionErrorException e) { + String expectedMessage = + "Failed to validate the server name \"" + containerIp + "\"in a certificate during Secure Sockets Layer (SSL) initialization."; + if (!e.getExceptionMessage().contains(expectedMessage)) { + fail("exception message was " + e.getExceptionMessage() + "\n expected: " + expectedMessage); + } + } + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateNoTrustHostnameAlternateHostname(CertificateKey certificateKey) throws Exception { + final String containerIp = InetAddress.getByName(testDb.getContainer().getHost()).getHostAddress(); + if (certificateKey.isValid) { + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.configBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate, + "hostNameInCertificate", testDb.getContainer().getHost())) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testDb.getContainer().getFirstMappedPort()) + .withCredentials() + .withDatabase() + .build(); + AirbyteCatalog catalog = new MssqlSource().discover(config); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java index 83e6662bcf92..054b8c3d6f64 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java @@ -5,79 +5,22 @@ package io.airbyte.integrations.source.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.factory.DataSourceFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import java.sql.JDBCType; -import java.util.Map; import java.util.Optional; -import javax.sql.DataSource; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; -import org.testcontainers.containers.MSSQLServerContainer; @Disabled public class MssqlStressTest extends JdbcStressTest { - private static MSSQLServerContainer dbContainer; - private JsonNode config; - - @BeforeAll - static void init() { - dbContainer = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); - dbContainer.start(); - } + private MsSQLTestDatabase testdb; @BeforeEach public void setup() throws Exception { - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) - .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, dbContainer.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, dbContainer.getPassword()) - .build()); - - final DataSource dataSource = DataSourceFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")); - - try { - final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); - - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - database.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("is_test", true); - - super.setup(); - } finally { - DataSourceFactory.close(dataSource); - } - } - - @AfterAll - public static void tearDown() { - dbContainer.close(); + testdb = MsSQLTestDatabase.in(MsSQLTestDatabase.BaseImage.MSSQL_2022); + super.setup(); } @Override @@ -87,14 +30,12 @@ public Optional getDefaultSchemaName() { @Override public JsonNode getConfig() { - return Jsons.clone(config); + return testdb.testConfigBuilder().with("is_test", true).build(); } @Override public AbstractJdbcSource getSource() { - final MssqlSource source = new MssqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MssqlSource(); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java index c28cf5a96dd2..22bd3b2bbf6a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java +++ b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java @@ -5,25 +5,21 @@ package io.airbyte.integrations.source.mssql; import io.airbyte.cdk.testutils.ContainerFactory; +import org.apache.commons.lang3.StringUtils; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.Network; import org.testcontainers.utility.DockerImageName; -public class MsSQLContainerFactory implements ContainerFactory> { +public class MsSQLContainerFactory extends ContainerFactory> { @Override - public MSSQLServerContainer createNewContainer(DockerImageName imageName) { - MSSQLServerContainer container = - new MSSQLServerContainer<>(imageName.asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server")).acceptLicense(); + protected MSSQLServerContainer createNewContainer(DockerImageName imageName) { + imageName = imageName.asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server"); + var container = new MSSQLServerContainer<>(imageName).acceptLicense(); container.addEnv("MSSQL_MEMORY_LIMIT_MB", "384"); return container; } - @Override - public Class getContainerClass() { - return MSSQLServerContainer.class; - } - /** * Create a new network and bind it to the container. */ @@ -35,4 +31,35 @@ public void withAgent(MSSQLServerContainer container) { container.addEnv("MSSQL_AGENT_ENABLED", "True"); } + public void withSslCertificates(MSSQLServerContainer container) { + // yes, this is uglier than sin. The reason why I'm doing this is because there's no command to + // reload a SqlServer config. So I need to create all the necessary files before I start the + // SQL server. Hence this horror + String command = StringUtils.replace( + """ + mkdir /tmp/certs/ && + openssl req -nodes -new -x509 -sha256 -keyout /tmp/certs/ca.key -out /tmp/certs/ca.crt -subj "/CN=ca" && + openssl req -nodes -new -x509 -sha256 -keyout /tmp/certs/dummy_ca.key -out /tmp/certs/dummy_ca.crt -subj "/CN=ca" && + openssl req -nodes -new -sha256 -keyout /tmp/certs/server.key -out /tmp/certs/server.csr -subj "/CN={hostName}" && + openssl req -nodes -new -sha256 -keyout /tmp/certs/dummy_server.key -out /tmp/certs/dummy_server.csr -subj "/CN={hostName}" && + + openssl x509 -req -in /tmp/certs/server.csr -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key -out /tmp/certs/server.crt -days 365 -sha256 && + openssl x509 -req -in /tmp/certs/dummy_server.csr -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key -out /tmp/certs/dummy_server.crt -days 365 -sha256 && + openssl x509 -req -in /tmp/certs/server.csr -CA /tmp/certs/dummy_ca.crt -CAkey /tmp/certs/dummy_ca.key -out /tmp/certs/server_dummy_ca.crt -days 365 -sha256 && + chmod 440 /tmp/certs/* && + { + cat > /var/opt/mssql/mssql.conf <<- EOF + [network] + tlscert = /tmp/certs/server.crt + tlskey = /tmp/certs/server.key + tlsprotocols = 1.2 + forceencryption = 1 + EOF + } && /opt/mssql/bin/sqlservr + """, + "{hostName}", container.getHost()); + container.withCommand("bash", "-c", command) + .withUrlParam("trustServerCertificate", "true"); + } + } diff --git a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java index 873f42cd40a8..a9deadded8e1 100644 --- a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java +++ b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java @@ -8,7 +8,10 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.testutils.TestDatabase; import io.debezium.connector.sqlserver.Lsn; +import java.io.IOException; +import java.io.UncheckedIOException; import java.sql.SQLException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -24,35 +27,37 @@ public class MsSQLTestDatabase extends TestDatabase, MsS static public final int MAX_RETRIES = 60; - public static enum BaseImage { + public enum BaseImage { MSSQL_2022("mcr.microsoft.com/mssql/server:2022-latest"), MSSQL_2017("mcr.microsoft.com/mssql/server:2017-latest"), ; - private final String reference; + public final String reference; - private BaseImage(String reference) { + BaseImage(final String reference) { this.reference = reference; } } - public static enum ContainerModifier { + public enum ContainerModifier { NETWORK("withNetwork"), - AGENT("withAgent"); + AGENT("withAgent"), + WITH_SSL_CERTIFICATES("withSslCertificates"), + ; - private final String methodName; + public final String methodName; - private ContainerModifier(String methodName) { + ContainerModifier(final String methodName) { this.methodName = methodName; } } - static public MsSQLTestDatabase in(BaseImage imageName, ContainerModifier... methods) { - String[] methodNames = Stream.of(methods).map(im -> im.methodName).toList().toArray(new String[0]); + static public MsSQLTestDatabase in(final BaseImage imageName, final ContainerModifier... methods) { + final String[] methodNames = Stream.of(methods).map(im -> im.methodName).toList().toArray(new String[0]); final var container = new MsSQLContainerFactory().shared(imageName.reference, methodNames); final var testdb = new MsSQLTestDatabase(container); return testdb @@ -61,18 +66,10 @@ static public MsSQLTestDatabase in(BaseImage imageName, ContainerModifier... met .initialized(); } - public MsSQLTestDatabase(MSSQLServerContainer container) { + public MsSQLTestDatabase(final MSSQLServerContainer container) { super(container); } - public MsSQLTestDatabase withSnapshotIsolation() { - return with("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION ON;", getDatabaseName()); - } - - public MsSQLTestDatabase withoutSnapshotIsolation() { - return with("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION OFF;", getDatabaseName()); - } - public MsSQLTestDatabase withCdc() { return with("EXEC sys.sp_cdc_enable_db;"); } @@ -99,6 +96,11 @@ public MsSQLTestDatabase withWaitUntilAgentStopped() { return self(); } + public MsSQLTestDatabase withShortenedCapturePollingInterval() { + return with("EXEC sys.sp_cdc_change_job @job_type = 'capture', @pollinginterval = %d;", + MssqlCdcTargetPosition.MAX_LSN_QUERY_DELAY_TEST.toSeconds()); + } + private void waitForAgentState(final boolean running) { final String expectedValue = running ? "Running." : "Stopped."; LOGGER.debug("Waiting for SQLServerAgent state to change to '{}'.", expectedValue); @@ -110,12 +112,12 @@ private void waitForAgentState(final boolean running) { return; } LOGGER.debug("Retrying, SQLServerAgent state {} does not match expected '{}'.", r, expectedValue); - } catch (SQLException e) { + } catch (final SQLException e) { LOGGER.debug("Retrying agent state query after catching exception {}.", e.getMessage()); } try { Thread.sleep(1_000); // Wait one second between retries. - } catch (InterruptedException e) { + } catch (final InterruptedException e) { throw new RuntimeException(e); } } @@ -132,12 +134,12 @@ public MsSQLTestDatabase withWaitUntilMaxLsnAvailable() { return self(); } LOGGER.debug("Retrying, max LSN still not available for database {}.", getDatabaseName()); - } catch (SQLException e) { + } catch (final SQLException e) { LOGGER.warn("Retrying max LSN query after catching exception {}", e.getMessage()); } try { Thread.sleep(1_000); // Wait one second between retries. - } catch (InterruptedException e) { + } catch (final InterruptedException e) { throw new RuntimeException(e); } } @@ -183,7 +185,7 @@ public void dropDatabaseAndUser() { String.format("DROP DATABASE %s", getDatabaseName())))); } - public Stream mssqlCmd(Stream sql) { + public Stream mssqlCmd(final Stream sql) { return Stream.of("/opt/mssql-tools/bin/sqlcmd", "-U", getContainer().getUsername(), "-P", getContainer().getPassword(), @@ -201,6 +203,44 @@ public SQLDialect getSqlDialect() { return SQLDialect.DEFAULT; } + public static enum CertificateKey { + + CA(true), + DUMMY_CA(false), + SERVER(true), + DUMMY_SERVER(false), + SERVER_DUMMY_CA(false), + ; + + public final boolean isValid; + + CertificateKey(final boolean isValid) { + this.isValid = isValid; + } + + } + + private Map cachedCerts; + + public synchronized String getCertificate(final CertificateKey certificateKey) { + if (cachedCerts == null) { + final Map cachedCerts = new HashMap<>(); + try { + for (final CertificateKey key : CertificateKey.values()) { + final String command = "cat /tmp/certs/" + key.name().toLowerCase() + ".crt"; + final String certificate = getContainer().execInContainer("bash", "-c", command).getStdout().trim(); + cachedCerts.put(key, certificate); + } + } catch (final IOException e) { + throw new UncheckedIOException(e); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + this.cachedCerts = cachedCerts; + } + return cachedCerts.get(certificateKey); + } + @Override public MsSQLConfigBuilder configBuilder() { return new MsSQLConfigBuilder(this); @@ -208,19 +248,21 @@ public MsSQLConfigBuilder configBuilder() { static public class MsSQLConfigBuilder extends ConfigBuilder { - protected MsSQLConfigBuilder(MsSQLTestDatabase testDatabase) { + protected MsSQLConfigBuilder(final MsSQLTestDatabase testDatabase) { + super(testDatabase); + with(JdbcUtils.JDBC_URL_PARAMS_KEY, "loginTimeout=2"); + } public MsSQLConfigBuilder withCdcReplication() { - return with("replication_method", Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds(), - "snapshot_isolation", "Snapshot")); + return with("is_test", true) + .with("replication_method", Map.of( + "method", "CDC", + "initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds())); } - public MsSQLConfigBuilder withSchemas(String... schemas) { + public MsSQLConfigBuilder withSchemas(final String... schemas) { return with(JdbcUtils.SCHEMAS_KEY, List.of(schemas)); } @@ -229,11 +271,26 @@ public MsSQLConfigBuilder withoutSsl() { return withSsl(Map.of("ssl_method", "unencrypted")); } - @Override - public MsSQLConfigBuilder withSsl(Map sslMode) { + @Deprecated + public MsSQLConfigBuilder withSsl(final Map sslMode) { return with("ssl_method", sslMode); } + public MsSQLConfigBuilder withEncrytedTrustServerCertificate() { + return withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")); + } + + public MsSQLConfigBuilder withEncrytedVerifyServerCertificate(final String certificate, final String hostnameInCertificate) { + if (hostnameInCertificate != null) { + return withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate, + "hostNameInCertificate", hostnameInCertificate)); + } else { + return withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)); + } + } + } } diff --git a/airbyte-integrations/connectors/source-my-hours/main.py b/airbyte-integrations/connectors/source-my-hours/main.py index 050526fca166..d4e838cb4810 100644 --- a/airbyte-integrations/connectors/source-my-hours/main.py +++ b/airbyte-integrations/connectors/source-my-hours/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_my_hours import SourceMyHours +from source_my_hours.run import run if __name__ == "__main__": - source = SourceMyHours() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-my-hours/metadata.yaml b/airbyte-integrations/connectors/source-my-hours/metadata.yaml index ab7bf1e3e5be..27da449d698a 100644 --- a/airbyte-integrations/connectors/source-my-hours/metadata.yaml +++ b/airbyte-integrations/connectors/source-my-hours/metadata.yaml @@ -8,6 +8,10 @@ data: icon: my-hours.svg license: MIT name: My Hours + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-my-hours registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-my-hours/setup.py b/airbyte-integrations/connectors/source-my-hours/setup.py index eb4d9a7c9734..0f1e5c67af1a 100644 --- a/airbyte-integrations/connectors/source-my-hours/setup.py +++ b/airbyte-integrations/connectors/source-my-hours/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-my-hours=source_my_hours.run:run", + ], + }, name="source_my_hours", description="Source implementation for My Hours.", author="Wisse Jelgersma", author_email="wisse@vrowl.nl", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py b/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py new file mode 100644 index 000000000000..adad265e0bac --- /dev/null +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_my_hours import SourceMyHours + + +def run(): + source = SourceMyHours() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 403e87a16f4c..c0f1e6ef5114 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -1,43 +1,36 @@ import org.jsonschema2pojo.SourceType plugins { - id 'application' id 'airbyte-java-connector' id 'org.jsonschema2pojo' version '1.2.1' } airbyteJavaConnector { - cdkVersionRequired = '0.7.1' + cdkVersionRequired = '0.23.1' features = ['db-sources'] useLocalCdk = false } -configurations.all { - resolutionStrategy { - force libs.jooq +java { + compileJava { + options.compilerArgs += "-Xlint:-try,-rawtypes" } } - - application { mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation libs.jooq - implementation 'mysql:mysql-connector-java:8.0.30' - implementation 'org.apache.commons:commons-lang3:3.11' + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'io.debezium:debezium-connector-mysql:2.4.0.Final' - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation libs.junit.jupiter.system.stubs - testImplementation libs.testcontainers.mysql - testFixturesImplementation libs.testcontainers.mysql - performanceTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql') + testFixturesImplementation 'org.testcontainers:mysql:1.19.0' + testImplementation 'org.hamcrest:hamcrest-all:1.3' + testImplementation 'org.testcontainers:mysql:1.19.0' } jsonSchema2Pojo { diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index 7d464a64ea8a..f80f0c555620 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.2.4 + dockerImageTag: 3.3.11 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql @@ -30,5 +30,4 @@ data: supportLevel: certified tags: - language:java - - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index ac57eb771b80..d6117aec6e5b 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -6,8 +6,8 @@ import static io.airbyte.cdk.db.jdbc.JdbcUtils.EQUALS; import static io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters; import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getCursorBasedSyncStatusForStreams; @@ -44,14 +44,13 @@ import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.integrations.source.mysql.cdc.CdcConfigurationHelper; import io.airbyte.integrations.source.mysql.cursor_based.MySqlCursorBasedStateManager; -import io.airbyte.integrations.source.mysql.helpers.CdcConfigurationHelper; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadHandler; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStreamStateManager; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil; @@ -70,6 +69,8 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.SyncMode; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.time.Instant; import java.util.ArrayList; @@ -261,7 +262,7 @@ public Collection> readStreams(final JsonN final AirbyteStateType supportedStateType = getSupportedStateType(config); final StateManager stateManager = StateManagerFactory.createStateManager(supportedStateType, - StateGeneratorUtils.deserializeInitialState(state, featureFlags.useStreamCapableState(), supportedStateType), catalog); + StateGeneratorUtils.deserializeInitialState(state, supportedStateType), catalog); final Instant emittedAt = Instant.now(); final JdbcDatabase database = createDatabase(config); @@ -307,7 +308,7 @@ protected void logPreSyncDebugData(final JdbcDatabase database, final Configured @Override public JsonNode toDatabaseConfig(final JsonNode config) { - final String encodedDatabaseName = HostPortResolver.encodeValue(config.get(JdbcUtils.DATABASE_KEY).asText()); + final String encodedDatabaseName = URLEncoder.encode(config.get(JdbcUtils.DATABASE_KEY).asText(), StandardCharsets.UTF_8); final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asText(), @@ -380,10 +381,6 @@ private static boolean isCdc(final JsonNode config) { @Override protected AirbyteStateType getSupportedStateType(final JsonNode config) { - if (!featureFlags.useStreamCapableState()) { - return AirbyteStateType.LEGACY; - } - return isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; } @@ -504,10 +501,6 @@ private boolean convertToBoolean(final String value) { return "1".equalsIgnoreCase(value) || "true".equalsIgnoreCase(value); } - private String toSslJdbcParam(final SslMode sslMode) { - return toSslJdbcParamInternal(sslMode); - } - private boolean cloudDeploymentMode() { return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); } @@ -517,7 +510,7 @@ protected int getStateEmissionFrequency() { return INTERMEDIATE_STATE_EMISSION_FREQUENCY; } - protected static String toSslJdbcParamInternal(final SslMode sslMode) { + public static String toSslJdbcParam(final SslMode sslMode) { final var result = switch (sslMode) { case DISABLED, PREFERRED, REQUIRED, VERIFY_CA, VERIFY_IDENTITY -> sslMode.name(); default -> throw new IllegalArgumentException("unexpected ssl mode"); @@ -529,13 +522,15 @@ protected static String toSslJdbcParamInternal(final SslMode sslMode) { public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { // return super.createDatabase(sourceConfig, this::getConnectionProperties); final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = this.getConnectionProperties(sourceConfig); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - this.getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSpecConstants.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSpecConstants.java new file mode 100644 index 000000000000..7735470482da --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSpecConstants.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +// Constants defined in +// airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json. +public class MySqlSpecConstants { + + public static final String INVALID_CDC_CURSOR_POSITION_PROPERTY = "invalid_cdc_cursor_position_behavior"; + public static final String FAIL_SYNC_OPTION = "Fail sync"; + public static final String RESYNC_DATA_OPTION = "Re-sync data"; + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CdcConfigurationHelper.java similarity index 98% rename from airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CdcConfigurationHelper.java index 5e2e93ddc579..2f5d02caf72f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CdcConfigurationHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mysql.helpers; +package io.airbyte.integrations.source.mysql.cdc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CustomMySQLTinyIntOneToBooleanConverter.java similarity index 90% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CustomMySQLTinyIntOneToBooleanConverter.java index f353edaebc02..eae5f20256b3 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/CustomMySQLTinyIntOneToBooleanConverter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; import io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter; import io.debezium.spi.converter.RelationalColumn; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySQLDateTimeConverter.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySQLDateTimeConverter.java index 6d8fd94c1f16..233948c5b31b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySQLDateTimeConverter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; import io.airbyte.cdk.db.jdbc.DateTimeConverter; import io.airbyte.cdk.db.jdbc.JdbcDatabase; @@ -27,9 +27,8 @@ * https://debezium.io/documentation/reference/2.1/development/converters.html This is built from * reference with {@link io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter} If you * rename this class then remember to rename the datetime.type property value in - * {@link io.airbyte.integrations.source.mysql.MySqlCdcProperties#commonProperties(JdbcDatabase)} - * (If you don't rename, a test would still fail but it might be tricky to figure out where to - * change the property name) + * {@link MySqlCdcProperties#commonProperties(JdbcDatabase)} (If you don't rename, a test would + * still fail but it might be tricky to figure out where to change the property name) */ public class MySQLDateTimeConverter implements CustomConverter { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcConnectorMetadataInjector.java similarity index 92% rename from airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcConnectorMetadataInjector.java index bfe231a96b4f..d43f83e725c9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcConnectorMetadataInjector.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mysql; +package io.airbyte.integrations.source.mysql.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; @@ -13,7 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcPosition.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcPosition.java index 6047ee695918..04cc8430142f 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcPosition.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; import java.util.Objects; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcProperties.java similarity index 95% rename from airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcProperties.java index 15bc34eefdcf..9b54d0588f67 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcProperties.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mysql; +package io.airbyte.integrations.source.mysql.cdc; import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; @@ -13,9 +13,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.internals.mysql.CustomMySQLTinyIntOneToBooleanConverter; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySQLDateTimeConverter; import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.integrations.source.mysql.MySqlSource; import java.net.URI; import java.nio.file.Path; import java.time.Duration; @@ -90,7 +89,7 @@ private static Properties commonProperties(final JdbcDatabase database) { // https://debezium.io/documentation/reference/2.2/connectors/mysql.html#mysql-property-database-ssl-mode if (!sourceConfig.has(JdbcUtils.SSL_KEY) || sourceConfig.get(JdbcUtils.SSL_KEY).asBoolean()) { if (dbConfig.has(SSL_MODE) && !dbConfig.get(SSL_MODE).asText().isEmpty()) { - props.setProperty("database.ssl.mode", MySqlSource.toSslJdbcParamInternal(SslMode.valueOf(dbConfig.get(SSL_MODE).asText()))); + props.setProperty("database.ssl.mode", MySqlSource.toSslJdbcParam(SslMode.valueOf(dbConfig.get(SSL_MODE).asText()))); if (dbConfig.has(TRUST_KEY_STORE_URL) && !dbConfig.get(TRUST_KEY_STORE_URL).asText().isEmpty()) { props.setProperty("database.ssl.truststore", Path.of(URI.create(dbConfig.get(TRUST_KEY_STORE_URL).asText())).toString()); diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcSavedInfoFetcher.java similarity index 79% rename from airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcSavedInfoFetcher.java index b5d3d3a81643..7d23671afacb 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcSavedInfoFetcher.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mysql; +package io.airbyte.integrations.source.mysql.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.IS_COMPRESSED; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_DB_HISTORY; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.IS_COMPRESSED; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.MYSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.debezium.CdcSavedInfoFetcher; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcStateHandler.java similarity index 90% rename from airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcStateHandler.java index 93a489a56b08..a6f672c7ab40 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcStateHandler.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mysql; +package io.airbyte.integrations.source.mysql.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.serialize; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.COMPRESSION_ENABLED; +import static io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil.serialize; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.COMPRESSION_ENABLED; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.debezium.CdcStateHandler; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcTargetPosition.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcTargetPosition.java index f36876620144..18a7ff5875fb 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlCdcTargetPosition.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlDebeziumStateUtil.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlDebeziumStateUtil.java index 08444e1044a3..dda584d1de03 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MySqlDebeziumStateUtil.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.COMPRESSION_ENABLED; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.COMPRESSION_ENABLED; import static io.debezium.relational.RelationalDatabaseConnectorConfig.DATABASE_NAME; import com.fasterxml.jackson.databind.JsonNode; @@ -169,10 +169,9 @@ public Optional savedOffset(final Properties baseP return Optional.empty(); } - final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog, - AirbyteFileOffsetBackingStore.initializeState(cdcOffset, Optional.empty()), - Optional.empty()); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + final var offsetManager = AirbyteFileOffsetBackingStore.initializeState(cdcOffset, Optional.empty()); + final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); return parseSavedOffset(debeziumProperties); } @@ -238,19 +237,19 @@ public JsonNode constructInitialDebeziumState(final Properties properties, // We use the schema_only_recovery property cause using this mode will instruct Debezium to // construct the db schema history. properties.setProperty("snapshot.mode", "schema_only_recovery"); + final String dbName = database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText(); + // Topic.prefix is sanitized version of database name. At this stage properties does not have this + // value - it's set in RelationalDbDebeziumPropertiesManager. final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState( - constructBinlogOffset(database, database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText()), + constructBinlogOffset(database, dbName, DebeziumPropertiesManager.sanitizeTopicPrefix(dbName)), Optional.empty()); final AirbyteSchemaHistoryStorage schemaHistoryStorage = AirbyteSchemaHistoryStorage.initializeDBHistory(new SchemaHistory<>(Optional.empty(), false), COMPRESSION_ENABLED); final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(); - try (final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(properties, - database.getSourceConfig(), - catalog, - offsetManager, - Optional.of(schemaHistoryStorage), - DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB)) { - publisher.start(queue); + final var debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(properties, database.getSourceConfig(), catalog); + + try (final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(debeziumPropertiesManager)) { + publisher.start(queue, offsetManager, Optional.of(schemaHistoryStorage)); final Instant engineStartTime = Instant.now(); while (!publisher.hasClosed()) { final ChangeEvent event = queue.poll(10, TimeUnit.SECONDS); @@ -307,13 +306,13 @@ public static JsonNode serialize(final Map offset, final SchemaH * Method to construct initial Debezium state which can be passed onto Debezium engine to make it * process binlogs from a specific file and position and skip snapshot phase */ - private JsonNode constructBinlogOffset(final JdbcDatabase database, final String dbName) { - return format(getStateAttributesFromDB(database), dbName, Instant.now()); + private JsonNode constructBinlogOffset(final JdbcDatabase database, final String debeziumName, final String topicPrefixName) { + return format(getStateAttributesFromDB(database), debeziumName, topicPrefixName, Instant.now()); } @VisibleForTesting - public JsonNode format(final MysqlDebeziumStateAttributes attributes, final String dbName, final Instant time) { - final String key = "[\"" + dbName + "\",{\"server\":\"" + dbName + "\"}]"; + public JsonNode format(final MysqlDebeziumStateAttributes attributes, final String debeziumName, final String topicPrefixName, final Instant time) { + final String key = "[\"" + debeziumName + "\",{\"server\":\"" + topicPrefixName + "\"}]"; final String gtidSet = attributes.gtidSet().isPresent() ? ",\"gtids\":\"" + attributes.gtidSet().get() + "\"" : ""; final String value = "{\"transaction_id\":null,\"ts_sec\":" + time.getEpochSecond() + ",\"file\":\"" + attributes.binlogFilename() + "\",\"pos\":" diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MysqlCdcStateConstants.java similarity index 85% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java rename to airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MysqlCdcStateConstants.java index 68608775fc1f..cac1bfd997d5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cdc/MysqlCdcStateConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.mysql; +package io.airbyte.integrations.source.mysql.cdc; public class MysqlCdcStateConstants { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java index 3393e68ae124..e810d860e4c8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java @@ -86,7 +86,8 @@ public void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, } @Override - public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, + final JsonNode streamStateForIncrementalRun) { streamsThatHaveCompletedSnapshot.add(pair); final List streamStates = new ArrayList<>(); streamsThatHaveCompletedSnapshot.forEach(stream -> { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java index 727cfc9b1cc2..236c65659f82 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java @@ -11,8 +11,11 @@ import com.google.common.annotations.VisibleForTesting; import com.mysql.cj.MysqlType; import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIterator; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIteratorManager; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; @@ -180,13 +183,16 @@ private AutoCloseableIterator augmentWithState(final AutoCloseab final Duration syncCheckpointDuration = config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) - : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_DURATION; + : DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() - : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_RECORDS; + : DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; + + final SourceStateIteratorManager processor = + new MySqlInitialSyncStateIteratorManager(pair, initialLoadStateManager, incrementalState, + syncCheckpointDuration, syncCheckpointRecords); return AutoCloseableIterators.transformIterator( - r -> new MySqlInitialSyncStateIterator(r, pair, initialLoadStateManager, incrementalState, - syncCheckpointDuration, syncCheckpointRecords), + r -> new SourceStateIterator<>(r, processor), recordIterator, pair); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java index 3b811d3047cc..74d9f985e3ac 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; import io.airbyte.integrations.source.mysql.MySqlSourceOperations; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java index 7bb6a7b846ae..be5cec573294 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java @@ -25,7 +25,8 @@ public interface MySqlInitialLoadStateManager { void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus); // Returns the final state message for the initial sync. - AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun); + AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, + final JsonNode streamStateForIncrementalRun); // Returns the previous state emitted, represented as a {@link PrimaryKeyLoadStatus} associated with // the stream. diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java index 8ca08abb0ffd..47aa83ee09fc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java @@ -4,9 +4,12 @@ package io.airbyte.integrations.source.mysql.initialsync; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; +import static io.airbyte.cdk.db.DbAnalyticsUtils.cdcCursorInvalidMessage; import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getTableSizeInfoForStreams; import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.prettyPrintConfiguredAirbyteStreamList; +import static io.airbyte.integrations.source.mysql.MySqlSpecConstants.FAIL_SYNC_OPTION; +import static io.airbyte.integrations.source.mysql.MySqlSpecConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadGlobalStateManager.STATE_TYPE_KEY; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; @@ -16,25 +19,27 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcPosition; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumEventConverter; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; -import io.airbyte.integrations.source.mysql.MySqlCdcProperties; -import io.airbyte.integrations.source.mysql.MySqlCdcSavedInfoFetcher; -import io.airbyte.integrations.source.mysql.MySqlCdcStateHandler; import io.airbyte.integrations.source.mysql.MySqlQueryUtils; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcPosition; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcProperties; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcSavedInfoFetcher; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcStateHandler; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcTargetPosition; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadSourceOperations.CdcMetadataInjector; import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; @@ -57,7 +62,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -109,6 +113,12 @@ public static List> getCdcReadIterators(fi savedOffset.isPresent() && mySqlDebeziumStateUtil.savedOffsetStillPresentOnServer(database, savedOffset.get()); if (!savedOffsetStillPresentOnServer) { + AirbyteTraceMessageUtility.emitAnalyticsTrace(cdcCursorInvalidMessage()); + if (!sourceConfig.get("replication_method").has(INVALID_CDC_CURSOR_POSITION_PROPERTY) || sourceConfig.get("replication_method").get( + INVALID_CDC_CURSOR_POSITION_PROPERTY).asText().equals(FAIL_SYNC_OPTION)) { + throw new ConfigErrorException( + "Saved offset no longer present on the server. Please reset the connection, and then increase binlog retention or reduce sync frequency. See https://docs.airbyte.com/integrations/sources/mysql/mysql-troubleshooting#under-cdc-incremental-mode-there-are-still-full-refresh-syncs for more details."); + } LOGGER.warn("Saved offset no longer present on the server, Airbyte is going to trigger a sync from scratch"); } @@ -156,16 +166,14 @@ public static List> getCdcReadIterators(fi true, firstRecordWaitTime, subsequentRecordWaitTime, - OptionalInt.empty()); - - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, - new MySqlCdcSavedInfoFetcher(stateToBeUsed), - new MySqlCdcStateHandler(stateManager), - metadataInjector, - MySqlCdcProperties.getDebeziumProperties(database), - DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, - emittedAt, + AirbyteDebeziumHandler.QUEUE_CAPACITY, false); + final var propertiesManager = new RelationalDbDebeziumPropertiesManager( + MySqlCdcProperties.getDebeziumProperties(database), sourceConfig, catalog); + final var eventConverter = new RelationalDbDebeziumEventConverter(metadataInjector, emittedAt); + + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators( + propertiesManager, eventConverter, new MySqlCdcSavedInfoFetcher(stateToBeUsed), new MySqlCdcStateHandler(stateManager)); // This starts processing the binglogs as soon as initial sync is complete, this is a bit different // from the current cdc syncs. diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java deleted file mode 100644 index 03a62dc50203..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql.initialsync; - -import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.MYSQL_STATUS_VERSION; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.AbstractIterator; -import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; -import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import java.time.Duration; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.util.Iterator; -import java.util.Objects; -import javax.annotation.CheckForNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MySqlInitialSyncStateIterator extends AbstractIterator implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialSyncStateIterator.class); - public static final Duration SYNC_CHECKPOINT_DURATION = DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; - public static final Integer SYNC_CHECKPOINT_RECORDS = DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; - - private final Iterator messageIterator; - private final AirbyteStreamNameNamespacePair pair; - private boolean hasEmittedFinalState = false; - private PrimaryKeyLoadStatus pkStatus; - private final JsonNode streamStateForIncrementalRun; - private final MySqlInitialLoadStateManager stateManager; - private long recordCount = 0L; - private Instant lastCheckpoint = Instant.now(); - private final Duration syncCheckpointDuration; - private final Long syncCheckpointRecords; - private final String pkFieldName; - - public MySqlInitialSyncStateIterator(final Iterator messageIterator, - final AirbyteStreamNameNamespacePair pair, - final MySqlInitialLoadStateManager stateManager, - final JsonNode streamStateForIncrementalRun, - final Duration checkpointDuration, - final Long checkpointRecords) { - this.messageIterator = messageIterator; - this.pair = pair; - this.stateManager = stateManager; - this.streamStateForIncrementalRun = streamStateForIncrementalRun; - this.syncCheckpointDuration = checkpointDuration; - this.syncCheckpointRecords = checkpointRecords; - this.pkFieldName = stateManager.getPrimaryKeyInfo(pair).pkFieldName(); - this.pkStatus = stateManager.getPrimaryKeyLoadStatus(pair); - } - - @CheckForNull - @Override - protected AirbyteMessage computeNext() { - if (messageIterator.hasNext()) { - if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) - && Objects.nonNull(pkStatus)) { - LOGGER.info("Emitting initial sync pk state for stream {}, state is {}", pair, pkStatus); - recordCount = 0L; - lastCheckpoint = Instant.now(); - return new AirbyteMessage() - .withType(Type.STATE) - .withState(stateManager.createIntermediateStateMessage(pair, pkStatus)); - } - // Use try-catch to catch Exception that could occur when connection to the database fails - try { - final AirbyteMessage message = messageIterator.next(); - if (Objects.nonNull(message)) { - final String lastPk = message.getRecord().getData().get(pkFieldName).asText(); - pkStatus = new PrimaryKeyLoadStatus() - .withVersion(MYSQL_STATUS_VERSION) - .withStateType(StateType.PRIMARY_KEY) - .withPkName(pkFieldName) - .withPkVal(lastPk) - .withIncrementalState(streamStateForIncrementalRun); - stateManager.updatePrimaryKeyLoadState(pair, pkStatus); - } - recordCount++; - return message; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } else if (!hasEmittedFinalState) { - hasEmittedFinalState = true; - final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); - LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); - return new AirbyteMessage() - .withType(Type.STATE) - .withState(finalStateMessage); - } else { - return endOfData(); - } - } - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java new file mode 100644 index 000000000000..f7722e1844da --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.MYSQL_STATUS_VERSION; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIteratorManager; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlInitialSyncStateIteratorManager implements SourceStateIteratorManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialSyncStateIteratorManager.class); + + private final AirbyteStreamNameNamespacePair pair; + private PrimaryKeyLoadStatus pkStatus; + private final JsonNode streamStateForIncrementalRun; + private final MySqlInitialLoadStateManager stateManager; + private final Duration syncCheckpointDuration; + private final Long syncCheckpointRecords; + private final String pkFieldName; + + public MySqlInitialSyncStateIteratorManager( + final AirbyteStreamNameNamespacePair pair, + final MySqlInitialLoadStateManager stateManager, + final JsonNode streamStateForIncrementalRun, + final Duration checkpointDuration, + final Long checkpointRecords) { + this.pair = pair; + this.stateManager = stateManager; + this.streamStateForIncrementalRun = streamStateForIncrementalRun; + this.syncCheckpointDuration = checkpointDuration; + this.syncCheckpointRecords = checkpointRecords; + this.pkFieldName = stateManager.getPrimaryKeyInfo(pair).pkFieldName(); + this.pkStatus = stateManager.getPrimaryKeyLoadStatus(pair); + } + + @Override + public AirbyteStateMessage generateStateMessageAtCheckpoint() { + LOGGER.info("Emitting initial sync pk state for stream {}, state is {}", pair, pkStatus); + return stateManager.createIntermediateStateMessage(pair, pkStatus); + } + + @Override + public AirbyteMessage processRecordMessage(final AirbyteMessage message) { + if (Objects.nonNull(message)) { + final String lastPk = message.getRecord().getData().get(pkFieldName).asText(); + pkStatus = new PrimaryKeyLoadStatus() + .withVersion(MYSQL_STATUS_VERSION) + .withStateType(StateType.PRIMARY_KEY) + .withPkName(pkFieldName) + .withPkVal(lastPk) + .withIncrementalState(streamStateForIncrementalRun); + stateManager.updatePrimaryKeyLoadState(pair, pkStatus); + } + return message; + } + + @Override + public AirbyteStateMessage createFinalStateMessage() { + final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); + LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); + return finalStateMessage; + } + + @Override + public boolean shouldEmitStateMessage(long recordCount, Instant lastCheckpoint) { + return (recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) + && Objects.nonNull(pkStatus); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 841fa1f3bdba..78450b13aabd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -211,6 +211,15 @@ "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", "order": 2, "always_show": true + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 3, + "always_show": true } } }, diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java index 689e6a531139..8eb622e72716 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java @@ -12,10 +12,10 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.io.File; import java.io.IOException; +import java.util.Base64; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -480,7 +480,7 @@ private String getLogString(final int length) { private String getFileDataInBase64() { final File file = new File(getClass().getClassLoader().getResource("test.png").getFile()); try { - return Base64.encodeBase64String(FileUtils.readFileToByteArray(file)); + return Base64.getEncoder().encodeToString(FileUtils.readFileToByteArray(file)); } catch (final IOException e) { LOGGER.error(String.format("Fail to read the file: %s. Error: %s", file.getAbsoluteFile(), e.getMessage())); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java index a5d57eeb336c..61b79b8d48df 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java @@ -10,8 +10,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.Field; @@ -30,11 +28,6 @@ public abstract class AbstractSshMySqlSourceAcceptanceTest extends SourceAccepta private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - private JsonNode config; public abstract Path getConfigFilePath(); @@ -90,9 +83,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java index fd7ea961688a..9400c4e66368 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java @@ -8,6 +8,7 @@ import io.airbyte.cdk.db.Database; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import org.junit.jupiter.api.Test; public class CDCMySqlDatatypeAccuracyTest extends MySqlDatatypeAccuracyTest { @@ -26,4 +27,11 @@ protected Database setupDatabase() { return testdb.getDatabase(); } + // Temporarily disable this test since it's causing trouble on GHA. + @Override + @Test + public void testDataContent() { + // Do Nothing + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java index 04fdacb09b50..82ab112d7e15 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java @@ -8,8 +8,6 @@ import com.google.common.collect.Iterables; import io.airbyte.cdk.db.Database; import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; @@ -24,11 +22,6 @@ public class CdcBinlogsMySqlSourceDatatypeTest extends AbstractMySqlSourceDataty private JsonNode stateAfterFirstSync; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java index 6a3550b561cc..6b971c86927c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java @@ -6,18 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; public class CdcInitialSnapshotMySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java index 08aaa08e0ee3..9e12122460b7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java @@ -15,8 +15,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; @@ -44,11 +42,6 @@ public class CdcMySqlSourceAcceptanceTest extends SourceAcceptanceTest { protected MySQLTestDatabase testdb; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected String getImageName() { return "airbyte/source-mysql:dev"; @@ -139,7 +132,7 @@ public void testIncrementalSyncShouldNotFailIfBinlogIsDeleted() throws Exception // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); + final JsonNode latestState = Jsons.jsonNode(List.of(Iterables.getLast(stateMessages))); // RESET MASTER removes all binary log files that are listed in the index file, // leaving only a single, empty binary log file with a numeric suffix of .000001 testdb.with("RESET MASTER;"); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java index 07597d1ab27c..516d6c20a425 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java @@ -8,8 +8,6 @@ import com.mysql.cj.MysqlType; import io.airbyte.cdk.db.Database; import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mysql.MySQLContainerFactory; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.protocol.models.JsonSchemaType; @@ -20,11 +18,6 @@ public class MySqlDatatypeAccuracyTest extends AbstractMySqlSourceDatatypeTest { - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java index 9c15db51a820..d6a2adffe2c9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java @@ -9,8 +9,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; @@ -42,11 +40,6 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - protected ContainerModifier[] getContainerModifiers() { return ArrayUtils.toArray(); } @@ -102,9 +95,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java index 5b4f86eae403..cbfa689562dc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java @@ -6,18 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mysql.MySQLContainerFactory; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; public class MySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected JsonNode getConfig() { return testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json index 50d717a95886..871b7c0c38bb 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json @@ -189,6 +189,15 @@ "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", "order": 2, "always_show": true + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 3, + "always_show": true } } }, diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json index 1a884d8de813..7ffbbad5f718 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json @@ -211,6 +211,15 @@ "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", "order": 2, "always_show": true + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 3, + "always_show": true } } }, diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java index 39c2a8ff3227..d9ff7f362056 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java @@ -10,6 +10,7 @@ import io.airbyte.cdk.db.factory.DSLContextFactory; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import io.airbyte.commons.json.Jsons; @@ -60,7 +61,8 @@ protected Database setupDatabase(final String dbName) throws Exception { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))); + Map.of("zeroDateTimeBehavior", "convertToNull"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT)); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java index 2b58bc5255b5..d65977312b04 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java @@ -10,6 +10,7 @@ import io.airbyte.cdk.db.factory.DSLContextFactory; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -42,7 +43,7 @@ protected void setupDatabase(final String dbName) throws Exception { .put("replication_method", plainConfig.get("replication_method")) .build()); - try (final DSLContext dslContext = DSLContextFactory.create( + final DSLContext dslContext = DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), @@ -51,14 +52,14 @@ protected void setupDatabase(final String dbName) throws Exception { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))) { + Map.of("zeroDateTimeBehavior", "convertToNull"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); - final Database database = new Database(dslContext); + final Database database = new Database(dslContext); - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.execute("SET @@sql_mode=''")); - } + // It disable strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + database.query(ctx -> ctx.execute("SET @@sql_mode=''")); } /** diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcConfigurationHelperTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcConfigurationHelperTest.java index 8484226ffaa0..6c6a370c15fd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcConfigurationHelperTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcConfigurationHelperTest.java @@ -9,7 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.mysql.helpers.CdcConfigurationHelper; +import io.airbyte.integrations.source.mysql.cdc.CdcConfigurationHelper; import java.util.Collections; import java.util.Map; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 7e9ec4db1bd5..b88b5baa6420 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -5,14 +5,15 @@ package io.airbyte.integrations.source.mysql; import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.IS_COMPRESSED; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; -import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_DB_HISTORY; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; +import static io.airbyte.integrations.source.mysql.MySqlSpecConstants.FAIL_SYNC_OPTION; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.IS_COMPRESSED; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.source.mysql.cdc.MysqlCdcStateConstants.MYSQL_DB_HISTORY; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.STATE_TYPE_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -20,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; @@ -33,14 +35,14 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.debezium.CdcSourceTest; import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcProperties; +import io.airbyte.integrations.source.mysql.cdc.MySqlCdcTargetPosition; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -84,9 +86,7 @@ protected MySQLTestDatabase createTestDatabase() { @Override protected MySqlSource source() { - final var source = new MySqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MySqlSource(); } @Override @@ -103,7 +103,12 @@ protected void purgeAllBinaryLogs() { @Override protected String createSchemaSqlFmt() { - return "CREATE DATABASE IF NOT EXISTS %s;"; + return "CREATE DATABASE IF NOT EXISTS `%s`;"; + } + + @Override + protected String createTableSqlFmt() { + return "CREATE TABLE `%s`.`%s`(%s);"; } @Override @@ -179,6 +184,36 @@ protected void addCdcDefaultCursorField(final AirbyteStream stream) { } } + @Override + protected void writeRecords( + final JsonNode recordJson, + final String dbName, + final String streamName, + final String idCol, + final String makeIdCol, + final String modelCol) { + testdb.with("INSERT INTO `%s` .`%s` (%s, %s, %s) VALUES (%s, %s, '%s');", dbName, streamName, + idCol, makeIdCol, modelCol, + recordJson.get(idCol).asInt(), recordJson.get(makeIdCol).asInt(), + recordJson.get(modelCol).asText()); + } + + @Override + protected void deleteMessageOnIdCol(final String streamName, final String idCol, final int idValue) { + testdb.with("DELETE FROM `%s`.`%s` WHERE %s = %s", modelsSchema(), streamName, idCol, idValue); + } + + @Override + protected void deleteCommand(final String streamName) { + testdb.with("DELETE FROM `%s`.`%s`", modelsSchema(), streamName); + } + + @Override + protected void updateCommand(final String streamName, final String modelCol, final String modelVal, final String idCol, final int idValue) { + testdb.with("UPDATE `%s`.`%s` SET %s = '%s' WHERE %s = %s", modelsSchema(), streamName, + modelCol, modelVal, COL_ID, 11); + } + @Test protected void syncWithReplicationClientPrivilegeRevokedFailsCheck() throws Exception { testdb.with("REVOKE REPLICATION CLIENT ON *.* FROM %s@'%%';", testdb.getUserName()); @@ -245,6 +280,12 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { dataFromSecondBatch); assertEquals((recordsToCreate * 2) + recordsCreatedBeforeTestCount, recordsFromSecondBatch.size(), "Expected 46 records to be replicated in the second sync."); + + JsonNode failSyncConfig = testdb.testConfigBuilder() + .withCdcReplication(FAIL_SYNC_OPTION) + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); + assertThrows(ConfigErrorException.class, () -> source().read(failSyncConfig, getConfiguredCatalog(), state)); } /** @@ -292,6 +333,14 @@ protected void assertExpectedStateMessages(final List state assertStateTypes(stateMessages, 4); } + protected void assertExpectedStateMessagesWithTotalCount(final List stateMessages, final long totalRecordCount) { + long actualRecordCount = 0L; + for (final AirbyteStateMessage message : stateMessages) { + actualRecordCount += message.getSourceStats().getRecordCount(); + } + assertEquals(actualRecordCount, totalRecordCount); + } + @Override protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { assertEquals(1, stateMessages.size()); @@ -357,7 +406,7 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List streamsInSnapshotState = stateMessage.getGlobal().getStreamStates() @@ -382,7 +431,7 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List streamsInSnapshotState = secondLastSateMessage.getGlobal().getStreamStates() @@ -399,7 +448,7 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() @@ -437,6 +486,7 @@ public void syncWouldWorkWithDBWithInvalidTimezone() throws Exception { assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages); assertExpectedStateMessages(stateMessages); + assertExpectedStateMessagesWithTotalCount(stateMessages, 6); } @Test @@ -455,6 +505,7 @@ public void testCompositeIndexInitialLoad() throws Exception { final List stateMessages1 = extractStateMessages(actualRecords1); assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages1); assertExpectedStateMessages(stateMessages1); + assertExpectedStateMessagesWithTotalCount(stateMessages1, 6); // Re-run the sync with state associated with record w/ id = 15 (second to last record). // We expect to read 2 records, since in the case of a composite PK we issue a >= query. @@ -518,6 +569,8 @@ public void testTwoStreamSync() throws Exception { final Set recordMessages1 = extractRecordMessages(actualRecords1); final List stateMessages1 = extractStateMessages(actualRecords1); assertEquals(13, stateMessages1.size()); + assertExpectedStateMessagesWithTotalCount(stateMessages1, 12); + JsonNode sharedState = null; StreamDescriptor firstStreamInState = null; for (int i = 0; i < stateMessages1.size(); i++) { @@ -586,6 +639,8 @@ public void testTwoStreamSync() throws Exception { final List stateMessages2 = extractStateMessages(actualRecords2); assertEquals(6, stateMessages2.size()); + // State was reset to the 7th; thus 5 remaining records were expected to be reloaded. + assertExpectedStateMessagesWithTotalCount(stateMessages2, 5); for (int i = 0; i < stateMessages2.size(); i++) { final AirbyteStateMessage stateMessage = stateMessages2.get(i); assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceWithSpecialDbNameTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceWithSpecialDbNameTest.java new file mode 100644 index 000000000000..9dd4170500ec --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceWithSpecialDbNameTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import org.testcontainers.containers.MySQLContainer; + +public class CdcMysqlSourceWithSpecialDbNameTest extends CdcMysqlSourceTest { + + @Override + protected MySQLTestDatabase createTestDatabase() { + var container = new MySQLContainerFactory().shared( + BaseImage.MYSQL_8.reference, + ContainerModifier.INVALID_TIMEZONE_CEST.methodName, + ContainerModifier.CUSTOM_NAME.methodName); + return new TestDatabaseWithInvalidDatabaseName(container) + .initialized() + .withCdcPermissions(); + } + + static class TestDatabaseWithInvalidDatabaseName extends MySQLTestDatabase { + + public static final String INVALID_DB_NAME = "invalid@name"; + + public TestDatabaseWithInvalidDatabaseName(MySQLContainer container) { + super(container); + } + + @Override + public String getDatabaseName() { + return INVALID_DB_NAME; + } + + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java index 7058a7f87acf..67c8f4d68687 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java @@ -4,15 +4,12 @@ package io.airbyte.integrations.source.mysql; import io.airbyte.cdk.integrations.debug.DebugUtil; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; public class MySqlDebugger { @SuppressWarnings({"unchecked", "deprecation", "resource"}) public static void main(final String[] args) throws Exception { final MySqlSource mysqlSource = new MySqlSource(); - mysqlSource.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); DebugUtil.debug(mysqlSource); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 42999aa1dceb..d6597cd2b023 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -23,8 +23,6 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; @@ -40,6 +38,7 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -72,9 +71,7 @@ protected JsonNode config() { @Override protected MySqlSource source() { - final var source = new MySqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MySqlSource(); } @Override @@ -94,7 +91,8 @@ protected boolean supportsSchemas() { } @Test - void testReadMultipleTablesIncrementally() throws Exception { + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { final var config = config(); ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1); final String streamOneName = TABLE_NAME + "one"; @@ -384,13 +382,13 @@ protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configure @Override protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, @@ -405,15 +403,10 @@ protected List getExpectedAirbyteMessagesSecondSync(final String .withCursor("5") .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2L)); return expectedMessages; } - @Override - protected boolean supportsPerStream() { - return true; - } - @Override protected List getTestMessages() { return getTestMessages(streamName()); @@ -485,31 +478,28 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { // Override from parent class as we're no longer including the legacy Data field. @Override - protected List createExpectedTestMessages(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s))))) - .collect( - Collectors.toList()) - : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY))); + protected List createExpectedTestMessages(final List states, final long numRecords) { + return states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) numRecords)))) + .collect( + Collectors.toList()); } @Override protected List createState(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s)))) - .collect( - Collectors.toList()) - : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY)); + return states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 83fec037e3e3..ef8bb7646677 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -17,8 +17,6 @@ import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource.PrimaryKeyAttributesFromDb; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; @@ -41,9 +39,7 @@ public class MySqlSourceTests { public MySqlSource source() { - final var source = new MySqlSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new MySqlSource(); } @Test diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index febcd7a5c0c1..66fba410c3c0 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -22,7 +22,6 @@ import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -79,11 +78,6 @@ public void setup() throws Exception { super.setup(); } - @AfterEach - void tearDown() { - dslContext.close(); - } - @AfterAll static void cleanUp() { container.close(); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlDebeziumStateUtilTest.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java rename to airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlDebeziumStateUtilTest.java index 7ba8a705691e..284fdaa300f0 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlDebeziumStateUtilTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals; +package io.airbyte.integrations.source.mysql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -14,10 +14,10 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; -import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil; +import io.airbyte.integrations.source.mysql.cdc.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -91,8 +91,8 @@ public void debeziumInitialStateConstructTest() throws SQLException { @Test public void formatTestWithGtid() { final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); - final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes("binlog.000002", 633, - Optional.of("3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5")), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MysqlDebeziumStateAttributes("binlog.000002", 633, + Optional.of("3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5")), "db_fgnfxvllud", "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); final Map stateAsMap = Jsons.object(debeziumState, Map.class); Assertions.assertEquals(1, stateAsMap.size()); Assertions.assertTrue(stateAsMap.containsKey("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); @@ -113,15 +113,15 @@ public void formatTestWithGtid() { debeziumState, config); Assertions.assertTrue(parsedOffset.isPresent()); final JsonNode stateGeneratedUsingParsedOffset = - mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); Assertions.assertEquals(debeziumState, stateGeneratedUsingParsedOffset); } @Test public void formatTestWithoutGtid() { final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); - final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes("binlog.000002", 633, - Optional.empty()), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MysqlDebeziumStateAttributes("binlog.000002", 633, + Optional.empty()), "db_fgnfxvllud", "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); final Map stateAsMap = Jsons.object(debeziumState, Map.class); Assertions.assertEquals(1, stateAsMap.size()); Assertions.assertTrue(stateAsMap.containsKey("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); @@ -141,7 +141,7 @@ public void formatTestWithoutGtid() { debeziumState, config); Assertions.assertTrue(parsedOffset.isPresent()); final JsonNode stateGeneratedUsingParsedOffset = - mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); Assertions.assertEquals(debeziumState, stateGeneratedUsingParsedOffset); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json index 52441e124b17..66f0b3bdf647 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json @@ -205,6 +205,15 @@ "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", "order": 2, "always_show": true + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 3, + "always_show": true } } }, diff --git a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json index 841fa1f3bdba..78450b13aabd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json @@ -211,6 +211,15 @@ "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", "order": 2, "always_show": true + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 3, + "always_show": true } } }, diff --git a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java index 74c745cb7f7f..2e9fba65fbb7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java +++ b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java @@ -11,18 +11,13 @@ import org.testcontainers.containers.Network; import org.testcontainers.utility.DockerImageName; -public class MySQLContainerFactory implements ContainerFactory> { +public class MySQLContainerFactory extends ContainerFactory> { @Override - public MySQLContainer createNewContainer(DockerImageName imageName) { + protected MySQLContainer createNewContainer(DockerImageName imageName) { return new MySQLContainer<>(imageName.asCompatibleSubstituteFor("mysql")); } - @Override - public Class getContainerClass() { - return MySQLContainer.class; - } - /** * Create a new network and bind it to the container. */ @@ -40,6 +35,8 @@ public void withMoscowTimezone(MySQLContainer container) { container.withEnv("TZ", "Europe/Moscow"); } + public void withCustomName(MySQLContainer container) {} // do nothing + public void withRootAndServerCertificates(MySQLContainer container) { execInContainer(container, "sed -i '31 a ssl' /etc/my.cnf", diff --git a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java index cd0565ebf25c..219d5e90f479 100644 --- a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java +++ b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.source.mysql; +import static io.airbyte.integrations.source.mysql.MySqlSpecConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; +import static io.airbyte.integrations.source.mysql.MySqlSpecConstants.RESYNC_DATA_OPTION; + import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.testutils.TestDatabase; @@ -17,31 +20,32 @@ public class MySQLTestDatabase extends TestDatabase, MySQLTestDatabase, MySQLTestDatabase.MySQLConfigBuilder> { - public static enum BaseImage { + public enum BaseImage { MYSQL_8("mysql:8.0"), ; - private final String reference; + public final String reference; - private BaseImage(String reference) { + BaseImage(String reference) { this.reference = reference; } } - public static enum ContainerModifier { + public enum ContainerModifier { MOSCOW_TIMEZONE("withMoscowTimezone"), INVALID_TIMEZONE_CEST("withInvalidTimezoneCEST"), ROOT_AND_SERVER_CERTIFICATES("withRootAndServerCertificates"), CLIENT_CERTITICATE("withClientCertificate"), NETWORK("withNetwork"), - ; - private final String methodName; + CUSTOM_NAME("withCustomName"); + + public final String methodName; - private ContainerModifier(String methodName) { + ContainerModifier(String methodName) { this.methodName = methodName; } @@ -73,20 +77,26 @@ public MySQLTestDatabase withoutStrictMode() { @Override protected Stream> inContainerBootstrapCmd() { - return Stream.of(mysqlCmd(Stream.of( - String.format("SET GLOBAL max_connections=%d", MAX_CONNECTIONS), - String.format("CREATE DATABASE %s", getDatabaseName()), - String.format("CREATE USER '%s' IDENTIFIED BY '%s'", getUserName(), getPassword()), - // Grant privileges also to the container's user, which is not root. - String.format("GRANT ALL PRIVILEGES ON *.* TO '%s', '%s' WITH GRANT OPTION", getUserName(), - getContainer().getUsername())))); + // Besides setting up user and privileges, we also need to create a soft link otherwise + // airbyte-ci on github runner would not be able to connect to DB, because the sock file does not + // exist. + return Stream.of(Stream.of( + "sh", "-c", "ln -s -f /var/lib/mysql/mysql.sock /var/run/mysqld/mysqld.sock"), + mysqlCmd(Stream.of( + String.format("SET GLOBAL max_connections=%d", MAX_CONNECTIONS), + String.format("CREATE DATABASE \\`%s\\`", getDatabaseName()), + String.format("CREATE USER '%s' IDENTIFIED BY '%s'", getUserName(), getPassword()), + // Grant privileges also to the container's user, which is not root. + String.format("GRANT ALL PRIVILEGES ON *.* TO '%s', '%s' WITH GRANT OPTION", getUserName(), + getContainer().getUsername())))); + } @Override protected Stream inContainerUndoBootstrapCmd() { return mysqlCmd(Stream.of( String.format("DROP USER '%s'", getUserName()), - String.format("DROP DATABASE %s", getDatabaseName()))); + String.format("DROP DATABASE \\`%s\\`", getDatabaseName()))); } @Override @@ -121,12 +131,17 @@ public MySQLConfigBuilder withStandardReplication() { } public MySQLConfigBuilder withCdcReplication() { + return withCdcReplication(RESYNC_DATA_OPTION); + } + + public MySQLConfigBuilder withCdcReplication(String cdcCursorFailBehaviour) { return this .with("is_test", true) .with("replication_method", ImmutableMap.builder() .put("method", "CDC") .put("initial_waiting_seconds", 5) .put("server_time_zone", "America/Los_Angeles") + .put(INVALID_CDC_CURSOR_POSITION_PROPERTY, cdcCursorFailBehaviour) .build()); } diff --git a/airbyte-integrations/connectors/source-n8n/main.py b/airbyte-integrations/connectors/source-n8n/main.py index 0762610cb5f1..b40f7c348752 100644 --- a/airbyte-integrations/connectors/source-n8n/main.py +++ b/airbyte-integrations/connectors/source-n8n/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_n8n import SourceN8n +from source_n8n.run import run if __name__ == "__main__": - source = SourceN8n() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-n8n/metadata.yaml b/airbyte-integrations/connectors/source-n8n/metadata.yaml index f34328d9ee75..afabe0ee24aa 100644 --- a/airbyte-integrations/connectors/source-n8n/metadata.yaml +++ b/airbyte-integrations/connectors/source-n8n/metadata.yaml @@ -8,6 +8,10 @@ data: icon: n8n.svg license: MIT name: n8n + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-n8n registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-n8n/setup.py b/airbyte-integrations/connectors/source-n8n/setup.py index 82cdff7f8589..3592a5b8939f 100644 --- a/airbyte-integrations/connectors/source-n8n/setup.py +++ b/airbyte-integrations/connectors/source-n8n/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-n8n=source_n8n.run:run", + ], + }, name="source_n8n", description="Source implementation for N8n.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-n8n/source_n8n/run.py b/airbyte-integrations/connectors/source-n8n/source_n8n/run.py new file mode 100644 index 000000000000..04f48f6c0696 --- /dev/null +++ b/airbyte-integrations/connectors/source-n8n/source_n8n/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_n8n import SourceN8n + + +def run(): + source = SourceN8n() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-nasa/main.py b/airbyte-integrations/connectors/source-nasa/main.py index c1627273e227..5887db860888 100644 --- a/airbyte-integrations/connectors/source-nasa/main.py +++ b/airbyte-integrations/connectors/source-nasa/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_nasa import SourceNasa +from source_nasa.run import run if __name__ == "__main__": - source = SourceNasa() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-nasa/metadata.yaml b/airbyte-integrations/connectors/source-nasa/metadata.yaml index f5fa8ac2e653..8bc0a54a45a0 100644 --- a/airbyte-integrations/connectors/source-nasa/metadata.yaml +++ b/airbyte-integrations/connectors/source-nasa/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.nasa.gov + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-nasa registries: oss: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/nasa tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py index ecd81e7b2fd4..849b81fce45d 100644 --- a/airbyte-integrations/connectors/source-nasa/setup.py +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-nasa=source_nasa.run:run", + ], + }, name="source_nasa", description="Source implementation for Nasa.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/run.py b/airbyte-integrations/connectors/source-nasa/source_nasa/run.py new file mode 100644 index 000000000000..6b14f0280427 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nasa import SourceNasa + + +def run(): + source = SourceNasa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-netsuite/main.py b/airbyte-integrations/connectors/source-netsuite/main.py index 7b88a055cfe2..492266da15e2 100644 --- a/airbyte-integrations/connectors/source-netsuite/main.py +++ b/airbyte-integrations/connectors/source-netsuite/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_netsuite import SourceNetsuite +from source_netsuite.run import run if __name__ == "__main__": - source = SourceNetsuite() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-netsuite/metadata.yaml b/airbyte-integrations/connectors/source-netsuite/metadata.yaml index c2451ba1bfa3..2ce3fb426c0c 100644 --- a/airbyte-integrations/connectors/source-netsuite/metadata.yaml +++ b/airbyte-integrations/connectors/source-netsuite/metadata.yaml @@ -12,6 +12,10 @@ data: icon: netsuite.svg license: MIT name: Netsuite + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-netsuite registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-netsuite/setup.py b/airbyte-integrations/connectors/source-netsuite/setup.py index 42288908347c..e16d4d5b270f 100644 --- a/airbyte-integrations/connectors/source-netsuite/setup.py +++ b/airbyte-integrations/connectors/source-netsuite/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-netsuite=source_netsuite.run:run", + ], + }, name="source_netsuite", description="Source implementation for Netsuite Soap.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py new file mode 100644 index 000000000000..c1a2043c455d --- /dev/null +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_netsuite import SourceNetsuite + + +def run(): + source = SourceNetsuite() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-news-api/main.py b/airbyte-integrations/connectors/source-news-api/main.py index 0b2c02a55247..835032115f02 100644 --- a/airbyte-integrations/connectors/source-news-api/main.py +++ b/airbyte-integrations/connectors/source-news-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_news_api import SourceNewsApi +from source_news_api.run import run if __name__ == "__main__": - source = SourceNewsApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-news-api/metadata.yaml b/airbyte-integrations/connectors/source-news-api/metadata.yaml index d33ea74b3d18..e4f2c6d7e45b 100644 --- a/airbyte-integrations/connectors/source-news-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-news-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: newsapi.svg license: MIT name: News API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-news-api registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-news-api/setup.py b/airbyte-integrations/connectors/source-news-api/setup.py index 733fc131e9ab..db15b71d90fa 100644 --- a/airbyte-integrations/connectors/source-news-api/setup.py +++ b/airbyte-integrations/connectors/source-news-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-news-api=source_news_api.run:run", + ], + }, name="source_news_api", description="Source implementation for News Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/run.py b/airbyte-integrations/connectors/source-news-api/source_news_api/run.py new file mode 100644 index 000000000000..188ce3f53c6f --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_news_api import SourceNewsApi + + +def run(): + source = SourceNewsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-newsdata/main.py b/airbyte-integrations/connectors/source-newsdata/main.py index 2f7c8b1ecbac..81dc024c6fb2 100644 --- a/airbyte-integrations/connectors/source-newsdata/main.py +++ b/airbyte-integrations/connectors/source-newsdata/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_newsdata import SourceNewsdata +from source_newsdata.run import run if __name__ == "__main__": - source = SourceNewsdata() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-newsdata/metadata.yaml b/airbyte-integrations/connectors/source-newsdata/metadata.yaml index e27f67670526..1ae1a6bb448c 100644 --- a/airbyte-integrations/connectors/source-newsdata/metadata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-newsdata license: MIT name: Newsdata + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-newsdata registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-newsdata/setup.py b/airbyte-integrations/connectors/source-newsdata/setup.py index a7bc19daa4ba..55b34a9fb708 100644 --- a/airbyte-integrations/connectors/source-newsdata/setup.py +++ b/airbyte-integrations/connectors/source-newsdata/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-newsdata=source_newsdata.run:run", + ], + }, name="source_newsdata", description="Source implementation for Newsdata.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py b/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py new file mode 100644 index 000000000000..2cdae722bfb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_newsdata import SourceNewsdata + + +def run(): + source = SourceNewsdata() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-notion/README.md b/airbyte-integrations/connectors/source-notion/README.md index 2be67fc963b2..13b0ba9f2b6b 100644 --- a/airbyte-integrations/connectors/source-notion/README.md +++ b/airbyte-integrations/connectors/source-notion/README.md @@ -1,119 +1,55 @@ -# Notion Source +# Notion source connector + This is the repository for the Notion source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/notion). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/notion). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/notion) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/notion) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source notion test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-notion spec +poetry run source-notion check --config secrets/config.json +poetry run source-notion discover --config secrets/config.json +poetry run source-notion read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-notion build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-notion:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-notion:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-notion:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-notion:dev . -# Running the spec command against your patched connector -docker run airbyte/source-notion:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-notion:dev spec @@ -122,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-notion:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-notion test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-notion test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/notion.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/notion.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 27bf2dc60ca8..9989f84792de 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ {"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} {"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} {"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} -{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1697023281967} -{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green"}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow"}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red"}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default"}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default"}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default"}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default"}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default"}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1697023281968} -{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red"}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue"}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1697023281968} +{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green", "description": null}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow", "description": null}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red", "description": null}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default", "description": null}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default", "description": null}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default", "description": null}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default", "description": null}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default", "description": null}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red", "description": null}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green", "description": null}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue", "description": null}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow", "description": null}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange", "description": null}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1708341487319} {"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} {"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} {"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} diff --git a/airbyte-integrations/connectors/source-notion/main.py b/airbyte-integrations/connectors/source-notion/main.py index dd188d547982..671d6cd692fa 100644 --- a/airbyte-integrations/connectors/source-notion/main.py +++ b/airbyte-integrations/connectors/source-notion/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_notion import SourceNotion +from source_notion.run import run if __name__ == "__main__": - source = SourceNotion() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 69f953d8a5e2..58b329ffe793 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 2.0.8 + dockerImageTag: 2.1.0 dockerRepository: airbyte/source-notion documentationUrl: https://docs.airbyte.com/integrations/sources/notion githubIssueLabel: source-notion icon: notion.svg license: MIT name: Notion + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-notion registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-notion/poetry.lock b/airbyte-integrations/connectors/source-notion/poetry.lock new file mode 100644 index 000000000000..deaacddf791a --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/poetry.lock @@ -0,0 +1,1032 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.52.7" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.52.7.tar.gz", hash = "sha256:73c55464ed57e030681fb4407613a5c0f07c519a4ba63aa9a4dd43d05cdf100b"}, + {file = "airbyte_cdk-0.52.7-py3-none-any.whl", hash = "sha256:f11665fc8f8dd2632d94e57f53991f7aaba8e9643a8ee7557f3040e40ea677ff"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "fde07907def42fe31b6513c3d374b49cd501667cb0368ed468012b58391fb29f" diff --git a/airbyte-integrations/connectors/source-notion/pyproject.toml b/airbyte-integrations/connectors/source-notion/pyproject.toml new file mode 100644 index 000000000000..3c4250763099 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.1.0" +name = "source-notion" +description = "Source implementation for Notion." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/notion" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_notion" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.52.7" +pendulum = "==2.1.2" + +[tool.poetry.scripts] +source-notion = "source_notion.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" +freezegun = "^1.4.0" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-notion/requirements.txt b/airbyte-integrations/connectors/source-notion/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-notion/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py deleted file mode 100644 index 80a1ed81533f..000000000000 --- a/airbyte-integrations/connectors/source-notion/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "pendulum==2.1.2", -] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "requests-mock", - "freezegun", -] - -setup( - name="source_notion", - description="Source implementation for Notion.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/run.py b/airbyte-integrations/connectors/source-notion/source_notion/run.py new file mode 100644 index 000000000000..df14df5ee9e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_notion import SourceNotion + + +def run(): + source = SourceNotion() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index cc003048b875..e8ecf832da80 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -32,6 +32,17 @@ "properties": { "type": { "type": ["null", "string"] + }, + "info": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + } + } } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index d893b1147b7e..8c5000dba205 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -37,10 +37,38 @@ "type": { "type": "string" }, + "info": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "person": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, "workspace": { "type": ["null", "boolean"] } } + }, + "workspace_name": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index a0546d2116c4..f7cb9e456e25 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -252,6 +252,22 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> params["start_cursor"] = next_page_token["next_cursor"] return params + def transform(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + owner = record.get("bot", {}).get("owner") + if owner: + owner_type = owner.get("type") + owner_info = owner.get(owner_type) + if owner_type and owner_info: + record["bot"]["owner"]["info"] = owner_info + del record["bot"]["owner"][owner_type] + return record + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # sometimes notion api returns response without results object + data = response.json().get("results", []) + for record in data: + yield self.transform(record) + class Databases(IncrementalNotionStream): """ @@ -313,6 +329,20 @@ def stream_slices( yield {"page_id": page_id} + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + transform_object_field = record.get("type") + + if transform_object_field: + rich_text = record.get(transform_object_field, {}).get("rich_text", []) + for r in rich_text: + mention = r.get("mention") + if mention: + type_info = mention[mention["type"]] + record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"]["info"] = type_info + del record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"][mention["type"]] + + return record + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: # pages and databases blocks are already fetched in their streams, so no # need to do it again @@ -321,7 +351,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, records = super().parse_response(response, stream_state=stream_state, **kwargs) for record in records: if record["type"] not in ("child_page", "child_database", "ai_block"): - yield record + yield self.transform(record) def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: # if reached recursive limit, don't read anymore diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index ceeb5b1ca5cb..d369b201becc 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -312,3 +312,41 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, stream.should_retry(response=response) assert stream.page_size == expected_page_size + + +def test_users_record_transformer(): + stream = Users(config=MagicMock()) + response_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + expected_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + assert stream.transform(response_record) == expected_record + + +def test_block_record_transformer(): + stream = Blocks(parent=None, config=MagicMock()) + response_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, + "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + } + expected_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], + "color": "default"} + } + assert stream.transform(response_record) == expected_record diff --git a/airbyte-integrations/connectors/source-nytimes/main.py b/airbyte-integrations/connectors/source-nytimes/main.py index b3e9d432ee28..ed513aa4248e 100644 --- a/airbyte-integrations/connectors/source-nytimes/main.py +++ b/airbyte-integrations/connectors/source-nytimes/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_nytimes import SourceNytimes +from source_nytimes.run import run if __name__ == "__main__": - source = SourceNytimes() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-nytimes/metadata.yaml b/airbyte-integrations/connectors/source-nytimes/metadata.yaml index fa67687b7293..abb5e9e358cc 100644 --- a/airbyte-integrations/connectors/source-nytimes/metadata.yaml +++ b/airbyte-integrations/connectors/source-nytimes/metadata.yaml @@ -8,6 +8,10 @@ data: icon: nytimes.svg license: MIT name: New York Times + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-nytimes registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-nytimes/setup.py b/airbyte-integrations/connectors/source-nytimes/setup.py index ae57a1144c9f..6c0cc1d179b5 100644 --- a/airbyte-integrations/connectors/source-nytimes/setup.py +++ b/airbyte-integrations/connectors/source-nytimes/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-nytimes=source_nytimes.run:run", + ], + }, name="source_nytimes", description="Source implementation for Nytimes.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py b/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py new file mode 100644 index 000000000000..ef4ddbd555e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nytimes import SourceNytimes + + +def run(): + source = SourceNytimes() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-okta/main.py b/airbyte-integrations/connectors/source-okta/main.py index 93c853b816f5..488cb6ca1b6f 100644 --- a/airbyte-integrations/connectors/source-okta/main.py +++ b/airbyte-integrations/connectors/source-okta/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_okta import SourceOkta +from source_okta.run import run if __name__ == "__main__": - source = SourceOkta() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-okta/metadata.yaml b/airbyte-integrations/connectors/source-okta/metadata.yaml index 011fb07a3b8b..2a7b9541606e 100644 --- a/airbyte-integrations/connectors/source-okta/metadata.yaml +++ b/airbyte-integrations/connectors/source-okta/metadata.yaml @@ -12,6 +12,10 @@ data: icon: okta.svg license: MIT name: Okta + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-okta registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-okta/setup.py b/airbyte-integrations/connectors/source-okta/setup.py index a3fac533b39e..1d2c69920b88 100644 --- a/airbyte-integrations/connectors/source-okta/setup.py +++ b/airbyte-integrations/connectors/source-okta/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-okta=source_okta.run:run", + ], + }, name="source_okta", description="Source implementation for Okta.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-okta/source_okta/run.py b/airbyte-integrations/connectors/source-okta/source_okta/run.py new file mode 100644 index 000000000000..f40c87dc0e37 --- /dev/null +++ b/airbyte-integrations/connectors/source-okta/source_okta/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_okta import SourceOkta + + +def run(): + source = SourceOkta() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-omnisend/main.py b/airbyte-integrations/connectors/source-omnisend/main.py index 01c8cb75e133..bf2d072e4c00 100644 --- a/airbyte-integrations/connectors/source-omnisend/main.py +++ b/airbyte-integrations/connectors/source-omnisend/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_omnisend import SourceOmnisend +from source_omnisend.run import run if __name__ == "__main__": - source = SourceOmnisend() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-omnisend/metadata.yaml b/airbyte-integrations/connectors/source-omnisend/metadata.yaml index 1f350fea734d..3b27557209fa 100644 --- a/airbyte-integrations/connectors/source-omnisend/metadata.yaml +++ b/airbyte-integrations/connectors/source-omnisend/metadata.yaml @@ -8,6 +8,10 @@ data: icon: omnisend.svg license: MIT name: Omnisend + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-omnisend registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-omnisend/setup.py b/airbyte-integrations/connectors/source-omnisend/setup.py index 653728e512b5..990506f80f9b 100644 --- a/airbyte-integrations/connectors/source-omnisend/setup.py +++ b/airbyte-integrations/connectors/source-omnisend/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-omnisend=source_omnisend.run:run", + ], + }, name="source_omnisend", description="Source implementation for Omnisend.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py b/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py new file mode 100644 index 000000000000..46479c299f0a --- /dev/null +++ b/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_omnisend import SourceOmnisend + + +def run(): + source = SourceOmnisend() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-onesignal/main.py b/airbyte-integrations/connectors/source-onesignal/main.py index ccff041b46e3..e735cf13bacb 100644 --- a/airbyte-integrations/connectors/source-onesignal/main.py +++ b/airbyte-integrations/connectors/source-onesignal/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_onesignal import SourceOnesignal +from source_onesignal.run import run if __name__ == "__main__": - source = SourceOnesignal() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-onesignal/metadata.yaml b/airbyte-integrations/connectors/source-onesignal/metadata.yaml index c63f53736d6d..1c57a5eebb55 100644 --- a/airbyte-integrations/connectors/source-onesignal/metadata.yaml +++ b/airbyte-integrations/connectors/source-onesignal/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "onesignal.com" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-onesignal registries: oss: enabled: true @@ -20,6 +24,6 @@ data: releaseDate: 2023-08-31 releaseStage: alpha tags: - - language:lowcode + - language:low-code supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-onesignal/setup.py b/airbyte-integrations/connectors/source-onesignal/setup.py index 47af71852c6a..aac6704b131b 100644 --- a/airbyte-integrations/connectors/source-onesignal/setup.py +++ b/airbyte-integrations/connectors/source-onesignal/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test"] setup( + entry_points={ + "console_scripts": [ + "source-onesignal=source_onesignal.run:run", + ], + }, name="source_onesignal", description="Source implementation for Onesignal.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py b/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py new file mode 100644 index 000000000000..e5690d762c47 --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_onesignal import SourceOnesignal + + +def run(): + source = SourceOnesignal() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/main.py b/airbyte-integrations/connectors/source-open-exchange-rates/main.py index b311cb841f61..6223dbf6da8f 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/main.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_open_exchange_rates import SourceOpenExchangeRates +from source_open_exchange_rates.run import run if __name__ == "__main__": - source = SourceOpenExchangeRates() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml index f93f96677127..4cce2373e1b4 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - openexchangerates.org + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-open-exchange-rates registries: oss: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/open-exchange-rates tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py index 55fd4589213f..4b12738562df 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-open-exchange-rates=source_open_exchange_rates.run:run", + ], + }, name="source_open_exchange_rates", description="Source implementation for Open Exchange Rates.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py new file mode 100644 index 000000000000..b2d83005c0ed --- /dev/null +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_open_exchange_rates import SourceOpenExchangeRates + + +def run(): + source = SourceOpenExchangeRates() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-openweather/main.py b/airbyte-integrations/connectors/source-openweather/main.py index 398948d54171..381fb38ae2dc 100644 --- a/airbyte-integrations/connectors/source-openweather/main.py +++ b/airbyte-integrations/connectors/source-openweather/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_openweather import SourceOpenweather +from source_openweather.run import run if __name__ == "__main__": - source = SourceOpenweather() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index 1e71bb97fc7f..e92518c4f92b 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.openweathermap.org + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-openweather registries: oss: enabled: true @@ -20,5 +24,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/openweather tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-openweather/setup.py b/airbyte-integrations/connectors/source-openweather/setup.py index fd32d0604449..2f978664b057 100644 --- a/airbyte-integrations/connectors/source-openweather/setup.py +++ b/airbyte-integrations/connectors/source-openweather/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-openweather=source_openweather.run:run", + ], + }, name="source_openweather", description="Source implementation for Openweather.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/run.py b/airbyte-integrations/connectors/source-openweather/source_openweather/run.py new file mode 100644 index 000000000000..f18939c72eb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_openweather import SourceOpenweather + + +def run(): + source = SourceOpenweather() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-opsgenie/Dockerfile b/airbyte-integrations/connectors/source-opsgenie/Dockerfile index b8cc08291b97..2e6395d06e2e 100644 --- a/airbyte-integrations/connectors/source-opsgenie/Dockerfile +++ b/airbyte-integrations/connectors/source-opsgenie/Dockerfile @@ -34,5 +34,5 @@ COPY source_opsgenie ./source_opsgenie ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-opsgenie diff --git a/airbyte-integrations/connectors/source-opsgenie/main.py b/airbyte-integrations/connectors/source-opsgenie/main.py index 0acd6dee136d..4f5798deffe1 100644 --- a/airbyte-integrations/connectors/source-opsgenie/main.py +++ b/airbyte-integrations/connectors/source-opsgenie/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_opsgenie import SourceOpsgenie +from source_opsgenie.run import run if __name__ == "__main__": - source = SourceOpsgenie() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml index 4af87df040a9..1c7f9af426a1 100644 --- a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml @@ -2,11 +2,15 @@ data: connectorSubtype: api connectorType: source definitionId: 06bdb480-2598-40b8-8b0f-fc2e2d2abdda - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-opsgenie githubIssueLabel: source-opsgenie license: MIT name: Opsgenie + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-opsgenie registries: cloud: enabled: false @@ -15,7 +19,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-opsgenie/setup.py b/airbyte-integrations/connectors/source-opsgenie/setup.py index 48fa326b15b9..4bf256910f2b 100644 --- a/airbyte-integrations/connectors/source-opsgenie/setup.py +++ b/airbyte-integrations/connectors/source-opsgenie/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-opsgenie=source_opsgenie.run:run", + ], + }, name="source_opsgenie", description="Source implementation for Opsgenie.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml index a1c6b1e07e72..83473e392ca4 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml @@ -107,6 +107,7 @@ definitions: cursor_field: updatedAt cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%S.%fZ" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%s" start_datetime: datetime: "{{ config['start_date'] }}" diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py new file mode 100644 index 000000000000..63e0eb89c264 --- /dev/null +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_opsgenie import SourceOpsgenie + + +def run(): + source = SourceOpsgenie() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index bfd298afacb1..000000000000 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-oracle-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle index 3878e876b4e9..a295c243cda4 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.oracle_strict_encrypt.OracleStrictEncryptSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -26,14 +16,7 @@ application { dependencies { implementation project(':airbyte-integrations:connectors:source-oracle') - // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) - implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - - implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.testcontainers.oracle.xe - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-oracle-strict-encrypt') + testImplementation 'org.testcontainers:oracle-xe:1.19.4' } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml index 1903adbfff84..e550b010941e 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.2 dockerRepository: airbyte/source-oracle-strict-encrypt githubIssueLabel: source-oracle icon: oracle.svg diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java index d48fe6ef6630..fa2cfdc59fbc 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java @@ -17,11 +17,16 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import java.sql.SQLException; +import java.time.Duration; import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class OracleSourceNneAcceptanceTest extends OracleStrictEncryptSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Test public void testEncryption() throws SQLException { final ObjectNode clone = (ObjectNode) Jsons.clone(getConfig()); @@ -44,7 +49,8 @@ public void testEncryption() throws SQLException { clone.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -75,7 +81,8 @@ public void testCheckProtocol() throws SQLException { clone.get(JdbcUtils.PORT_KEY).asInt(), clone.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";"))); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java index ae4d1645d20e..64c45075348a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java @@ -14,31 +14,30 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.ssh.SshHelpers; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.source.oracle.OracleSource; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -47,18 +46,24 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class OracleStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class OracleStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(OracleStrictEncryptJdbcSourceAcceptanceTest.class); - private static AirbyteOracleTestContainer ORACLE_DB; + private static final AirbyteOracleTestContainer ORACLE_DB = new AirbyteOracleTestContainer() + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") + .withEnv("RELAX_SECURITY", "1") + .withUsername("TEST_ORA") + .withPassword("oracle") + .usingSid() + .withEnv("RELAX_SECURITY", "1"); @BeforeAll static void init() { @@ -88,62 +93,21 @@ static void init() { CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20))"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)')"; INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS'))"; - - ORACLE_DB = new AirbyteOracleTestContainer() - .withUsername("test") - .withPassword("oracle") - .usingSid() - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") - .withEnv("RELAX_SECURITY", "1"); - ORACLE_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", ORACLE_DB.getHost()) - .put("port", ORACLE_DB.getFirstMappedPort()) - .put("sid", ORACLE_DB.getSid()) - .put("username", ORACLE_DB.getUsername()) - .put("password", ORACLE_DB.getPassword()) - .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "client_nne") - .put("encryption_algorithm", "3DES168") - .build())) - .build()); - - // Because Oracle doesn't let me create database easily I need to clean up - cleanUpTables(); - - super.setup(); - } - - @AfterEach - public void tearDownOracle() throws Exception { - // ORA-12519 - // https://stackoverflow.com/questions/205160/what-can-cause-intermittent-ora-12519-tns-no-appropriate-handler-found-errors - // sleep for 1000 - executeOracleStatement(String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - Thread.sleep(1000); } + @Override protected void incrementalDateCheck() throws Exception { // https://stackoverflow.com/questions/47712930/resultset-meta-data-return-timestamp-instead-of-date-oracle-jdbc // Oracle DATE is a java.sql.Timestamp (java.sql.Types.TIMESTAMP) as far as JDBC (and the SQL // standard) is concerned as it has both a date and time component. incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } - void cleanUpTables() throws SQLException { + static void cleanUpTables() throws SQLException { final Connection connection = DriverManager.getConnection( ORACLE_DB.getJdbcUrl(), ORACLE_DB.getUsername(), @@ -164,29 +128,25 @@ void cleanUpTables() throws SQLException { } @Override - public boolean supportsSchemas() { - // See https://www.oratable.com/oracle-user-schema-difference/ - return true; + protected OracleStrictEncryptTestDatabase createTestDatabase() { + ORACLE_DB.start(); + return new OracleStrictEncryptTestDatabase(ORACLE_DB, List.of(SCHEMA_NAME, SCHEMA_NAME2)).initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { - return new OracleSource(); + public boolean supportsSchemas() { + // See https://www.oratable.com/oracle-user-schema-difference/ + return true; } @Override - public Source getSource() { + protected OracleStrictEncryptSource source() { return new OracleStrictEncryptSource(); } @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return OracleSource.DRIVER_CLASS; + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @AfterAll @@ -195,7 +155,7 @@ static void cleanUp() { } @Override - public void createSchemas() throws SQLException { + public void createSchemas() { // In Oracle, `CREATE USER` creates a schema. // See https://www.oratable.com/oracle-user-schema-difference/ if (supportsSchemas()) { @@ -208,9 +168,13 @@ public void createSchemas() throws SQLException { } } - @Override - protected String getJdbcParameterDelimiter() { - return ";"; + static void cleanUpTablesAndWait() { + try { + cleanUpTables(); + Thread.sleep(1000); + } catch (final Exception e) { + throw new RuntimeException(e); + } } public void executeOracleStatement(final String query) { @@ -267,12 +231,13 @@ public static boolean ignoreSQLException(final String sqlState) { @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); assertEquals(expected, actual); } + @Override protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( @@ -305,32 +270,33 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List getTestMessages() { return Lists.newArrayList( - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19T00:00:00.000000Z")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + COL_UPDATED_AT, "2004-10-19T00:00:00.000000")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, - "2005-10-19T00:00:00.000000Z")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + "2005-10-19T00:00:00.000000")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -340,50 +306,54 @@ void testReadOneTableIncrementallyTwice() throws Exception { }); final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() - .filter(r -> r.getType() == AirbyteMessage.Type.STATE).findFirst(); + .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); Assertions.assertEquals(2, - (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == AirbyteMessage.Type.RECORD).count()); + (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage() - .withType(AirbyteMessage.Type.STATE) + .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamNamespace(namespace) + .withStreamName(streamName()) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L)))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -400,8 +370,8 @@ void testReadOneTableIncrementallyTwice() throws Exception { void testIncrementalTimestampCheckCursor() throws Exception { incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java index 5469b7e2b713..0fc32d0c865a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java @@ -24,14 +24,18 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; +@Disabled public class OracleStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); protected AirbyteOracleTestContainer container; protected JsonNode config; @@ -66,7 +70,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.PORT_KEY).asInt(), config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( 3DES168 )", ";")); + "oracle.net.encryption_types_client=( 3DES168 )", ";"), + CONNECTION_TIMEOUT); try { final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java new file mode 100644 index 000000000000..662bcd5868fa --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.oracle_strict_encrypt; + +import static io.airbyte.integrations.source.oracle_strict_encrypt.OracleStrictEncryptJdbcSourceAcceptanceTest.cleanUpTablesAndWait; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class OracleStrictEncryptTestDatabase extends + TestDatabase { + + private final AirbyteOracleTestContainer container; + private final List schemaNames; + + protected OracleStrictEncryptTestDatabase(final AirbyteOracleTestContainer container, final List schemaNames) { + super(container); + this.container = container; + this.schemaNames = schemaNames; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.ORACLE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public OracleStrictEncryptDbConfigBuilder configBuilder() { + return new OracleStrictEncryptDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("sid", container.getSid()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SCHEMAS_KEY, schemaNames) + .with(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "3DES168") + .build())); + } + + @Override + public void close() { + cleanUpTablesAndWait(); + } + + static public class OracleStrictEncryptDbConfigBuilder extends ConfigBuilder { + + protected OracleStrictEncryptDbConfigBuilder(final OracleStrictEncryptTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml b/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml deleted file mode 100644 index 1dad4c01150e..000000000000 --- a/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-oracle:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" - timeout_seconds: 300 diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 35abccdd41fa..0eb2700d9a30 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -1,39 +1,20 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.oracle.OracleSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - - // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) - implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - - implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.testcontainers.oracle.xe + testImplementation 'org.testcontainers:oracle-xe:1.19.4' } diff --git a/airbyte-integrations/connectors/source-oracle/metadata.yaml b/airbyte-integrations/connectors/source-oracle/metadata.yaml index c12f2a06ba72..56a6d42d4ff3 100644 --- a/airbyte-integrations/connectors/source-oracle/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.2 dockerRepository: airbyte/source-oracle documentationUrl: https://docs.airbyte.com/integrations/sources/oracle githubIssueLabel: source-oracle @@ -18,7 +18,7 @@ data: name: Oracle DB registries: cloud: - dockerImageTag: 0.3.17 + dockerImageTag: 0.5.2 dockerRepository: airbyte/source-oracle-strict-encrypt enabled: true oss: diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 43193f980687..7223b6e9f37c 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -164,7 +164,7 @@ private static void convertAndImportCertificate(final String certificate) throws } private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { - final Process pr = run.exec(cmd); + final Process pr = run.exec(cmd.split(" ")); if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index b2df271938ab..501bf107aed6 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -28,8 +28,10 @@ import java.util.List; import java.util.Objects; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Network; +@Disabled public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index e0296b4c428e..8045f1c89e89 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; @@ -32,15 +31,16 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -49,20 +49,26 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class OracleJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class OracleJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(OracleJdbcSourceAcceptanceTest.class); protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - private static AirbyteOracleTestContainer ORACLE_DB; + private static final AirbyteOracleTestContainer ORACLE_DB = new AirbyteOracleTestContainer() + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") + .withEnv("RELAX_SECURITY", "1") + .withUsername("TEST_ORA") + .withPassword("oracle") + .usingSid() + .withEnv("RELAX_SECURITY", "1"); @BeforeAll static void init() { @@ -93,34 +99,59 @@ static void init() { CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20))"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)')"; INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS'))"; + } + + @AfterAll + static void cleanUp() { + ORACLE_DB.close(); + } + + @Override + public boolean supportsSchemas() { + // See https://www.oratable.com/oracle-user-schema-difference/ + return true; + } + + @Override + protected OracleSource source() { + return new OracleSource(); + } - ORACLE_DB = new AirbyteOracleTestContainer() - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") - .withEnv("RELAX_SECURITY", "1") - .withUsername("TEST_ORA") - .withPassword("oracle") - .usingSid() - .withEnv("RELAX_SECURITY", "1"); + @Override + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); + } + + @Override + protected OracleTestDatabase createTestDatabase() { ORACLE_DB.start(); + return new OracleTestDatabase(ORACLE_DB, List.of(SCHEMA_NAME, SCHEMA_NAME2)).initialized(); } - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", ORACLE_DB.getHost()) - .put("port", ORACLE_DB.getFirstMappedPort()) - .put("sid", ORACLE_DB.getSid()) - .put("username", ORACLE_DB.getUsername()) - .put("password", ORACLE_DB.getPassword()) - .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .build()); - - // Because Oracle doesn't let me create database easily I need to clean up - cleanUpTables(); - - super.setup(); + @Override + public void createSchemas() { + // In Oracle, `CREATE USER` creates a schema. + // See https://www.oratable.com/oracle-user-schema-difference/ + if (supportsSchemas()) { + for (final String schemaName : TEST_SCHEMAS) { + executeOracleStatement( + String.format( + "CREATE USER %s IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS", + schemaName)); + } + } } + static void cleanUpTablesAndWait() { + try { + cleanUpTables(); + Thread.sleep(1000); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override protected void incrementalDateCheck() throws Exception { // https://stackoverflow.com/questions/47712930/resultset-meta-data-return-timestamp-instead-of-date-oracle-jdbc // Oracle DATE is a java.sql.Timestamp (java.sql.Types.TIMESTAMP) as far as JDBC (and the SQL @@ -132,6 +163,7 @@ protected void incrementalDateCheck() throws Exception { Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } + @Override protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( @@ -161,20 +193,7 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } - @AfterEach - public void tearDownOracle() throws Exception { - // ORA-12519 - // https://stackoverflow.com/questions/205160/what-can-cause-intermittent-ora-12519-tns-no-appropriate-handler-found-errors - // sleep for 1000 - executeOracleStatement(String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - Thread.sleep(1000); - } - - void cleanUpTables() throws SQLException { + static void cleanUpTables() throws SQLException { final Connection connection = DriverManager.getConnection( ORACLE_DB.getJdbcUrl(), ORACLE_DB.getUsername(), @@ -197,14 +216,14 @@ void cleanUpTables() throws SQLException { protected List getTestMessages() { return Lists.newArrayList( new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", COL_UPDATED_AT, "2004-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_2, @@ -212,7 +231,7 @@ protected List getTestMessages() { COL_UPDATED_AT, "2005-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, @@ -230,7 +249,8 @@ void testIncrementalTimestampCheckCursor() throws Exception { } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -240,38 +260,34 @@ void testReadOneTableIncrementallyTwice() throws Exception { }); final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); Assertions.assertEquals(2, (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", @@ -279,11 +295,19 @@ void testReadOneTableIncrementallyTwice() throws Exception { expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateType.LEGACY) + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamNamespace(namespace) + .withStreamName(streamName()) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L)))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -296,46 +320,6 @@ void testReadOneTableIncrementallyTwice() throws Exception { assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); } - @Override - public boolean supportsSchemas() { - // See https://www.oratable.com/oracle-user-schema-difference/ - return true; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - return new OracleSource(); - } - - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return OracleSource.DRIVER_CLASS; - } - - @AfterAll - static void cleanUp() { - ORACLE_DB.close(); - } - - @Override - public void createSchemas() throws SQLException { - // In Oracle, `CREATE USER` creates a schema. - // See https://www.oratable.com/oracle-user-schema-difference/ - if (supportsSchemas()) { - for (final String schemaName : TEST_SCHEMAS) { - executeOracleStatement( - String.format( - "CREATE USER %s IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS", - schemaName)); - } - } - } - public void executeOracleStatement(final String query) { try ( final Connection connection = DriverManager.getConnection( @@ -392,7 +376,7 @@ public static boolean ignoreSQLException(final String sqlState) { @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); assertEquals(expected, actual); @@ -402,10 +386,11 @@ void testSpec() throws Exception { void testCheckIncorrectPasswordFailure() throws Exception { // by using a fake password oracle can block user account so we will create separate account for // this test + final JsonNode config = config(); executeOracleStatement(String.format("CREATE USER locked_user IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS")); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "locked_user"); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertEquals("State code: 72000; Error code: 1017; Message: ORA-01017: invalid username/password; logon denied\n", @@ -414,34 +399,38 @@ void testCheckIncorrectPasswordFailure() throws Exception { @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 72000; Error code: 1017;")); } @Test public void testCheckIncorrectHostFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08006; Error code: 17002;")); } @Test public void testCheckIncorrectPortFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.PORT_KEY, "0000"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08006; Error code: 17002;")); } @Test public void testUserHasNoPermissionToDataBase() throws Exception { + final JsonNode config = config(); executeOracleStatement(String.format("CREATE USER %s IDENTIFIED BY %s", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION)); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 72000; Error code: 1045;")); } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index b90420e4fbee..73c9d9081e30 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -25,7 +25,9 @@ import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; +@Disabled public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 03f912494cf0..5f708dfe2dd7 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -23,9 +23,11 @@ import java.util.List; import java.util.TimeZone; import org.jooq.DSLContext; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled public class OracleSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private AirbyteOracleTestContainer container; @@ -88,7 +90,6 @@ protected String getNameSpace() { @Override protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); container.close(); } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java index 6d77435b437e..822ac556650b 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -17,11 +17,16 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import java.sql.SQLException; +import java.time.Duration; import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Test public void testEncrytion() throws SQLException { final JsonNode clone = Jsons.clone(getConfig()); @@ -44,7 +49,8 @@ public void testEncrytion() throws SQLException { clone.get("connection_data").get("service_name").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -95,7 +101,8 @@ public void testCheckProtocol() throws SQLException { config.get("connection_data").get("service_name").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index dca0c20c73f6..fb8de2ff61bc 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -35,8 +35,10 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled class OracleSourceTest { private static final String STREAM_NAME = "TEST.ID_AND_NAME"; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java new file mode 100644 index 000000000000..5197bd2baf06 --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.oracle; + +import static io.airbyte.integrations.source.oracle.OracleJdbcSourceAcceptanceTest.cleanUpTablesAndWait; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class OracleTestDatabase extends + TestDatabase { + + private final AirbyteOracleTestContainer container; + private final List schemaNames; + + protected OracleTestDatabase(final AirbyteOracleTestContainer container, final List schemaNames) { + super(container); + this.container = container; + this.schemaNames = schemaNames; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.ORACLE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public OracleDbConfigBuilder configBuilder() { + return new OracleDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("sid", container.getSid()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SCHEMAS_KEY, schemaNames); + } + + @Override + public void close() { + cleanUpTablesAndWait(); + } + + static public class OracleDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected OracleDbConfigBuilder(final OracleTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java index a7d5dfb8eec8..cc53839eb1d1 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.source.oracle; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshKeyOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java index 175969f8a75d..44c79902a7b3 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.source.oracle; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshPasswordOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-orb/main.py b/airbyte-integrations/connectors/source-orb/main.py index 08c8985a056a..725dc3553298 100644 --- a/airbyte-integrations/connectors/source-orb/main.py +++ b/airbyte-integrations/connectors/source-orb/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_orb import SourceOrb +from source_orb.run import run if __name__ == "__main__": - source = SourceOrb() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-orb/metadata.yaml b/airbyte-integrations/connectors/source-orb/metadata.yaml index 16fbdbb5ab0a..1c5c21ce8624 100644 --- a/airbyte-integrations/connectors/source-orb/metadata.yaml +++ b/airbyte-integrations/connectors/source-orb/metadata.yaml @@ -8,6 +8,10 @@ data: icon: orb.svg license: MIT name: Orb + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-orb registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-orb/setup.py b/airbyte-integrations/connectors/source-orb/setup.py index 38e64fa19ee1..da07d9e21ecd 100644 --- a/airbyte-integrations/connectors/source-orb/setup.py +++ b/airbyte-integrations/connectors/source-orb/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.13.3", "pendulum==2.1.2"] setup( + entry_points={ + "console_scripts": [ + "source-orb=source_orb.run:run", + ], + }, name="source_orb", description="Source implementation for Orb.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-orb/source_orb/run.py b/airbyte-integrations/connectors/source-orb/source_orb/run.py new file mode 100644 index 000000000000..05f19468aac9 --- /dev/null +++ b/airbyte-integrations/connectors/source-orb/source_orb/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_orb import SourceOrb + + +def run(): + source = SourceOrb() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-orbit/main.py b/airbyte-integrations/connectors/source-orbit/main.py index 82e25dd0caa5..16b6891a52b6 100644 --- a/airbyte-integrations/connectors/source-orbit/main.py +++ b/airbyte-integrations/connectors/source-orbit/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_orbit import SourceOrbit +from source_orbit.run import run if __name__ == "__main__": - source = SourceOrbit() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-orbit/metadata.yaml b/airbyte-integrations/connectors/source-orbit/metadata.yaml index 2d19e453ccdc..c4ce50c056f2 100644 --- a/airbyte-integrations/connectors/source-orbit/metadata.yaml +++ b/airbyte-integrations/connectors/source-orbit/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-orbit registries: oss: enabled: true @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/orbit tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-orbit/setup.py b/airbyte-integrations/connectors/source-orbit/setup.py index ae8d7e9b1c8b..9ca2a1394b2a 100644 --- a/airbyte-integrations/connectors/source-orbit/setup.py +++ b/airbyte-integrations/connectors/source-orbit/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-orbit=source_orbit.run:run", + ], + }, name="source_orbit", description="Source implementation for Orbit.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/run.py b/airbyte-integrations/connectors/source-orbit/source_orbit/run.py new file mode 100644 index 000000000000..593cfb9a326a --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_orbit import SourceOrbit + + +def run(): + source = SourceOrbit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-oura/main.py b/airbyte-integrations/connectors/source-oura/main.py index 35f15399d958..f72173126bd0 100644 --- a/airbyte-integrations/connectors/source-oura/main.py +++ b/airbyte-integrations/connectors/source-oura/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_oura import SourceOura +from source_oura.run import run if __name__ == "__main__": - source = SourceOura() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-oura/metadata.yaml b/airbyte-integrations/connectors/source-oura/metadata.yaml index 01992bf8de55..a8279f08a933 100644 --- a/airbyte-integrations/connectors/source-oura/metadata.yaml +++ b/airbyte-integrations/connectors/source-oura/metadata.yaml @@ -8,6 +8,10 @@ data: icon: oura.svg license: MIT name: Oura + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-oura registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-oura/setup.py b/airbyte-integrations/connectors/source-oura/setup.py index b78b256c87fb..36e29e32605c 100644 --- a/airbyte-integrations/connectors/source-oura/setup.py +++ b/airbyte-integrations/connectors/source-oura/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-oura=source_oura.run:run", + ], + }, name="source_oura", description="Source implementation for Oura.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-oura/source_oura/run.py b/airbyte-integrations/connectors/source-oura/source_oura/run.py new file mode 100644 index 000000000000..af46311895fb --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_oura import SourceOura + + +def run(): + source = SourceOura() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/main.py b/airbyte-integrations/connectors/source-outbrain-amplify/main.py index 2f2acbc0b627..dc58d39d5bcd 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/main.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_outbrain_amplify import SourceOutbrainAmplify +from source_outbrain_amplify.run import run if __name__ == "__main__": - source = SourceOutbrainAmplify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml index 502c3d43ef58..06fd36aa0e7d 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml +++ b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml @@ -1,4 +1,8 @@ data: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-outbrain-amplify registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/setup.py b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py index d8b5563f3dce..f4f709b99a49 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/setup.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-outbrain-amplify=source_outbrain_amplify.run:run", + ], + }, name="source_outbrain_amplify", description="Source implementation for Outbrain Amplify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py new file mode 100644 index 000000000000..8f90ae8e0469 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outbrain_amplify import SourceOutbrainAmplify + + +def run(): + source = SourceOutbrainAmplify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outreach/main.py b/airbyte-integrations/connectors/source-outreach/main.py index 77394c375d03..9ad460e89200 100644 --- a/airbyte-integrations/connectors/source-outreach/main.py +++ b/airbyte-integrations/connectors/source-outreach/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_outreach import SourceOutreach +from source_outreach.run import run if __name__ == "__main__": - source = SourceOutreach() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-outreach/metadata.yaml b/airbyte-integrations/connectors/source-outreach/metadata.yaml index 30fa65743964..3e5d2a105c5e 100644 --- a/airbyte-integrations/connectors/source-outreach/metadata.yaml +++ b/airbyte-integrations/connectors/source-outreach/metadata.yaml @@ -12,6 +12,10 @@ data: icon: outreach.svg license: MIT name: Outreach + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-outreach registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-outreach/setup.py b/airbyte-integrations/connectors/source-outreach/setup.py index 2367148b8d73..87c10aedcb88 100644 --- a/airbyte-integrations/connectors/source-outreach/setup.py +++ b/airbyte-integrations/connectors/source-outreach/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-outreach=source_outreach.run:run", + ], + }, name="source_outreach", description="Source implementation for Outreach.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py new file mode 100644 index 000000000000..2d48fe0806cd --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outreach import SourceOutreach + + +def run(): + source = SourceOutreach() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pagerduty/main.py b/airbyte-integrations/connectors/source-pagerduty/main.py index 61d193268f6b..22537946cc8c 100644 --- a/airbyte-integrations/connectors/source-pagerduty/main.py +++ b/airbyte-integrations/connectors/source-pagerduty/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pagerduty import SourcePagerduty +from source_pagerduty.run import run if __name__ == "__main__": - source = SourcePagerduty() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pagerduty/metadata.yaml b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml index 403e4bd393d7..1d1bce47b87e 100644 --- a/airbyte-integrations/connectors/source-pagerduty/metadata.yaml +++ b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.pagerduty.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pagerduty registries: oss: enabled: false @@ -21,7 +25,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/pagerduty tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-pagerduty/setup.py b/airbyte-integrations/connectors/source-pagerduty/setup.py index aec396ac036d..cba7caa9d5bd 100644 --- a/airbyte-integrations/connectors/source-pagerduty/setup.py +++ b/airbyte-integrations/connectors/source-pagerduty/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pagerduty=source_pagerduty.run:run", + ], + }, name="source_pagerduty", description="Source implementation for Pagerduty.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py new file mode 100644 index 000000000000..f73afe9c6592 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pagerduty import SourcePagerduty + + +def run(): + source = SourcePagerduty() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pardot/main.py b/airbyte-integrations/connectors/source-pardot/main.py index 8158c5fc7d66..6112e4e13e27 100644 --- a/airbyte-integrations/connectors/source-pardot/main.py +++ b/airbyte-integrations/connectors/source-pardot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pardot import SourcePardot +from source_pardot.run import run if __name__ == "__main__": - source = SourcePardot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pardot/metadata.yaml b/airbyte-integrations/connectors/source-pardot/metadata.yaml index 6a0ab9f2d1ff..6bc3843e5f5c 100644 --- a/airbyte-integrations/connectors/source-pardot/metadata.yaml +++ b/airbyte-integrations/connectors/source-pardot/metadata.yaml @@ -8,6 +8,10 @@ data: icon: salesforcepardot.svg license: MIT name: Pardot + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pardot registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-pardot/pardot.md b/airbyte-integrations/connectors/source-pardot/pardot.md index e877c48a103a..bb3e4470173a 100644 --- a/airbyte-integrations/connectors/source-pardot/pardot.md +++ b/airbyte-integrations/connectors/source-pardot/pardot.md @@ -53,4 +53,4 @@ The Pardot connector should not run into Pardot API limitations under normal usa - `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce - `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. - `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -- `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. +- `is_sandbox`: Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. diff --git a/airbyte-integrations/connectors/source-pardot/setup.py b/airbyte-integrations/connectors/source-pardot/setup.py index c04454b0bd05..da875f7e2057 100644 --- a/airbyte-integrations/connectors/source-pardot/setup.py +++ b/airbyte-integrations/connectors/source-pardot/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pardot=source_pardot.run:run", + ], + }, name="source_pardot", description="Source implementation for Pardot.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pardot/source_pardot/run.py b/airbyte-integrations/connectors/source-pardot/source_pardot/run.py new file mode 100644 index 000000000000..34bd6425a06d --- /dev/null +++ b/airbyte-integrations/connectors/source-pardot/source_pardot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pardot import SourcePardot + + +def run(): + source = SourcePardot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-partnerstack/main.py b/airbyte-integrations/connectors/source-partnerstack/main.py index 24bd01814100..d22642a3ee66 100644 --- a/airbyte-integrations/connectors/source-partnerstack/main.py +++ b/airbyte-integrations/connectors/source-partnerstack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_partnerstack import SourcePartnerstack +from source_partnerstack.run import run if __name__ == "__main__": - source = SourcePartnerstack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml index 726c61a4f52d..cf6483ba4899 100644 --- a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml @@ -8,6 +8,10 @@ data: icon: partnerstack.svg license: MIT name: PartnerStack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-partnerstack registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-partnerstack/setup.py b/airbyte-integrations/connectors/source-partnerstack/setup.py index 5e6542bbc4d6..4d13b3c870f8 100644 --- a/airbyte-integrations/connectors/source-partnerstack/setup.py +++ b/airbyte-integrations/connectors/source-partnerstack/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-partnerstack=source_partnerstack.run:run", + ], + }, name="source_partnerstack", description="Source implementation for Partnerstack.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py new file mode 100644 index 000000000000..1b7dad130b55 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_partnerstack import SourcePartnerstack + + +def run(): + source = SourcePartnerstack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/.coveragerc b/airbyte-integrations/connectors/source-paypal-transaction/.coveragerc new file mode 100644 index 000000000000..4e4eba3bda57 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_paypal_transaction/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md index d84557504900..7cddffd5f108 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md +++ b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md @@ -2,3 +2,12 @@ ## 0.1.0 Source implementation with support of Transactions and Balances streams + +## 1.0.0 +Mark Client ID and Client Secret as required files + +## 2.1.0 +Migration to Low code + +## 2.3.0 +Adding New Streams - Payments, Disputes, Invoices, Product Catalog \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md index a451e1a48d23..6cff7acce3aa 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/README.md +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -1,32 +1,66 @@ -# Paypal Transaction Source +# Paypal-Transaction source connector -This is the repository for the Paypal Transaction configuration based source connector. +This is the repository for the Paypal-Transaction source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/paypal-transaction). ## Local development -#### Create credentials + +#### Prerequisites + * Python (~=3.9) + * Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) + * Paypal Client ID and Client Secret + * If you are going to use the data generator scripts you need to setup yourPaypal Sandbox and a Buyer user in your sandbox, to simulate the data. YOu cna get that information in the [Apps & Credentials page](https://developer.paypal.com/dashboard/applications/live). + * Buyer Username + * Buyer Password + * Payer ID (Account ID) + +### Installing the connector + +From this connector directory, run: +```bash +poetry install --with dev +``` + +### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/paypal-transaction) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. + +* You must have created your credentials under the `secrets/` folder +* For the read command, you can create separate catalogs to test the streams individually. All catalogs are under the folder `integration_tests`. Select the one you want to test with the read command. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Locally running the connector + +``` +poetry run source-paypal-transaction spec +poetry run source-paypal-transaction check --config secrets/config.json +poetry run source-paypal-transaction discover --config secrets/config.json +#Example with list_payments catalog and the debug flag +poetry run source-paypal-transaction read --config secrets/config.json --catalog integration_tests/configured_catalog_list_payments.json --debug +``` + +### Running unit tests +To run unit tests locally, from the connector directory run: + +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Installing the connector +From this connector directory, run: ```bash -airbyte-ci connectors --name=source-paypal-transaction build +poetry install --with dev ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-paypal-transaction:dev`. ##### Customizing our build process When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. @@ -44,74 +78,136 @@ if TYPE_CHECKING: # Feel free to check the dagger documentation for more information on the Container object and its methods. # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: +An image will be available on your host with the tag `airbyte/source-paypal-transaction:dev`. -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-paypal-transaction:latest -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-paypal-transaction:dev . -# Running the spec command against your patched connector -docker run airbyte/source-paypal-transaction:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-paypal-transaction:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config_oauth.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config_oauth.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config_oauth.json --catalog /integration_tests/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing + +### Running our CI test suite + You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-paypal-transaction test ``` +If you are testing locally, you can use your local credentials (config.json file) by using `--use-local-secrets` + +```bash +airbyte-ci connectors --name source-paypal-transaction --use-local-secrets test +``` + ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Running Unit tests locally + +To run unit tests locally, form the root `source_paypal_transaction` directory run: + +```bash +python -m pytest unit_test +``` + +## Test changes in the sandbox + +If you have a [Paypal Sandbox](https://developer.paypal.com/tools/sandbox/accounts/) you will be able to use some APIs to create new data and test how data is being created in your destinaiton and choose the best syn strategy that suits better your use case. +Some endpoints will require special permissions on the sandbox to update and change some values. + +In the `bin` folder you will find several data generator scripts: + +* **disputes_generator.py:** + * Update dispute: Uses the _PATCH_ method of the `https://api-m.paypal.com/v1/customer/disputes/{dispute_id}` endpoint. You need the ID and create a payload to pass it as an argument. See more information [here](https://developer.paypal.com/docs/api/customer-disputes/v1/#disputes_patch). + + ```bash + python disputes_generator.py update DISPUTE_ID ''[{"op": "replace", "path": "/reason", "value": "The new reason"}]' + ``` + + * Update Evidence status: Uses the _POST_ method of the `https://api-m.paypal.com/v1/customer/disputes/{dispute_id}/require-evidence` endpoint. You need the ID and select an option to pass it as an argument. See more information [here](https://developer.paypal.com/docs/api/customer-disputes/v1/#disputes_require-evidence) + ```bash + python update_dispute.py require-evidence DISPUTE_ID SELLER_EVIDENCE + ``` + +* **invoices.py:** + * Create draft invoice: Uses the _POST_ method of the `https://api-m.sandbox.paypal.com/v2/invoicing/invoices` endpoint. It will automatically generate an invoice (no need to pass any parameters). See more information [here](https://developer.paypal.com/docs/api/invoicing/v2/#invoices_create). + + ```bash + python invoices.py create_draft + ``` + + * Send a Draft Invoice: Uses the _POST_ method of the `https://api-m.sandbox.paypal.com/v2/invoicing/invoices/{invoice_id}/send` endpoint. You need the Invoice ID, a subject and a note (just to have something to update) and an email as an argument. See more information [here](https://developer.paypal.com/docs/api/invoicing/v2/#invoices_send) + ```bash + python invoices.py send_draft --invoice_id "INV2-XXXX-XXXX-XXXX-XXXX" --subject "Your Invoice Subject" --note "Your custom note" --additional_recipients example@email.com + ``` + +* **payments_generator.py:** + * Partially update payment: Uses the _PATCH_ method of the `https://api-m.paypal.com/v1/payments/payment/{payment_id}` endpoint. You need the payment ID and a payload with new values. (no need to pass any parameters). See more information [here](https://developer.paypal.com/docs/api/invoicing/v2/#invoices_create). + + ```bash + python script_name.py update PAYMENT_ID '[{"op": "replace", "path": "/transactions/0/amount", "value": {"total": "50.00", "currency": "USD"}}]' + ``` + +* **paypal_transaction_generator.py:** + Make sure you have the `buyer_username`, `buyer_password` and `payer_id` in your config file. You can get the sample configuratin in the `sample_config.json`. + + * Generate transactions: This uses Selenium, so you will be prompted to your account to simulate the complete transaction flow. You can add a number at the end of the command to do more than one transaction. By default the script runs 3 transactions. + + **NOTE: Be midnfu of the number of transactions, as it will be interacting with your machine, and you may not be able to use it while creating the transactions** + + ```bash + python paypal_transaction_generator.py [NUMBER_OF_DESIRED_TRANSACTIONS] + ``` + +* **product_catalog.py:** + * Create a product: Uses the _POST_ method of the `https://api-m.sandbox.paypal.com/v1/catalogs/products` endpoint. You need to add the description and the category in the command line. For the proper category see more information [here](https://developer.paypal.com/docs/api/catalog-products/v1/#products_create). + + ```bash + python product_catalog.py --action create --description "YOUR DESCRIPTION" --category PAYPAL_CATEGORY + ``` + + * Update a product: Uses the _PATCH_ method of the `https://developer.paypal.com/docs/api/catalog-products/v1/#products_patch` endpoint. You need the product ID, a description and the Category as an argument. See more information [here](https://developer.paypal.com/docs/api/catalog-products/v1/#products_patch) + ```bash + python product_catalog.py --action update --product_id PRODUCT_ID --update_payload '[{"op": "replace", "path": "/description", "value": "My Update. Does it changes it?"}]' + ``` + ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: * required for your connector to work need to go to `MAIN_REQUIREMENTS` list. * required for the testing need to go to `TEST_REQUIREMENTS` list -### Publishing a new version of the connector + +All of your dependencies should be managed via Poetry. + +To add a new dependency, run: +```bash +poetry add +``` + + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-paypal-transaction test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/paypal-transaction.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/paypal-transaction.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index c4ebc718cf3b..f4682c9534da 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -1,69 +1,84 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests +# Make sure the paths you have in each path matches with your data. +#For multiple env testing, you can duplicate the tests and change the path to the proper credentials file connector_image: airbyte/source-paypal-transaction:dev test_strictness_level: high acceptance_tests: spec: tests: + #Test with Prod credentials (Make sure you purt the right ones) - spec_path: "source_paypal_transaction/spec.yaml" - config_path: secrets/config_oauth.json + config_path: secrets/config.json backward_compatibility_tests_config: disable_for_version: "0.1.13" connection: tests: - - config_path: secrets/config_oauth.json + #Test With Prod Credentials + - config_path: secrets/config.json status: succeed - - config_path: secrets/config_oauth_sandbox.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed - - config_path: integration_tests/invalid_config_oauth.json + #Test with Invalid Credentials + - config_path: integration_tests/sample_files/invalid_config.json status: failed + #Test with Sandbox Credentials + # - config_path: secrets/config_sandbox.json + # status: succeed discovery: tests: - - config_path: secrets/config_oauth.json + - config_path: secrets/config.json + # - config_path: secrets/config_sandbox.json backward_compatibility_tests_config: disable_for_version: "2.0.0" # Change in cursor field for transactions stream basic_read: tests: - - config_path: secrets/config_oauth.json - ignored_fields: - balances: - - name: last_refresh_time - bypass_reason: "field changes during every read" + #Test Prod Environment - Uncomment and change according to your prod setup + #Change the expected records, remember to align them with the timeframe you have selected + #Do not select streams that take more than 5 mins to load data as that can lead to timeouts + #You can comment the lines if you are sure you have data for the below streams. + - config_path: secrets/config.json + # - config_path: secrets/config_sandbox.json empty_streams: - - name: transactions - bypass_reason: "can not populate" - timeout_seconds: 1200 - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - - config_path: secrets/config_oauth_sandbox.json + - name: show_product_details + bypass_reason: "Products may not exist" + - name: list_products + bypass_reason: "Product List may be too big causing timeout errors" + - name: search_invoices + bypass_reason: "Order makes the diff fail." + #Have to add for testing PR CI. + - name: list_disputes + bypass_reason: "Disputes may not exist." ignored_fields: balances: - name: last_refresh_time bypass_reason: "field changes during every read" - timeout_seconds: 1200 + list_products: + - name: description + bypass_reason: "Sometimes it is not contained in the response" + timeout_seconds: 3200 expect_records: - path: "integration_tests/expected_records_sandbox.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + path: "integration_tests/sample_files/expected_records_sandbox.jsonl" + #path: "integration_tests/sample_files/expected_records.jsonl" + extra_fields: yes + exact_order: yes + extra_records: no + fail_on_extra_columns: False incremental: tests: - - config_path: secrets/config_oauth.json - configured_catalog_path: integration_tests/configured_catalog.json + - config_path: secrets/config.json + # - config_path: secrets/config_sandbox.json + configured_catalog_path: integration_tests/incremental_catalog.json future_state: - future_state_path: integration_tests/abnormal_state.json + future_state_path: integration_tests/sample_files/abnormal_state.json skip_comprehensive_incremental_tests: true full_refresh: tests: - - config_path: secrets/config_oauth.json + - config_path: secrets/config.json + # - config_path: secrets/config_sandbox.json + configured_catalog_path: integration_tests/full_refresh_catalog.json ignored_fields: balances: - name: last_refresh_time bypass_reason: "field changes during every read" - configured_catalog_path: integration_tests/configured_catalog.json + list_products: + - name: description + bypass_reason: "Sometimes it is not contained in the response" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py new file mode 100644 index 000000000000..c371024b7ff9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py @@ -0,0 +1,89 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# REQUIREMENTS: +# 1. Put your sandbox credentials in ../secrets/config.json (Create them if it doesn't exist). +# Use the following body (change all the values): +# { +# "client_id": "YOUT_CLIENT_ID", +# "client_secret": "YOUR_SECRET_CLIENT_ID", +# "start_date": "2021-06-01T00:00:00Z", +# "end_date": "2024-06-10T00:00:00Z", +# "is_sandbox": true +# } + +# HOW TO USE: +# To create a new payment: python script_name.py create +# To update an existing dispute: +# python disputes_generator.py update DISPUTE_ID ''[{"op": "replace", "path": "/reason", "value": "The new reason"}]' +# To update a dispute status +# python update_dispute.py require-evidence DISPUTE_ID SELLER_EVIDENCE + +import base64 +import json +import sys + +import requests + + +# Function to get a PayPal OAuth token +def get_paypal_token(client_id, secret_id): + url = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + base64.b64encode(f"{client_id}:{secret_id}".encode()).decode(), + } + payload = {"grant_type": "client_credentials"} + response = requests.post(url=url, data=payload, headers=headers) + return response.json().get("access_token") + + +def update_dispute(token, dispute_id, updates): + """Update a PayPal dispute.""" + url = f"https://api-m.paypal.com/v1/customer/disputes/{dispute_id}" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} + response = requests.patch(url, headers=headers, json=updates) + print("RESPONSE: ", response.text) + return response.json() + + +def require_evidence(token, dispute_id, action): + """Require evidence for a PayPal dispute.""" + url = f"https://api-m.paypal.com/v1/customer/disputes/{dispute_id}/require-evidence" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} + payload = {"action": action} + response = requests.post(url, headers=headers, json=payload) + print("RESPONSE: ", response.text) + return response.json() + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + + +def main(): + + operation = sys.argv[1] + + CREDS = read_json("../secrets/config.json") + client_id = CREDS.get("client_id") + secret_id = CREDS.get("client_secret") + token = get_paypal_token(client_id, secret_id) + + if operation == "update": + dispute_id = sys.argv[2] + updates = json.loads(sys.argv[3]) # Expecting JSON string as the third argument + update_response = update_dispute(token, dispute_id, updates) + print("Update Response:", update_response) + + elif sys.argv[1] == "require-evidence": + dispute_id = sys.argv[2] + action = sys.argv[3] # Either 'BUYER_EVIDENCE' or 'SELLER_EVIDENCE' + evidence_response = require_evidence(token, dispute_id, action) + print("Evidence Requirement Response:", evidence_response) + else: + print("Invalid command. Use 'create', 'update', or 'require-evidence'.") + + +if __name__ == "__main__": + main() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/invoices.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/invoices.py new file mode 100644 index 000000000000..005a11c06169 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/invoices.py @@ -0,0 +1,197 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# REQUIREMENTS: +# 1. Put your sandbox credentials in ../secrets/config.json (Create them if it doesn't exist). +# Use the following body (change all the values): +# { +# "client_id": "YOUT_CLIENT_ID", +# "client_secret": "YOUR_SECRET_CLIENT_ID", +# "start_date": "2021-06-01T00:00:00Z", +# "end_date": "2024-06-10T00:00:00Z", +# "is_sandbox": true +# } +# How to Use: +# To Create a Draft Invoice: +# Execute the script with create_draft to generate a new invoice draft. +# The script automatically sets the invoice and due dates based on the current date and a 30-day term. +# python invoices.py create_draft +# To Send a Draft Invoice: +# Use send_draft action along with the required --invoice_id parameter, and optional parameters for email subject, note, and additional recipients. +# python invoices.py send_draft --invoice_id "INV2-XXXX-XXXX-XXXX-XXXX" --subject "Your Invoice Subject" --note "Your custom note" --additional_recipients example@email.com + +import argparse +import base64 +import json +import random +import string +from datetime import datetime, timedelta + +import requests + + +# Function to generate a random alphanumeric string +def generate_random_string(length=10): + return "".join(random.choices(string.ascii_letters + string.digits, k=length)) + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + + +# Function to get a PayPal OAuth token +def get_paypal_token(client_id, secret_id): + url = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + base64.b64encode(f"{client_id}:{secret_id}".encode()).decode(), + } + payload = {"grant_type": "client_credentials"} + response = requests.post(url=url, data=payload, headers=headers) + return response.json().get("access_token") + + +# Function to create a draft invoice +def create_draft_invoice(access_token, invoice_date, term_type, due_date): + url = "https://api-m.sandbox.paypal.com/v2/invoicing/invoices" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"} + data = { + "detail": { + "invoice_number": generate_random_string(8), + "invoice_date": invoice_date, + "payment_term": {"term_type": term_type, "due_date": due_date}, + "currency_code": "USD", + "reference": "", + "note": "", + "terms_and_conditions": "", + "memo": "", + }, + "invoicer": { + "name": {"given_name": "David", "surname": "Larusso"}, + "address": { + "address_line_1": "123 Townsend St", + "address_line_2": "Floor 6", + "admin_area_2": "San Francisco", + "admin_area_1": "CA", + "postal_code": "94107", + "country_code": "US", + }, + "phones": [{"country_code": "001", "national_number": "4085551234", "phone_type": "MOBILE"}], + "website": "www.example.com", + "tax_id": "XX-XXXXXXX", + "logo_url": "https://example.com/logo.png", + "additional_notes": "", + }, + "primary_recipients": [ + { + "billing_info": { + "name": {"given_name": "Stephanie", "surname": "Meyers"}, + "address": { + "address_line_1": "1234 Main Street", + "admin_area_2": "Anytown", + "admin_area_1": "CA", + "postal_code": "98765", + "country_code": "US", + }, + "email_address": "foobuyer@example.com", + "phones": [{"country_code": "001", "national_number": "4884551234", "phone_type": "HOME"}], + "additional_info_value": "add-info", + }, + "shipping_info": { + "name": {"given_name": "Stephanie", "surname": "Meyers"}, + "address": { + "address_line_1": "1234 Main Street", + "admin_area_2": "Anytown", + "admin_area_1": "CA", + "postal_code": "98765", + "country_code": "US", + }, + }, + } + ], + "items": [ + { + "name": "Yoga Mat", + "description": "Elastic mat to practice yoga.", + "quantity": "1", + "unit_amount": {"currency_code": "USD", "value": "50.00"}, + "tax": {"name": "Sales Tax", "percent": "7.25"}, + "discount": {"percent": "5"}, + "unit_of_measure": "QUANTITY", + }, + { + "name": "Yoga t-shirt", + "quantity": "1", + "unit_amount": {"currency_code": "USD", "value": "10.00"}, + "tax": {"name": "Sales Tax", "percent": "7.25"}, + "discount": {"amount": {"currency_code": "USD", "value": "5.00"}}, + "unit_of_measure": "QUANTITY", + }, + ], + "configuration": { + "partial_payment": {"allow_partial_payment": True, "minimum_amount_due": {"currency_code": "USD", "value": "20.00"}}, + "allow_tip": True, + "tax_calculated_after_discount": True, + "tax_inclusive": False, + }, + "amount": { + "breakdown": { + "custom": {"label": "Packing Charges", "amount": {"currency_code": "USD", "value": "10.00"}}, + "shipping": {"amount": {"currency_code": "USD", "value": "10.00"}, "tax": {"name": "Sales Tax", "percent": "7.25"}}, + "discount": {"invoice_discount": {"percent": "5"}}, + } + }, + } + response = requests.post(url, headers=headers, json=data) + return response.json() + + +# Function to send an existing draft invoice +def send_draft_invoice(access_token, invoice_id, subject, note, additional_recipients): + url = f"https://api-m.sandbox.paypal.com/v2/invoicing/invoices/{invoice_id}/send" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"} + data = { + "subject": subject, + "note": note, + "send_to_recipient": True, + "additional_recipients": additional_recipients, + "send_to_invoicer": False, + } + response = requests.post(url, headers=headers, json=data) + return response.json() + + +# Main function +def main(): + parser = argparse.ArgumentParser(description="PayPal Invoice Actions") + parser.add_argument("action", help="Action to perform: create_draft or send_draft") + parser.add_argument("--invoice_id", help="Invoice ID (required for send_draft)") + parser.add_argument("--subject", help="Subject for the invoice email") + parser.add_argument("--note", help="Note for the invoice email") + parser.add_argument("--additional_recipients", nargs="*", help="Additional recipients for the invoice email") + args = parser.parse_args() + + CREDS = read_json("../secrets/config.json") + + client_id = CREDS.get("client_id") + secret_id = CREDS.get("client_secret") + access_token = get_paypal_token(client_id, secret_id) + + if args.action == "create_draft": + invoice_date = datetime.now().strftime("%Y-%m-%d") + term_type = "NET_30" + due_date = (datetime.now() + timedelta(days=30)).strftime("%Y-%m-%d") + result = create_draft_invoice(access_token, invoice_date, term_type, due_date) + print("Draft Invoice Created:", result) + elif args.action == "send_draft": + if not args.invoice_id: + print("Invoice ID is required for sending a draft invoice.") + return + result = send_draft_invoice(access_token, args.invoice_id, args.subject, args.note, args.additional_recipients) + print("Draft Invoice Sent:", result) + else: + print("Invalid action specified") + + +if __name__ == "__main__": + main() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py new file mode 100644 index 000000000000..6a2f46c3b524 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py @@ -0,0 +1,106 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# REQUIREMENTS: +# 1. Put your sandbox credentials in ../secrets/config.json (Create them if it doesn't exist). +# Use the following body (change all the values): +# { +# "client_id": "YOUT_CLIENT_ID", +# "client_secret": "YOUR_SECRET_CLIENT_ID", +# "start_date": "2021-06-01T00:00:00Z", +# "end_date": "2024-06-10T00:00:00Z", +# "is_sandbox": true +# } + +# HOW TO USE: +# To create a new payment: python script_name.py create +# To update an existing product: +# python script_name.py update PAYMENT_ID '[{"op": "replace", "path": "/transactions/0/amount", "value": {"total": "50.00", "currency": "USD"}}]' +# +# NOTE: This is version does not work for CREATE PAYMENT as the HEADER requires data I can't get +# +# You may need to add a security context, but you need the proper set of permissions in your account to be able to send this context +# security_context = '{"actor":{"account_number":"","party_id":"","auth_claims":["AUTHORIZATION_CODE"],"auth_state":"ANONYMOUS","client_id":"zf3..4BQ0T9aw-ngFr9dm....Zx9D-Lf4"},"auth_token":"","auth_token_type":"ACCESS_TOKEN","last_validated":1393560555,"scopes":["https://api-m.sandbox.paypal.com/v1/payments/.*","https://api-m.sandbox.paypal.com/v1/vault/credit-card/.*","openid","https://uri.paypal.com/services/payments/futurepayments","https://api-m.sandbox.paypal.com/v1/vault/credit-card","https://api-m.sandbox.paypal.com/v1/payments/.*"],"subjects":[{"subject":{"account_number":"","party_id":"","auth_claims":["PASSWORD"],"auth_state":"LOGGEDIN"}}]}' + + +import base64 +import json +import sys + +import requests + + +# Function to get a PayPal OAuth token +def get_paypal_token(client_id, secret_id): + url = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + base64.b64encode(f"{client_id}:{secret_id}".encode()).decode(), + } + payload = {"grant_type": "client_credentials"} + response = requests.post(url=url, data=payload, headers=headers) + return response.json().get("access_token") + + +def create_payment(token, security_context): + """Create a PayPal payment.""" + url = "https://api-m.paypal.com/v1/payments/payment" + headers = { + "Content-Type": "application/json", + # "Authorization": f"Bearer {token}", + "X-PAYPAL-SECURITY-CONTEXT": security_context, + } + payload = { + "intent": "sale", + "transactions": [ + { + "amount": {"total": "30.00", "currency": "USD", "details": {"subtotal": "30.00"}}, + "description": "This is a test - Pines test.", + "item_list": { + "items": [{"name": "My item", "sku": "123445667", "price": "15.00", "currency": "USD", "quantity": 2}], + }, + } + ], + "payer": {"payment_method": "paypal"}, + "redirect_urls": {"return_url": "https://example.com/return", "cancel_url": "https://example.com/cancel"}, + } + + response = requests.post(url, headers=headers, json=payload) + return response.json() + + +def update_payment(token, payment_id, updates): + """Update a PayPal payment.""" + url = f"https://api-m.paypal.com/v1/payments/payment/{payment_id}" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} + response = requests.patch(url, headers=headers, json=updates) + return response.json() + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + + +def main(): + + CREDS = read_json("../secrets/config.json") + client_id = CREDS.get("client_id") + secret_id = CREDS.get("client_secret") + token = get_paypal_token(client_id, secret_id) + + if sys.argv[1] == "create": + payment = create_payment(token, security_context) + print("Created Payment:", payment) + + elif sys.argv[1] == "update": + payment_id = sys.argv[2] + updates = json.loads(sys.argv[3]) # Expecting JSON string as the third argument + update_response = update_payment(token, payment_id, updates) + print("Update Response:", update_response) + + else: + print("Invalid command. Use 'create' or 'update'.") + + +if __name__ == "__main__": + main() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py index 8fa96fae513f..d8067ec1e2bf 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -6,7 +6,18 @@ # REQUIREMENTS: # 1. sudo apt-get install chromium-chromedriver # 2. pip install selenium -# 3. ../secrets/creds.json with buyers email/password and account client_id/secret +# 3. ../secrets/config.json with buyers email/password and account client_id/secret +# { +# "client_id": "YOUT_CLIENT_ID", +# "client_secret": "YOUR_SECRET_CLIENT_ID", +# "start_date": "2021-06-01T00:00:00Z", +# "end_date": "2024-06-10T00:00:00Z", +# "is_sandbox": true, +# "buyer_username": "", #This could be also your test Sandbox email generated by the system +# "buyer_password": "", #This could be also your test Sandbox pawd generated by the system +# "payer_id": "" # This is the Account ID, yours or your Sandbox generated user + +# } # HOW TO USE: # python paypal_transaction_generator.py - will generate 3 transactions by default @@ -95,7 +106,7 @@ def read_json(filepath): def get_api_token(): client_id = CREDS.get("client_id") - secret = CREDS.get("secret") + secret = CREDS.get("client_secret") token_refresh_endpoint = "https://api-m.sandbox.paypal.com/v1/oauth2/token" data = "grant_type=client_credentials" @@ -103,7 +114,7 @@ def get_api_token(): auth = (client_id, secret) response = requests.request(method="POST", url=token_refresh_endpoint, data=data, headers=headers, auth=auth) response_json = response.json() - # print(response_json) + print("RESPONSE -->", response_json) API_TOKEN = response_json["access_token"] return API_TOKEN @@ -142,15 +153,16 @@ def make_payment(): # APPROVE PAYMENT def login(): - driver = webdriver.Chrome("/usr/bin/chromedriver") + # driver = webdriver.Chrome("/usr/bin/chromedriver") + driver = webdriver.Chrome() # SIGN_IN driver.get("https://www.sandbox.paypal.com/ua/signin") - driver.find_element_by_id("email").send_keys(CREDS["buyer_username"]) - driver.find_element_by_id("btnNext").click() + driver.find_element(By.ID, "email").send_keys(CREDS["buyer_username"]) + driver.find_element(By.ID, "btnNext").click() sleep(2) - driver.find_element_by_id("password").send_keys(CREDS["buyer_password"]) - driver.find_element_by_id("btnLogin").click() + driver.find_element(By.ID, "password").send_keys(CREDS["buyer_password"]) + driver.find_element(By.ID, "btnLogin").click() return driver @@ -160,13 +172,14 @@ def approve_payment(driver, url): sleep(3) if not cookies_accepted: - cookies = driver.find_element_by_id("acceptAllButton") - if cookies: - cookies.click() + try: + cookies_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "acceptAllButton"))) + cookies_button.click() + cookies_accepted = True + except Exception as e: + print("Could not find the accept all cookies button, exception:", e) - cookies_accepted = True driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") - element = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "payment-submit-btn"))) sleep(1) element.click() @@ -180,14 +193,26 @@ def approve_payment(driver, url): def execute_payment(url): - response = requests.request(method="POST", url=url, data='{"payer_id": "ZE5533HZPGMC6"}', headers=headers) - response_json = response.json() - print(f'Payment executed: {url} with STATE: {response_json["state"]}') + try: + # Attempt to make the POST request + response = requests.post(url, data=json.dumps({"payer_id": CREDS.get("payer_id")}), headers=headers) + response_json = response.json() + # Check if the request was successful + if response.status_code == 200: + print(f"Your payment has been successfully executed to {url} with STATE: {response_json['state']}") + else: + # If the response code is not 200, print the error message + print( + f"Your payment execution was not successful. You got {response.status_code} with message {response.json().get('message', 'No message available')}." + ) + except requests.exceptions.RequestException as e: + # If an error occurs during the request, print the error + print(f"An error occurred: {e}") TOTAL_TRANSACTIONS = int(sys.argv[1]) if len(sys.argv) > 1 else 3 -CREDS = read_json("../secrets/creds.json") +CREDS = read_json("../secrets/config.json") headers = {"Authorization": f"Bearer {get_api_token()}", "Content-Type": "application/json"} driver = login() cookies_accepted = False diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py new file mode 100755 index 000000000000..9ad9a7553814 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py @@ -0,0 +1,115 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# +# REQUIREMENTS: +# 1. Put your sandbox credentials in ../secrets/config.json (Create them if it doesn't exist). +# Use the following body (change all the values): +# { +# "client_id": "YOUT_CLIENT_ID", +# "client_secret": "YOUR_SECRET_CLIENT_ID", +# "start_date": "2021-06-01T00:00:00Z", +# "end_date": "2024-06-10T00:00:00Z", +# "is_sandbox": true +# } + +# HOW TO USE: +# To create a new product: +# python product_catalog.py --action create --description "This is a test product" --category TRAVEL +# To update an existing product: +# python product_catalog.py --action update --product_id PRODUCT_ID --update_payload '[{"op": "replace", "path": "/description", "value": "My Update. Does it changes it?"}]' +# The CATEGORY must be one of the listed in this page: https://developer.paypal.com/docs/api/catalog-products/v1/#products_create +# NOTE: This is version one, it conly creates 1 product at a time. This has not been parametrized +# TODO: Generate N products in one run. + +import argparse +import base64 +import json +import random +import string + +import requests + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + + +def generate_random_string(length=10): + """Generate a random string of fixed length.""" + letters = string.ascii_letters + return "".join(random.choice(letters) for i in range(length)) + + +def get_paypal_token(client_id, secret_id): + """Get a bearer token from PayPal.""" + url = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + base64.b64encode(f"{client_id}:{secret_id}".encode()).decode(), + } + payload = {"grant_type": "client_credentials"} + response = requests.post(url=url, data=payload, headers=headers) + return response.json().get("access_token") + + +def create_paypal_product(access_token, description="Cotton XL", category="clothing"): + """Create a product in PayPal.""" + url = "https://api-m.sandbox.paypal.com/v1/catalogs/products" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"} + payload = { + "name": "Pines-T-Shirt-" + generate_random_string(5), + "type": "PHYSICAL", + "id": generate_random_string(10), + "description": description, + "category": category, + "image_url": "https://example.com/gallary/images/" + generate_random_string(10) + ".jpg", + "home_url": "https://example.com/catalog/" + generate_random_string(10) + ".jpg", + } + response = requests.post(url=url, json=payload, headers=headers) + return response.json() + + +def update_paypal_product(access_token, product_id, updates): + """Update a product in PayPal.""" + url = f"https://api-m.sandbox.paypal.com/v1/catalogs/products/{product_id}" + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"} + response = requests.patch(url=url, json=updates, headers=headers) + if response.status_code == 204: + print(f"Update Successful. Response {response.status_code}. This succesful repsonse has no response body") + return None + else: + print(f"Error: {response.status_code}, {response.text}") + return None + + +# Parse command line arguments +parser = argparse.ArgumentParser(description="Create or Update a PayPal Product.") +parser.add_argument("--action", choices=["create", "update"], required=True, help="Action to perform: create or update") +parser.add_argument("--description", help="Product description for create action", required=True) +parser.add_argument("--category", help="Product category for create action", required=True) +parser.add_argument("--product_id", help="Product ID for update action", required=True) +parser.add_argument("--update_payload", help="Operation for update action", required=True) + +args = parser.parse_args() + +# Common setup +CREDS = read_json("../secrets/config.json") +client_id = CREDS.get("client_id") +secret_id = CREDS.get("client_secret") +access_token = get_paypal_token(client_id, secret_id) + +# Perform action based on arguments +if args.action == "create": + product = create_paypal_product(access_token, args.description, args.category) + print("Created product:", product) +elif args.action == "update" and args.product_id and args.update_payload: + try: + # updates = [{"op": "replace", "path": "/description", "value": "My Update. Does it changes it?"}] + operations = json.loads(args.update_payload) + product = update_paypal_product(access_token, args.product_id, operations) + print("Updated product:", product) + except json.JSONDecodeError: + print(f"Invalid JSON in update payload") +else: + print("Invalid arguments") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json deleted file mode 100644 index dc44c707ad24..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "as_of_time": "2033-06-09T00:00:00Z" - }, - "stream_descriptor": { - "name": "balances" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "transaction_updated_date": "2033-06-09T00:00:00Z" - }, - "stream_descriptor": { - "name": "transactions" - } - } - } -] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index e0992dea17b0..0c7e6869f3a9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -20,6 +20,58 @@ }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "list_products", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "show_product_details", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "list_disputes", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time_cut"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_invoices", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "list_payments", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_disputes.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_disputes.json new file mode 100644 index 000000000000..ce3ee289234a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_disputes.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "list_disputes", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time_cut"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_payments.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_payments.json new file mode 100644 index 000000000000..07cca1e51c96 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_payments.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "list_payments", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_products.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_products.json new file mode 100644 index 000000000000..2fc44b85cdb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_list_products.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "list_products", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_search_invoices.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_search_invoices.json new file mode 100644 index 000000000000..32b0401ece61 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_search_invoices.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "search_invoices", + "json_schema": {}, + "default_cursor_field": ["invoice_updated_date"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_show_product_details.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_show_product_details.json new file mode 100644 index 000000000000..556bfd63366a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_show_product_details.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "show_product_details", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl deleted file mode 100644 index 86cbdca7392b..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"stream": "balances", "data": {"balances": [{"currency": "USD", "primary": true, "total_balance": {"currency_code": "USD", "value": "0.00"}, "available_balance": {"currency_code": "USD", "value": "0.00"}, "withheld_balance": {"currency_code": "USD", "value": "0.00"}}], "account_id": "QJQSC8WXYCA2L", "as_of_time": "2021-07-03T00:00:00Z", "last_refresh_time": "2023-09-18T13:29:59Z"}, "emitted_at": 1695051482452} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl deleted file mode 100644 index da4775fecc29..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "23N61105X92314351", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-04T17:13:23+0000", "transaction_updated_date": "2021-07-04T17:13:23+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "202.58"}, "available_balance": {"currency_code": "USD", "value": "202.58"}, "invoice_id": "48787580055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "48787580055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "48787580055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-04T17:13:23Z", "transaction_id": "23N61105X92314351"}, "emitted_at": 1694795587519} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1FN09943JY662130R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T22:56:54+0000", "transaction_updated_date": "2021-07-05T22:56:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "231.52"}, "available_balance": {"currency_code": "USD", "value": "231.52"}, "invoice_id": "65095789448", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "65095789448"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "65095789448"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T22:56:54Z", "transaction_id": "1FN09943JY662130R"}, "emitted_at": 1694795587522} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0M443597T0019954R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:01:13+0000", "transaction_updated_date": "2021-07-05T23:01:13+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "260.46"}, "available_balance": {"currency_code": "USD", "value": "260.46"}, "invoice_id": "41468340464", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "41468340464"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "41468340464"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:01:13Z", "transaction_id": "0M443597T0019954R"}, "emitted_at": 1694795587524} -{"stream": "balances", "data": {"balances": [{"currency": "USD", "primary": true, "total_balance": {"currency_code": "USD", "value": "173.64"}, "available_balance": {"currency_code": "USD", "value": "173.64"}, "withheld_balance": {"currency_code": "USD", "value": "0.00"}}], "account_id": "MDXWPD67GEP5W", "as_of_time": "2021-07-03T00:00:00Z", "last_refresh_time": "2023-09-18T08:59:59Z"}, "emitted_at": 1695051579296} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/full_refresh_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/full_refresh_catalog.json new file mode 100644 index 000000000000..b50a9c88795a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/full_refresh_catalog.json @@ -0,0 +1,47 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["transaction_updated_date"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "list_disputes", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time_cut"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_invoices", + "json_schema": {}, + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "list_payments", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/incremental_catalog.json new file mode 100644 index 000000000000..4d75f9e3b060 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/incremental_catalog.json @@ -0,0 +1,47 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["transaction_updated_date"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "balances", + "json_schema": {}, + "default_cursor_field": ["as_of_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "list_disputes", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time_cut"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "list_payments", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": ["update_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json deleted file mode 100644 index 771ae5dbac0a..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "credentials": { - "auth_type": "oauth2.0", - "client_id": "AWA__", - "client_secret": "ENC__", - "refresh_token": "__" - }, - "start_date": "2021-07-03T00:00:00+00:00", - "end_date": "2021-07-04T23:59:59+00:00", - "is_sandbox": false -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json deleted file mode 100644 index 11d8539eeb3f..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "client_id": "PAYPAL_CLIENT_ID", - "client_secret": "PAYPAL_SECRET", - "start_date": "2021-06-01T00:00:00+00:00", - "is_sandbox": false -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/abnormal_state.json new file mode 100644 index 000000000000..2465b3a531dc --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/abnormal_state.json @@ -0,0 +1,46 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "as_of_time": "2033-06-09T00:00:00Z" + }, + "stream_descriptor": { + "name": "balances" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "transaction_updated_date": "2033-06-09T00:00:00Z" + }, + "stream_descriptor": { + "name": "transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "update_time": "2033-06-09T00:00:00Z" + }, + "stream_descriptor": { + "name": "list_payments" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time_cut": "2033-06-09T00:00:00.000Z" + }, + "stream_descriptor": { + "name": "list_disputes" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records.jsonl new file mode 100644 index 000000000000..dfc00c24767c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records.jsonl @@ -0,0 +1,16 @@ +{"stream": "transactions", "data": {"transaction_info": {"transaction_id": "69B759611M2733128","transaction_event_code": "T1503","transaction_initiation_date": "2024-02-01T00:01:23+0000","transaction_updated_date": "2024-02-01T00:01:23+0000","transaction_amount": {"currency_code": "USD","value": "-60.75"},"transaction_status": "S","ending_balance": {"currency_code": "USD","value": "309800.06"},"available_balance": {"currency_code": "USD","value": "309800.06"},"protection_eligibility": "02"},"payer_info": {"address_status": "N","payer_name": {}},"shipping_info": {},"cart_info": {},"store_info": {},"auction_info": {},"incentive_info": {}}, "emitted_at": 1695051482452} +{"stream": "transactions", "data": {"transaction_info": {"transaction_id": "1N809273356042704","transaction_event_code": "T0001","transaction_initiation_date": "2024-02-01T00:01:24+0000","transaction_updated_date": "2024-02-01T00:01:24+0000","transaction_amount": {"currency_code": "USD","value": "-10.00"},"fee_amount": {"currency_code": "USD","value": "-0.25"},"transaction_status": "P","transaction_subject": "You have a payout!","transaction_note": "Thanks for your patronage!","ending_balance": {"currency_code": "USD","value": "309789.81"},"available_balance": {"currency_code": "USD","value": "309789.81"},"custom_field": "201403140001","protection_eligibility": "02"},"payer_info": {"email_address": "Alexa.Dietrich@gmail.com","address_status": "N","payer_name": {}},"shipping_info": {"name": "John, Merchant"},"cart_info": {},"store_info": {},"auction_info": {},"incentive_info": {}}, "emitted_at": 1695052482453} +{"stream": "transactions", "data": {"transaction_info": {"transaction_id": "70B03706PU258313Y","paypal_reference_id": "1N809273356042704","paypal_reference_id_type": "TXN","transaction_event_code": "T1105","transaction_initiation_date": "2024-02-01T00:01:24+0000","transaction_updated_date": "2024-02-01T00:01:24+0000","transaction_amount": {"currency_code": "USD","value": "10.25"},"transaction_status": "S","transaction_subject": "You have a payout!","transaction_note": "Thanks for your patronage!","ending_balance": {"currency_code": "USD","value": "309800.06"},"available_balance": {"currency_code": "USD","value": "309800.06"},"protection_eligibility": "02"},"payer_info": {"address_status": "N","payer_name": {}},"shipping_info": {},"cart_info": {},"store_info": {},"auction_info": {},"incentive_info": {}}, "emitted_at": 1695053482454} +{"stream": "transactions", "data": {"transaction_info": {"transaction_id": "60D327402F0012324","transaction_event_code": "T0001","transaction_initiation_date": "2024-02-01T00:01:24+0000","transaction_updated_date": "2024-02-01T00:01:24+0000","transaction_amount": {"currency_code": "USD","value": "-20.00"},"fee_amount": {"currency_code": "USD","value": "-0.25"},"transaction_status": "P","transaction_subject": "You have a payout!","transaction_note": "Thanks for your support!","ending_balance": {"currency_code": "USD","value": "309779.81"},"available_balance": {"currency_code": "USD","value": "309779.81"},"custom_field": "201403140002","protection_eligibility": "02"},"payer_info": {"address_status": "N","payer_name": {}},"shipping_info": {"name": "John, Merchant"},"cart_info": {},"store_info": {},"auction_info": {},"incentive_info": {}}, "emitted_at": 1695054482455} +{"stream": "balances", "data": {"balances":[{"currency":"CHF","total_balance":{"currency_code":"CHF","value":"1001.19"},"available_balance":{"currency_code":"CHF","value":"1001.19"},"withheld_balance":{"currency_code":"CHF","value":"0.00"}},{"currency":"HKD","total_balance":{"currency_code":"HKD","value":"10833.47"},"available_balance":{"currency_code":"HKD","value":"10833.47"},"withheld_balance":{"currency_code":"HKD","value":"0.00"}},{"currency":"TWD","total_balance":{"currency_code":"TWD","value":"6289.00"},"available_balance":{"currency_code":"TWD","value":"6289.00"},"withheld_balance":{"currency_code":"TWD","value":"0.00"}},{"currency":"MXN","total_balance":{"currency_code":"MXN","value":"3827706.05"},"available_balance":{"currency_code":"MXN","value":"3827706.05"},"withheld_balance":{"currency_code":"MXN","value":"0.00"}},{"currency":"EUR","total_balance":{"currency_code":"EUR","value":"418736.68"},"available_balance":{"currency_code":"EUR","value":"418736.68"},"withheld_balance":{"currency_code":"EUR","value":"0.00"}},{"currency":"USD","primary":true,"total_balance":{"currency_code":"USD","value":"309860.81"},"available_balance":{"currency_code":"USD","value":"309860.81"},"withheld_balance":{"currency_code":"USD","value":"0.00"}},{"currency":"CAD","total_balance":{"currency_code":"CAD","value":"1158.92"},"available_balance":{"currency_code":"CAD","value":"1158.92"},"withheld_balance":{"currency_code":"CAD","value":"0.00"}},{"currency":"NOK","total_balance":{"currency_code":"NOK","value":"0.85"},"available_balance":{"currency_code":"NOK","value":"0.85"},"withheld_balance":{"currency_code":"NOK","value":"0.00"}},{"currency":"THB","total_balance":{"currency_code":"THB","value":"96658.02"},"available_balance":{"currency_code":"THB","value":"96658.02"},"withheld_balance":{"currency_code":"THB","value":"0.00"}},{"currency":"AUD","total_balance":{"currency_code":"AUD","value":"2405.19"},"available_balance":{"currency_code":"AUD","value":"2405.19"},"withheld_balance":{"currency_code":"AUD","value":"0.00"}},{"currency":"ILS","total_balance":{"currency_code":"ILS","value":"0.00"},"available_balance":{"currency_code":"ILS","value":"0.00"},"withheld_balance":{"currency_code":"ILS","value":"0.00"}},{"currency":"SGD","total_balance":{"currency_code":"SGD","value":"5841.51"},"available_balance":{"currency_code":"SGD","value":"5841.51"},"withheld_balance":{"currency_code":"SGD","value":"0.00"}},{"currency":"JPY","total_balance":{"currency_code":"JPY","value":"45608"},"available_balance":{"currency_code":"JPY","value":"45608"},"withheld_balance":{"currency_code":"JPY","value":"0"}},{"currency":"PLN","total_balance":{"currency_code":"PLN","value":"621.42"},"available_balance":{"currency_code":"PLN","value":"621.42"},"withheld_balance":{"currency_code":"PLN","value":"0.00"}},{"currency":"GBP","total_balance":{"currency_code":"GBP","value":"671250.65"},"available_balance":{"currency_code":"GBP","value":"671250.65"},"withheld_balance":{"currency_code":"GBP","value":"0.00"}},{"currency":"HUF","total_balance":{"currency_code":"HUF","value":"0.00"},"available_balance":{"currency_code":"HUF","value":"0.00"},"withheld_balance":{"currency_code":"HUF","value":"0.00"}},{"currency":"SEK","total_balance":{"currency_code":"SEK","value":"90.92"},"available_balance":{"currency_code":"SEK","value":"90.92"},"withheld_balance":{"currency_code":"SEK","value":"0.00"}},{"currency":"NZD","total_balance":{"currency_code":"NZD","value":"813.27"},"available_balance":{"currency_code":"NZD","value":"813.27"},"withheld_balance":{"currency_code":"NZD","value":"0.00"}},{"currency":"PHP","total_balance":{"currency_code":"PHP","value":"291489.92"},"available_balance":{"currency_code":"PHP","value":"291489.92"},"withheld_balance":{"currency_code":"PHP","value":"0.00"}},{"currency":"BRL","total_balance":{"currency_code":"BRL","value":"0.00"},"available_balance":{"currency_code":"BRL","value":"0.00"},"withheld_balance":{"currency_code":"BRL","value":"0.00"}},{"currency":"RUB","total_balance":{"currency_code":"RUB","value":"274.53"},"available_balance":{"currency_code":"RUB","value":"274.53"},"withheld_balance":{"currency_code":"RUB","value":"0.00"}}],"account_id":"C7CYMKZDG8D6E","as_of_time":"2024-02-01T00:00:00Z","last_refresh_time":"2024-02-05T17:59:59Z"}, "emitted_at": 1695051579296} +{"stream": "list_disputes", "data": {"dispute_id":"PP-R-PNP-10089600","create_time":"2024-01-26T15:31:02.000Z","update_time":"2024-02-04T12:06:03.000Z","disputed_transactions":[{"buyer_transaction_id":"5CW48839XK1160452","seller":{"merchant_id":"C7CYMKZDG8D6E"}}],"reason":"MERCHANDISE_OR_SERVICE_NOT_RECEIVED","status":"RESOLVED","dispute_state":"RESOLVED","dispute_amount":{"currency_code":"USD","value":"40.00"},"dispute_life_cycle_stage":"INQUIRY","dispute_channel":"INTERNAL","outcome":"LOST","links":[{"href":"https://api-m.sandbox.paypal.com/v1/customer/disputes/PP-R-PNP-10089600","rel":"self","method":"GET"}]}, "emitted_at": 1695051579296} +{"stream": "list_products", "data": {"id": "1647236710","name": "Blue M","description": "Blue M","create_time": "2022-03-14T05:45:06Z","links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/1647236710","rel": "self","method": "GET"}]}, "emitted_at": 1695051579296} +{"stream": "list_products", "data": {"id": "1647236727","name": "Licenza Oro","create_time": "2022-03-14T05:45:23Z","links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/1647236727","rel": "self","method": "GET"}]}, "emitted_at": 1695051579396} +{"stream": "list_products", "data": {"id": "1647285840","name": "Licenza Premium","create_time": "2022-03-14T19:24:00Z","links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/1647285840","rel": "self","method": "GET"}]}, "emitted_at": 1695051579496} +{"stream": "list_products", "data": {"id": "1647288288","name": "T-Shirt","description": "Blue M","create_time": "2022-03-14T20:04:47Z","links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/1647288288","rel": "self","method": "GET"}]}, "emitted_at": 1695051579596} +{"stream": "search_invoices", "data": {"id":"INV2-BGPS-PKE7-6XSD-YWC7","status":"DRAFT","detail":{"currency_code":"USD","invoice_number":"1706813643331","invoice_date":"2023-12-14","viewed_by_recipient":false,"group_draft":false,"metadata":{"create_time":"2024-02-01T18:54:03Z"}},"primary_recipients":[{"billing_info":{"name":{"given_name":"Shehroz","surname":"Asmat","full_name":"Shehroz Asmat"},"email_address":"sasmat@trythunderbird.com"}}],"amount":{"currency_code":"USD","value":"50.00"},"due_amount":{"currency_code":"USD","value":"50.00"},"links":[{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-BGPS-PKE7-6XSD-YWC7","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-BGPS-PKE7-6XSD-YWC7/send","rel":"send","method":"POST"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-BGPS-PKE7-6XSD-YWC7","rel":"replace","method":"PUT"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-BGPS-PKE7-6XSD-YWC7","rel":"delete","method":"DELETE"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-BGPS-PKE7-6XSD-YWC7/payments","rel":"record-payment","method":"POST"}],"unilateral":false}, "emitted_at": 1695051679296} +{"stream": "search_invoices", "data": {"id":"INV2-6GP9-WLLD-6Q7K-ZYQ2","status":"DRAFT","detail":{"reference":"","currency_code":"USD","note":"","memo":"","invoice_number":"0042","invoice_date":"2022-02-04","payment_term":{"due_date":"2022-02-14"},"viewed_by_recipient":false,"group_draft":false,"metadata":{"create_time":"2024-02-01T11:01:58Z"}},"primary_recipients":[{"billing_info":{"name":{"given_name":"Stephanie","surname":"Meyers","full_name":"Stephanie Meyers"},"email_address":"sb-yww0b28455377@personal.example.com"},"shipping_info":{"name":{"given_name":"Stephanie","surname":"Meyers","full_name":"Stephanie Meyers"}}}],"amount":{"currency_code":"USD","value":"74.21"},"due_amount":{"currency_code":"USD","value":"74.21"},"links":[{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-6GP9-WLLD-6Q7K-ZYQ2","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-6GP9-WLLD-6Q7K-ZYQ2/send","rel":"send","method":"POST"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-6GP9-WLLD-6Q7K-ZYQ2","rel":"replace","method":"PUT"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-6GP9-WLLD-6Q7K-ZYQ2","rel":"delete","method":"DELETE"},{"href":"https://api.sandbox.paypal.com/v2/invoicing/invoices/INV2-6GP9-WLLD-6Q7K-ZYQ2/payments","rel":"record-payment","method":"POST"}],"unilateral":false}, "emitted_at": 1695051779296} +{"stream": "list_payments", "data": {"id":"PAYID-MW55RCA31D103955T218492B","intent":"sale","state":"approved","cart":"06J27273EH485262V","payer":{"payment_method":"paypal","status":"VERIFIED","payer_info":{"email":"sb-vxpcr15413769@personal.example.com","first_name":"John","last_name":"Doe","payer_id":"TWL7BJVYNS7GU","shipping_address":{"recipient_name":"John Doe","line1":"1 Main St","city":"San Jose","state":"CA","postal_code":"95131","country_code":"US"},"phone":"4083068029","country_code":"US"}},"transactions":[{"reference_id":"1000000000047","amount":{"total":"343.80","currency":"USD","details":{"subtotal":"343.80","shipping":"0.00","insurance":"0.00","handling_fee":"0.00","shipping_discount":"0.00","discount":"0.00"}},"payee":{"merchant_id":"C7CYMKZDG8D6E","email":"john_merchant@example.com"},"item_list":{"shipping_address":{"recipient_name":"John Doe","line1":"1 Main St","city":"San Jose","state":"CA","postal_code":"95131","country_code":"US"}},"related_resources":[{"sale":{"id":"7PE037460E080360M","state":"completed","amount":{"total":"343.80","currency":"USD","details":{"subtotal":"343.80","shipping":"0.00","insurance":"0.00","handling_fee":"0.00","shipping_discount":"0.00","discount":"0.00"}},"payment_mode":"INSTANT_TRANSFER","protection_eligibility":"ELIGIBLE","protection_eligibility_type":"ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE","transaction_fee":{"value":"12.49","currency":"USD"},"purchase_unit_reference_id":"1000000000047","parent_payment":"PAYID-MW55RCA31D103955T218492B","create_time":"2024-02-01T17:44:40Z","update_time":"2024-02-01T17:44:40Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/sale/7PE037460E080360M","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/sale/7PE037460E080360M/refund","rel":"refund","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW55RCA31D103955T218492B","rel":"parent_payment","method":"GET"}]}}]}],"create_time":"2024-02-01T17:44:40Z","update_time":"2024-02-01T17:44:40Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW55RCA31D103955T218492B","rel":"self","method":"GET"}]}, "emitted_at": 1695051779296} +{"stream": "list_payments", "data": {"id":"PAYID-MW53UPA6UB45753B0034831X","intent":"sale","state":"approved","cart":"9A220393SG7753433","payer":{"payment_method":"paypal","status":"VERIFIED","payer_info":{"email":"sb-g43l4x28821325@personal.example.com","first_name":"John","last_name":"Doe","payer_id":"889X39VDHV8QY","shipping_address":{"recipient_name":"John Doe","line1":"Via Unit? d'Italia, 5783296","city":"Napoli","state":"Napoli","postal_code":"80127","country_code":"IT"},"phone":"9393358454","country_code":"IT"}},"transactions":[{"reference_id":"default","amount":{"total":"100.00","currency":"USD","details":{"subtotal":"100.00","shipping":"0.00","insurance":"0.00","handling_fee":"0.00","shipping_discount":"0.00","discount":"0.00"}},"payee":{"merchant_id":"C7CYMKZDG8D6E","email":"john_merchant@example.com"},"description":"T-Shirt","item_list":{"items":[{"name":"T-Shirt","description":"Green XL","price":"100.00","currency":"USD","tax":"0.00","quantity":1}],"shipping_address":{"recipient_name":"John Doe","line1":"Via Unit? d'Italia, 5783296","city":"Napoli","state":"Napoli","postal_code":"80127","country_code":"IT"}},"related_resources":[{"sale":{"id":"29N28023XB153584X","state":"completed","amount":{"total":"100.00","currency":"USD","details":{"subtotal":"100.00","shipping":"0.00","insurance":"0.00","handling_fee":"0.00","shipping_discount":"0.00","discount":"0.00"}},"payment_mode":"INSTANT_TRANSFER","protection_eligibility":"ELIGIBLE","protection_eligibility_type":"ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE","transaction_fee":{"value":"5.48","currency":"USD"},"receivable_amount":{"value":"100.00","currency":"USD"},"exchange_rate":"1.098848913950027","purchase_unit_reference_id":"default","parent_payment":"PAYID-MW53UPA6UB45753B0034831X","create_time":"2024-02-01T15:35:25Z","update_time":"2024-02-01T15:35:25Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/sale/29N28023XB153584X","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/sale/29N28023XB153584X/refund","rel":"refund","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW53UPA6UB45753B0034831X","rel":"parent_payment","method":"GET"}]}}]}],"create_time":"2024-02-01T15:35:24Z","update_time":"2024-02-01T15:35:25Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW53UPA6UB45753B0034831X","rel":"self","method":"GET"}]}, "emitted_at": 1695061579296} +{"stream": "list_payments", "data": {"id":"PAY-81S181868H8011217MW526OI","intent":"authorize","state":"approved","payer":{"payment_method":"paypal","status":"VERIFIED","payer_info":{"email":"mihai.streza1@mi-pay.com","first_name":"Mihai","last_name":"Streza","payer_id":"QHD3E8SRDDSQL","shipping_address":{"recipient_name":"Mihai Streza"},"phone":"07534201211","country_code":"GB"}},"transactions":[{"amount":{"total":"20.00","currency":"EUR","details":{"subtotal":"20.00"}},"payee":{"merchant_id":"C7CYMKZDG8D6E"},"description":"topup","invoice_number":"100000000188917","soft_descriptor":"PAYPAL *JOHNMERCHAN","item_list":{"items":[{"name":"topup","price":"20.00","currency":"EUR","tax":"0.00","quantity":1}],"shipping_address":{"recipient_name":"Mihai Streza"}},"related_resources":[{"authorization":{"id":"05D21713M12255848","state":"captured","amount":{"total":"20.00","currency":"EUR","details":{"subtotal":"20.00"}},"payment_mode":"INSTANT_TRANSFER","protection_eligibility":"ELIGIBLE","protection_eligibility_type":"ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE","billing_agreement_id":"B-2B029484VC167663Y","parent_payment":"PAY-81S181868H8011217MW526OI","valid_until":"2024-03-01T14:48:26Z","create_time":"2024-02-01T14:48:26Z","update_time":"2024-02-01T14:48:30Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/capture","rel":"capture","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/void","rel":"void","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/reauthorize","rel":"reauthorize","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI","rel":"parent_payment","method":"GET"}]}},{"capture":{"id":"546282867R0022639","amount":{"total":"20.00","currency":"EUR"},"state":"completed","custom":"","transaction_fee":{"value":"1.39","currency":"EUR"},"parent_payment":"PAY-81S181868H8011217MW526OI","invoice_number":"100000000188917","create_time":"2024-02-01T14:48:30Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/capture/546282867R0022639","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/capture/546282867R0022639/refund","rel":"refund","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848","rel":"authorization","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI","rel":"parent_payment","method":"GET"}]}}]}],"create_time":"2024-02-01T14:48:25Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI","rel":"self","method":"GET"}]}, "emitted_at": 1695071579296} +{"stream": "list_payments", "data": {"id":"PAY-0L38757939422510JMW5ZJVA","intent":"authorize","state":"approved","payer":{"payment_method":"paypal","status":"VERIFIED","payer_info":{"email":"mihai.streza1@mi-pay.com","first_name":"Mihai","last_name":"Streza","payer_id":"QHD3E8SRDDSQL","shipping_address":{"recipient_name":"Mihai Streza"},"phone":"07534201211","country_code":"GB"}},"transactions":[{"amount":{"total":"20.00","currency":"EUR","details":{"subtotal":"20.00"}},"payee":{"merchant_id":"C7CYMKZDG8D6E"},"description":"topup","invoice_number":"100000000188897","soft_descriptor":"PAYPAL *JOHNMERCHAN","item_list":{"items":[{"name":"topup","price":"20.00","currency":"EUR","tax":"0.00","quantity":1}],"shipping_address":{"recipient_name":"Mihai Streza"}},"related_resources":[{"authorization":{"id":"3S025738SW168153S","state":"captured","amount":{"total":"20.00","currency":"EUR","details":{"subtotal":"20.00"}},"payment_mode":"INSTANT_TRANSFER","protection_eligibility":"ELIGIBLE","protection_eligibility_type":"ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE","billing_agreement_id":"B-42217126VD515152H","parent_payment":"PAY-0L38757939422510JMW5ZJVA","valid_until":"2024-03-01T12:55:48Z","create_time":"2024-02-01T12:55:48Z","update_time":"2024-02-01T12:55:51Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/capture","rel":"capture","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/void","rel":"void","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/reauthorize","rel":"reauthorize","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA","rel":"parent_payment","method":"GET"}]}},{"capture":{"id":"26U95072LD470800B","amount":{"total":"20.00","currency":"EUR"},"state":"completed","custom":"","transaction_fee":{"value":"1.39","currency":"EUR"},"parent_payment":"PAY-0L38757939422510JMW5ZJVA","invoice_number":"100000000188897","create_time":"2024-02-01T12:55:51Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/capture/26U95072LD470800B","rel":"self","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/capture/26U95072LD470800B/refund","rel":"refund","method":"POST"},{"href":"https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S","rel":"authorization","method":"GET"},{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA","rel":"parent_payment","method":"GET"}]}}]}],"create_time":"2024-02-01T12:55:48Z","links":[{"href":"https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA","rel":"self","method":"GET"}]}, "emitted_at": 1695081579296} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records_sandbox.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records_sandbox.jsonl new file mode 100644 index 000000000000..74d573c4ee73 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/expected_records_sandbox.jsonl @@ -0,0 +1,85 @@ +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "23N61105X92314351", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-04T17:13:23+0000", "transaction_updated_date": "2021-07-04T17:13:23+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "202.58"}, "available_balance": {"currency_code": "USD", "value": "202.58"}, "invoice_id": "48787580055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "48787580055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "48787580055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-04T17:13:23Z", "transaction_id": "23N61105X92314351"}, "emitted_at": 1707238889137} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1FN09943JY662130R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T22:56:54+0000", "transaction_updated_date": "2021-07-05T22:56:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "231.52"}, "available_balance": {"currency_code": "USD", "value": "231.52"}, "invoice_id": "65095789448", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "65095789448"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "65095789448"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T22:56:54Z", "transaction_id": "1FN09943JY662130R"}, "emitted_at": 1707238889139} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0M443597T0019954R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:01:13+0000", "transaction_updated_date": "2021-07-05T23:01:13+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "260.46"}, "available_balance": {"currency_code": "USD", "value": "260.46"}, "invoice_id": "41468340464", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "41468340464"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "41468340464"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:01:13Z", "transaction_id": "0M443597T0019954R"}, "emitted_at": 1707238889142} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "19C257131E850262B", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:02:46+0000", "transaction_updated_date": "2021-07-05T23:02:46+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "289.40"}, "available_balance": {"currency_code": "USD", "value": "289.40"}, "invoice_id": "23749371955", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "23749371955"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "23749371955"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:02:46Z", "transaction_id": "19C257131E850262B"}, "emitted_at": 1707238889144} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "6S892278N6406494Y", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:06:12+0000", "transaction_updated_date": "2021-07-05T23:06:12+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "318.34"}, "available_balance": {"currency_code": "USD", "value": "318.34"}, "invoice_id": "62173333941", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "62173333941"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "62173333941"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:06:12Z", "transaction_id": "6S892278N6406494Y"}, "emitted_at": 1707238889146} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0T320567TS5587836", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:09:04+0000", "transaction_updated_date": "2021-07-05T23:09:04+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "347.28"}, "available_balance": {"currency_code": "USD", "value": "347.28"}, "invoice_id": "56028534885", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "56028534885"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "56028534885"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:09:04Z", "transaction_id": "0T320567TS5587836"}, "emitted_at": 1707238889148} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3DF69605L9958744R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:12:40+0000", "transaction_updated_date": "2021-07-05T23:12:40+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "376.22"}, "available_balance": {"currency_code": "USD", "value": "376.22"}, "invoice_id": "31766547902", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "31766547902"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "31766547902"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:12:40Z", "transaction_id": "3DF69605L9958744R"}, "emitted_at": 1707238889150} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2F535603PS249601F", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:12:57+0000", "transaction_updated_date": "2021-07-05T23:12:57+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "405.16"}, "available_balance": {"currency_code": "USD", "value": "405.16"}, "invoice_id": "32577611997", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "32577611997"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "32577611997"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:12:57Z", "transaction_id": "2F535603PS249601F"}, "emitted_at": 1707238889153} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "243514451L952570P", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:02+0000", "transaction_updated_date": "2021-07-05T23:14:02+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "434.10"}, "available_balance": {"currency_code": "USD", "value": "434.10"}, "invoice_id": "23612058730", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "23612058730"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "23612058730"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:14:02Z", "transaction_id": "243514451L952570P"}, "emitted_at": 1707238889155} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "27881589Y9461861H", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:19+0000", "transaction_updated_date": "2021-07-05T23:14:19+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "463.04"}, "available_balance": {"currency_code": "USD", "value": "463.04"}, "invoice_id": "53296156982", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "53296156982"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "53296156982"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:14:19Z", "transaction_id": "27881589Y9461861H"}, "emitted_at": 1707238889157} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3MG39755337297727", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:36+0000", "transaction_updated_date": "2021-07-05T23:14:36+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "491.98"}, "available_balance": {"currency_code": "USD", "value": "491.98"}, "invoice_id": "53235397043", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "53235397043"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "53235397043"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:14:36Z", "transaction_id": "3MG39755337297727"}, "emitted_at": 1707238889159} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "32J59182JY5989507", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:52+0000", "transaction_updated_date": "2021-07-05T23:14:52+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "520.92"}, "available_balance": {"currency_code": "USD", "value": "520.92"}, "invoice_id": "18208641465", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "18208641465"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "18208641465"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:14:52Z", "transaction_id": "32J59182JY5989507"}, "emitted_at": 1707238889161} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "52795774C7828234R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:09+0000", "transaction_updated_date": "2021-07-05T23:15:09+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "549.86"}, "available_balance": {"currency_code": "USD", "value": "549.86"}, "invoice_id": "32274344746", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "32274344746"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "32274344746"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:15:09Z", "transaction_id": "52795774C7828234R"}, "emitted_at": 1707238889163} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "19B82038T92822940", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:26+0000", "transaction_updated_date": "2021-07-05T23:15:26+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "578.80"}, "available_balance": {"currency_code": "USD", "value": "578.80"}, "invoice_id": "36419288277", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "36419288277"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "36419288277"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:15:26Z", "transaction_id": "19B82038T92822940"}, "emitted_at": 1707238889166} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "61G749036D552760G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:42+0000", "transaction_updated_date": "2021-07-05T23:15:42+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "607.74"}, "available_balance": {"currency_code": "USD", "value": "607.74"}, "invoice_id": "88092228645", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "88092228645"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "88092228645"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:15:42Z", "transaction_id": "61G749036D552760G"}, "emitted_at": 1707238889168} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5EL311302L108363J", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:58+0000", "transaction_updated_date": "2021-07-05T23:15:58+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "636.68"}, "available_balance": {"currency_code": "USD", "value": "636.68"}, "invoice_id": "25494061224", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "25494061224"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "25494061224"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:15:58Z", "transaction_id": "5EL311302L108363J"}, "emitted_at": 1707238889170} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3VP82838NP358133N", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:16:15+0000", "transaction_updated_date": "2021-07-05T23:16:15+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "665.62"}, "available_balance": {"currency_code": "USD", "value": "665.62"}, "invoice_id": "82173600275", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "82173600275"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "82173600275"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:16:15Z", "transaction_id": "3VP82838NP358133N"}, "emitted_at": 1707238889172} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2N796839EY2539153", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:16:32+0000", "transaction_updated_date": "2021-07-05T23:16:32+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "694.56"}, "available_balance": {"currency_code": "USD", "value": "694.56"}, "invoice_id": "10442581967", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "10442581967"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "10442581967"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:16:32Z", "transaction_id": "2N796839EY2539153"}, "emitted_at": 1707238889174} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5WX252723D093564T", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:29+0000", "transaction_updated_date": "2021-07-05T23:23:29+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "723.50"}, "available_balance": {"currency_code": "USD", "value": "723.50"}, "invoice_id": "71987080514", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71987080514"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71987080514"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:23:29Z", "transaction_id": "5WX252723D093564T"}, "emitted_at": 1707238889176} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "4PW76195NN227720S", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:40+0000", "transaction_updated_date": "2021-07-05T23:23:40+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "752.44"}, "available_balance": {"currency_code": "USD", "value": "752.44"}, "invoice_id": "93025400757", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "93025400757"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "93025400757"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:23:40Z", "transaction_id": "4PW76195NN227720S"}, "emitted_at": 1707238889178} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0VE851712U5895412", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:51+0000", "transaction_updated_date": "2021-07-05T23:23:51+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "781.38"}, "available_balance": {"currency_code": "USD", "value": "781.38"}, "invoice_id": "46225965444", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "46225965444"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "46225965444"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:23:51Z", "transaction_id": "0VE851712U5895412"}, "emitted_at": 1707238889180} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "63U003588S1135607", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:26+0000", "transaction_updated_date": "2021-07-05T23:29:26+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "810.32"}, "available_balance": {"currency_code": "USD", "value": "810.32"}, "invoice_id": "34635559567", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "34635559567"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "34635559567"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:29:26Z", "transaction_id": "63U003588S1135607"}, "emitted_at": 1707238889182} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2AJ081444T051123A", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:37+0000", "transaction_updated_date": "2021-07-05T23:29:37+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "839.26"}, "available_balance": {"currency_code": "USD", "value": "839.26"}, "invoice_id": "92544485996", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "92544485996"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "92544485996"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:29:37Z", "transaction_id": "2AJ081444T051123A"}, "emitted_at": 1707238889184} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2KU13114TJ604181E", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:48+0000", "transaction_updated_date": "2021-07-05T23:29:48+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "868.20"}, "available_balance": {"currency_code": "USD", "value": "868.20"}, "invoice_id": "10184574713", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "10184574713"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "10184574713"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:29:48Z", "transaction_id": "2KU13114TJ604181E"}, "emitted_at": 1707238889186} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1ST090036H2235215", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:35+0000", "transaction_updated_date": "2021-07-05T23:31:35+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "897.14"}, "available_balance": {"currency_code": "USD", "value": "897.14"}, "invoice_id": "50350860865", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "50350860865"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "50350860865"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:31:35Z", "transaction_id": "1ST090036H2235215"}, "emitted_at": 1707238889188} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5BJ418934Y425901G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:46+0000", "transaction_updated_date": "2021-07-05T23:31:46+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "926.08"}, "available_balance": {"currency_code": "USD", "value": "926.08"}, "invoice_id": "12278283055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "12278283055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "12278283055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:31:46Z", "transaction_id": "5BJ418934Y425901G"}, "emitted_at": 1707238889190} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0SD21997LN026020M", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:56+0000", "transaction_updated_date": "2021-07-05T23:31:56+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "955.02"}, "available_balance": {"currency_code": "USD", "value": "955.02"}, "invoice_id": "52396214250", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "52396214250"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "52396214250"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:31:56Z", "transaction_id": "0SD21997LN026020M"}, "emitted_at": 1707238889192} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3BH630398E562901G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:42:41+0000", "transaction_updated_date": "2021-07-05T23:42:41+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "983.96"}, "available_balance": {"currency_code": "USD", "value": "983.96"}, "invoice_id": "18793521512", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "18793521512"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "18793521512"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:42:41Z", "transaction_id": "3BH630398E562901G"}, "emitted_at": 1707238889194} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "03D88325GF8461705", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:42:52+0000", "transaction_updated_date": "2021-07-05T23:42:52+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1012.90"}, "available_balance": {"currency_code": "USD", "value": "1012.90"}, "invoice_id": "71793513892", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71793513892"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71793513892"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:42:52Z", "transaction_id": "03D88325GF8461705"}, "emitted_at": 1707238889196} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "51852852PL0100404", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:43:03+0000", "transaction_updated_date": "2021-07-05T23:43:03+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1041.84"}, "available_balance": {"currency_code": "USD", "value": "1041.84"}, "invoice_id": "98653187889", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "98653187889"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "98653187889"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:43:03Z", "transaction_id": "51852852PL0100404"}, "emitted_at": 1707238889198} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "8MF4324694292993B", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:21+0000", "transaction_updated_date": "2021-07-05T23:44:21+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1070.78"}, "available_balance": {"currency_code": "USD", "value": "1070.78"}, "invoice_id": "12489150471", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "12489150471"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "12489150471"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:44:21Z", "transaction_id": "8MF4324694292993B"}, "emitted_at": 1707238889201} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "87S73342AS6001233", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:32+0000", "transaction_updated_date": "2021-07-05T23:44:32+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1099.72"}, "available_balance": {"currency_code": "USD", "value": "1099.72"}, "invoice_id": "99595079917", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "99595079917"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "99595079917"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:44:32Z", "transaction_id": "87S73342AS6001233"}, "emitted_at": 1707238889203} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "112146346A741221U", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:44+0000", "transaction_updated_date": "2021-07-05T23:44:44+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1128.66"}, "available_balance": {"currency_code": "USD", "value": "1128.66"}, "invoice_id": "93286331651", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "93286331651"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "93286331651"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:44:44Z", "transaction_id": "112146346A741221U"}, "emitted_at": 1707238889205} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0N2242037Y9449344", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:54+0000", "transaction_updated_date": "2021-07-05T23:44:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1157.60"}, "available_balance": {"currency_code": "USD", "value": "1157.60"}, "invoice_id": "71349988314", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71349988314"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71349988314"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:44:54Z", "transaction_id": "0N2242037Y9449344"}, "emitted_at": 1707238889207} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9NH78349H0388780F", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:05+0000", "transaction_updated_date": "2021-07-05T23:45:05+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1186.54"}, "available_balance": {"currency_code": "USD", "value": "1186.54"}, "invoice_id": "83951023481", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "83951023481"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "83951023481"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:45:05Z", "transaction_id": "9NH78349H0388780F"}, "emitted_at": 1707238889209} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "10S137566E4828249", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:16+0000", "transaction_updated_date": "2021-07-05T23:45:16+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1215.48"}, "available_balance": {"currency_code": "USD", "value": "1215.48"}, "invoice_id": "88168198250", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "88168198250"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "88168198250"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:45:16Z", "transaction_id": "10S137566E4828249"}, "emitted_at": 1707238889211} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "7N749695W59419057", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:27+0000", "transaction_updated_date": "2021-07-05T23:45:27+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1244.42"}, "available_balance": {"currency_code": "USD", "value": "1244.42"}, "invoice_id": "38296993497", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "38296993497"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "38296993497"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:45:27Z", "transaction_id": "7N749695W59419057"}, "emitted_at": 1707238889213} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "43X058357A257931N", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:39+0000", "transaction_updated_date": "2021-07-05T23:45:39+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1273.36"}, "available_balance": {"currency_code": "USD", "value": "1273.36"}, "invoice_id": "33391419042", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "33391419042"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "33391419042"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:45:39Z", "transaction_id": "43X058357A257931N"}, "emitted_at": 1707238889215} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5WL82051VY277550S", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:50+0000", "transaction_updated_date": "2021-07-05T23:45:50+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1302.30"}, "available_balance": {"currency_code": "USD", "value": "1302.30"}, "invoice_id": "69341308548", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "69341308548"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "69341308548"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:45:50Z", "transaction_id": "5WL82051VY277550S"}, "emitted_at": 1707238889217} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9CG36572NK0728016", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:46:01+0000", "transaction_updated_date": "2021-07-05T23:46:01+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1331.24"}, "available_balance": {"currency_code": "USD", "value": "1331.24"}, "invoice_id": "70491310163", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "70491310163"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "70491310163"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:46:01Z", "transaction_id": "9CG36572NK0728016"}, "emitted_at": 1707238889219} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9K759703FU663194K", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:46:43+0000", "transaction_updated_date": "2021-07-05T23:46:43+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1360.18"}, "available_balance": {"currency_code": "USD", "value": "1360.18"}, "invoice_id": "44794712899", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "44794712899"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "44794712899"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:46:43Z", "transaction_id": "9K759703FU663194K"}, "emitted_at": 1707238889222} +{"stream": "balances", "data": {"balances": [{"currency": "USD", "primary": true, "total_balance": {"currency_code": "USD", "value": "173.64"}, "available_balance": {"currency_code": "USD", "value": "173.64"}, "withheld_balance": {"currency_code": "USD", "value": "0.00"}}], "account_id": "MDXWPD67GEP5W", "as_of_time": "2021-07-01T00:00:00Z", "last_refresh_time": "2024-02-06T09:59:59Z"}, "emitted_at": 1707239624675} +{"stream": "list_products", "data": {"id": "bELKLtzpiO", "name": "Pines-T-Shirt-sSUVV", "description": "Anothe rUpdate. Let's see if something changes or not", "create_time": "2024-01-25T20:16:36Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/bELKLtzpiO", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239715683} +{"stream": "list_products", "data": {"id": "oObfKeWCTO", "name": "Pines-T-Shirt-HZBfE", "description": "My Update. Does it changes it?", "create_time": "2024-01-26T20:07:11Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/oObfKeWCTO", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239715685} +{"stream": "list_products", "data": {"id": "GXASPmLVeA", "name": "Pines-T-Shirt-vtpna", "description": "Cotton XL", "create_time": "2024-01-26T23:00:52Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/catalogs/products/GXASPmLVeA", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239715687} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTWI4W4210034A4751008", "intent": "sale", "state": "approved", "cart": "94M68693F5918712T", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "44794712899", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "9K759703FU663194K", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTWI4W4210034A4751008", "create_time": "2021-07-05T23:46:43Z", "update_time": "2021-07-05T23:46:43Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/9K759703FU663194K", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/9K759703FU663194K/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTWI4W4210034A4751008", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:46:33Z", "update_time": "2021-07-05T23:46:43Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTWI4W4210034A4751008", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814718} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTMA8WD99999KH443990H", "intent": "sale", "state": "approved", "cart": "8BD080945B510293Y", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "70491310163", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "9CG36572NK0728016", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTMA8WD99999KH443990H", "create_time": "2021-07-05T23:46:01Z", "update_time": "2021-07-05T23:46:01Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/9CG36572NK0728016", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/9CG36572NK0728016/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTMA8WD99999KH443990H", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:45:52Z", "update_time": "2021-07-05T23:46:01Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTMA8WD99999KH443990H", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814721} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTJA3GY064023P159724A", "intent": "sale", "state": "approved", "cart": "4XU51704H63205015", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "69341308548", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "5WL82051VY277550S", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTJA3GY064023P159724A", "create_time": "2021-07-05T23:45:50Z", "update_time": "2021-07-05T23:45:50Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/5WL82051VY277550S", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/5WL82051VY277550S/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTJA3GY064023P159724A", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:45:40Z", "update_time": "2021-07-05T23:45:50Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTJA3GY064023P159724A", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814724} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTGI9UK71240VU8020618", "intent": "sale", "state": "approved", "cart": "0RJ70999M3464401U", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "33391419042", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "43X058357A257931N", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTGI9UK71240VU8020618", "create_time": "2021-07-05T23:45:39Z", "update_time": "2021-07-05T23:45:39Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/43X058357A257931N", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/43X058357A257931N/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTGI9UK71240VU8020618", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:45:29Z", "update_time": "2021-07-05T23:45:39Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTGI9UK71240VU8020618", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814727} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTDQ5FS07530F9282151W", "intent": "sale", "state": "approved", "cart": "4JA813678M224812W", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "38296993497", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "7N749695W59419057", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTDQ5FS07530F9282151W", "create_time": "2021-07-05T23:45:27Z", "update_time": "2021-07-05T23:45:27Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/7N749695W59419057", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/7N749695W59419057/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTDQ5FS07530F9282151W", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:45:18Z", "update_time": "2021-07-05T23:45:27Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTDQ5FS07530F9282151W", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814730} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZTAY2N7514653J745150Y", "intent": "sale", "state": "approved", "cart": "4VA72722KX1105219", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "88168198250", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "10S137566E4828249", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZTAY2N7514653J745150Y", "create_time": "2021-07-05T23:45:16Z", "update_time": "2021-07-05T23:45:16Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/10S137566E4828249", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/10S137566E4828249/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTAY2N7514653J745150Y", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:45:07Z", "update_time": "2021-07-05T23:45:16Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZTAY2N7514653J745150Y", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814733} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZS6A3M9912977L184341S", "intent": "sale", "state": "approved", "cart": "4JY24848DR959552K", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "83951023481", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "9NH78349H0388780F", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZS6A3M9912977L184341S", "create_time": "2021-07-05T23:45:05Z", "update_time": "2021-07-05T23:45:05Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/9NH78349H0388780F", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/9NH78349H0388780F/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZS6A3M9912977L184341S", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:44:56Z", "update_time": "2021-07-05T23:45:05Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZS6A3M9912977L184341S", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814735} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZS3I50756242KE8498014", "intent": "sale", "state": "approved", "cart": "3PS97109CA147652A", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "71349988314", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "0N2242037Y9449344", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZS3I50756242KE8498014", "create_time": "2021-07-05T23:44:54Z", "update_time": "2021-07-05T23:44:54Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/0N2242037Y9449344", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/0N2242037Y9449344/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZS3I50756242KE8498014", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:44:45Z", "update_time": "2021-07-05T23:44:54Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZS3I50756242KE8498014", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814738} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZSYQ8A757181MC817782A", "intent": "sale", "state": "approved", "cart": "4KH490105A375162P", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "93286331651", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "112146346A741221U", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZSYQ8A757181MC817782A", "create_time": "2021-07-05T23:44:44Z", "update_time": "2021-07-05T23:44:44Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/112146346A741221U", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/112146346A741221U/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSYQ8A757181MC817782A", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:44:34Z", "update_time": "2021-07-05T23:44:44Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSYQ8A757181MC817782A", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814741} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZSVY92X22655TL083163M", "intent": "sale", "state": "approved", "cart": "1K4366307V579334M", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "99595079917", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "87S73342AS6001233", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZSVY92X22655TL083163M", "create_time": "2021-07-05T23:44:32Z", "update_time": "2021-07-05T23:44:32Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/87S73342AS6001233", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/87S73342AS6001233/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSVY92X22655TL083163M", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:44:23Z", "update_time": "2021-07-05T23:44:32Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSVY92X22655TL083163M", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814743} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZSSY3AX23091XE567400S", "intent": "sale", "state": "approved", "cart": "5FW555787Y189635P", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "12489150471", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "8MF4324694292993B", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZSSY3AX23091XE567400S", "create_time": "2021-07-05T23:44:21Z", "update_time": "2021-07-05T23:44:21Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/8MF4324694292993B", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/8MF4324694292993B/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSSY3AX23091XE567400S", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:44:11Z", "update_time": "2021-07-05T23:44:21Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZSSY3AX23091XE567400S", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814746} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZR7Q47A34643F6939564E", "intent": "sale", "state": "approved", "cart": "3CY93437B5650311M", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "98653187889", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "51852852PL0100404", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZR7Q47A34643F6939564E", "create_time": "2021-07-05T23:43:03Z", "update_time": "2021-07-05T23:43:03Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/51852852PL0100404", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/51852852PL0100404/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZR7Q47A34643F6939564E", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:42:54Z", "update_time": "2021-07-05T23:43:03Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZR7Q47A34643F6939564E", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814749} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZR4Y96946691TM615464M", "intent": "sale", "state": "approved", "cart": "0WG02233VF623084T", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "71793513892", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "03D88325GF8461705", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZR4Y96946691TM615464M", "create_time": "2021-07-05T23:42:52Z", "update_time": "2021-07-05T23:42:52Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/03D88325GF8461705", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/03D88325GF8461705/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZR4Y96946691TM615464M", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:42:43Z", "update_time": "2021-07-05T23:42:52Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZR4Y96946691TM615464M", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814751} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZRZY3BY320090L305572M", "intent": "sale", "state": "approved", "cart": "4RY36168C0517143X", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "18793521512", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "3BH630398E562901G", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZRZY3BY320090L305572M", "create_time": "2021-07-05T23:42:41Z", "update_time": "2021-07-05T23:42:41Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/3BH630398E562901G", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/3BH630398E562901G/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZRZY3BY320090L305572M", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:42:31Z", "update_time": "2021-07-05T23:42:41Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZRZY3BY320090L305572M", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814754} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZMYY6H275806JK843330N", "intent": "sale", "state": "approved", "cart": "9RC74442JD1611818", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "52396214250", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "0SD21997LN026020M", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZMYY6H275806JK843330N", "create_time": "2021-07-05T23:31:56Z", "update_time": "2021-07-05T23:31:56Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/0SD21997LN026020M", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/0SD21997LN026020M/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMYY6H275806JK843330N", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:31:47Z", "update_time": "2021-07-05T23:31:56Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMYY6H275806JK843330N", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814756} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZMWA7HV06523LM984963Y", "intent": "sale", "state": "approved", "cart": "4HC02299JX0560832", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "12278283055", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "5BJ418934Y425901G", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZMWA7HV06523LM984963Y", "create_time": "2021-07-05T23:31:46Z", "update_time": "2021-07-05T23:31:46Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/5BJ418934Y425901G", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/5BJ418934Y425901G/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMWA7HV06523LM984963Y", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:31:36Z", "update_time": "2021-07-05T23:31:46Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMWA7HV06523LM984963Y", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814758} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZMTI4X39217868749053E", "intent": "sale", "state": "approved", "cart": "5N3647919D810380T", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "50350860865", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "1ST090036H2235215", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZMTI4X39217868749053E", "create_time": "2021-07-05T23:31:35Z", "update_time": "2021-07-05T23:31:35Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/1ST090036H2235215", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/1ST090036H2235215/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMTI4X39217868749053E", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:31:25Z", "update_time": "2021-07-05T23:31:35Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZMTI4X39217868749053E", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814760} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZLYY9RB64928LK030220F", "intent": "sale", "state": "approved", "cart": "31F66461BJ2308523", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "10184574713", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "2KU13114TJ604181E", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZLYY9RB64928LK030220F", "create_time": "2021-07-05T23:29:48Z", "update_time": "2021-07-05T23:29:48Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/2KU13114TJ604181E", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/2KU13114TJ604181E/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLYY9RB64928LK030220F", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:29:39Z", "update_time": "2021-07-05T23:29:48Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLYY9RB64928LK030220F", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814763} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZLWA09E67167RS362822V", "intent": "sale", "state": "approved", "cart": "1B944298P8820581S", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "92544485996", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "2AJ081444T051123A", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZLWA09E67167RS362822V", "create_time": "2021-07-05T23:29:37Z", "update_time": "2021-07-05T23:29:37Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/2AJ081444T051123A", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/2AJ081444T051123A/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLWA09E67167RS362822V", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:29:28Z", "update_time": "2021-07-05T23:29:37Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLWA09E67167RS362822V", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814765} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZLTA1857385502613571B", "intent": "sale", "state": "approved", "cart": "07W53209604004921", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "34635559567", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "63U003588S1135607", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZLTA1857385502613571B", "create_time": "2021-07-05T23:29:26Z", "update_time": "2021-07-05T23:29:26Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/63U003588S1135607", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/63U003588S1135607/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLTA1857385502613571B", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:29:16Z", "update_time": "2021-07-05T23:29:26Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZLTA1857385502613571B", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239814767} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZI7Q61590832VE058581A", "intent": "sale", "state": "approved", "cart": "47F08854HR292433A", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "46225965444", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "0VE851712U5895412", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZI7Q61590832VE058581A", "create_time": "2021-07-05T23:23:51Z", "update_time": "2021-07-05T23:23:51Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/0VE851712U5895412", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/0VE851712U5895412/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZI7Q61590832VE058581A", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:23:42Z", "update_time": "2021-07-05T23:23:51Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZI7Q61590832VE058581A", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816765} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZI4Y1FS16045MX748772D", "intent": "sale", "state": "approved", "cart": "6JA90337U5912522U", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "93025400757", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "4PW76195NN227720S", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZI4Y1FS16045MX748772D", "create_time": "2021-07-05T23:23:40Z", "update_time": "2021-07-05T23:23:40Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/4PW76195NN227720S", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/4PW76195NN227720S/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZI4Y1FS16045MX748772D", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:23:31Z", "update_time": "2021-07-05T23:23:40Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZI4Y1FS16045MX748772D", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816770} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZIZY0YE90251N2470191A", "intent": "sale", "state": "approved", "cart": "79F61111E7201332F", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "71987080514", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "5WX252723D093564T", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZIZY0YE90251N2470191A", "create_time": "2021-07-05T23:23:29Z", "update_time": "2021-07-05T23:23:29Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/5WX252723D093564T", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/5WX252723D093564T/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZIZY0YE90251N2470191A", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:23:19Z", "update_time": "2021-07-05T23:23:29Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZIZY0YE90251N2470191A", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816774} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZFQI6GD87350UE545773N", "intent": "sale", "state": "approved", "cart": "1FP43302U0478942X", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "10442581967", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "2N796839EY2539153", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZFQI6GD87350UE545773N", "create_time": "2021-07-05T23:16:32Z", "update_time": "2021-07-05T23:16:32Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/2N796839EY2539153", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/2N796839EY2539153/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFQI6GD87350UE545773N", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:16:17Z", "update_time": "2021-07-05T23:16:32Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFQI6GD87350UE545773N", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816778} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZFMA0B969784C6725002V", "intent": "sale", "state": "approved", "cart": "8L091901278803646", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "82173600275", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "3VP82838NP358133N", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZFMA0B969784C6725002V", "create_time": "2021-07-05T23:16:15Z", "update_time": "2021-07-05T23:16:15Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/3VP82838NP358133N", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/3VP82838NP358133N/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFMA0B969784C6725002V", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:16:00Z", "update_time": "2021-07-05T23:16:15Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFMA0B969784C6725002V", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816781} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZFHY7SJ7565124295670P", "intent": "sale", "state": "approved", "cart": "2SM08874YP816313V", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "25494061224", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "5EL311302L108363J", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZFHY7SJ7565124295670P", "create_time": "2021-07-05T23:15:58Z", "update_time": "2021-07-05T23:15:58Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/5EL311302L108363J", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/5EL311302L108363J/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFHY7SJ7565124295670P", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:15:43Z", "update_time": "2021-07-05T23:15:58Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFHY7SJ7565124295670P", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816785} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZFDY16477729A65752911", "intent": "sale", "state": "approved", "cart": "31E8174415946984X", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "88092228645", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "61G749036D552760G", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZFDY16477729A65752911", "create_time": "2021-07-05T23:15:42Z", "update_time": "2021-07-05T23:15:42Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/61G749036D552760G", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/61G749036D552760G/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFDY16477729A65752911", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:15:27Z", "update_time": "2021-07-05T23:15:42Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZFDY16477729A65752911", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816788} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZE7Y09K97910J19191835", "intent": "sale", "state": "approved", "cart": "52852967MK8398427", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "36419288277", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "19B82038T92822940", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZE7Y09K97910J19191835", "create_time": "2021-07-05T23:15:26Z", "update_time": "2021-07-05T23:15:26Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/19B82038T92822940", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/19B82038T92822940/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZE7Y09K97910J19191835", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:15:11Z", "update_time": "2021-07-05T23:15:26Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZE7Y09K97910J19191835", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816791} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZE3Q7MV3302078506752V", "intent": "sale", "state": "approved", "cart": "8DT33424FM6083023", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "32274344746", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "52795774C7828234R", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZE3Q7MV3302078506752V", "create_time": "2021-07-05T23:15:09Z", "update_time": "2021-07-05T23:15:09Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/52795774C7828234R", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/52795774C7828234R/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZE3Q7MV3302078506752V", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:14:54Z", "update_time": "2021-07-05T23:15:09Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZE3Q7MV3302078506752V", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816794} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZEXI6AC38094H2421312E", "intent": "sale", "state": "approved", "cart": "934109456G4946916", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "18208641465", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "32J59182JY5989507", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZEXI6AC38094H2421312E", "create_time": "2021-07-05T23:14:52Z", "update_time": "2021-07-05T23:14:52Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/32J59182JY5989507", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/32J59182JY5989507/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEXI6AC38094H2421312E", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:14:37Z", "update_time": "2021-07-05T23:14:52Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEXI6AC38094H2421312E", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816797} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZETI37A91239LN8311810", "intent": "sale", "state": "approved", "cart": "83D98649PY104524E", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "53235397043", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "3MG39755337297727", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZETI37A91239LN8311810", "create_time": "2021-07-05T23:14:36Z", "update_time": "2021-07-05T23:14:36Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/3MG39755337297727", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/3MG39755337297727/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZETI37A91239LN8311810", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:14:21Z", "update_time": "2021-07-05T23:14:36Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZETI37A91239LN8311810", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816799} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZEPA540480516L1460710", "intent": "sale", "state": "approved", "cart": "7GV22540700208350", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "53296156982", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "27881589Y9461861H", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZEPA540480516L1460710", "create_time": "2021-07-05T23:14:19Z", "update_time": "2021-07-05T23:14:19Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/27881589Y9461861H", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/27881589Y9461861H/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEPA540480516L1460710", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:14:04Z", "update_time": "2021-07-05T23:14:19Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEPA540480516L1460710", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816802} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZEKQ6CT67951NS8257540", "intent": "sale", "state": "approved", "cart": "9GV5180502759472M", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "23612058730", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "243514451L952570P", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZEKQ6CT67951NS8257540", "create_time": "2021-07-05T23:14:02Z", "update_time": "2021-07-05T23:14:02Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/243514451L952570P", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/243514451L952570P/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEKQ6CT67951NS8257540", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:13:46Z", "update_time": "2021-07-05T23:14:02Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZEKQ6CT67951NS8257540", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816805} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZD2Q3A0909139J3152203", "intent": "sale", "state": "approved", "cart": "6AJ421654S4873922", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "32577611997", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "2F535603PS249601F", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZD2Q3A0909139J3152203", "create_time": "2021-07-05T23:12:57Z", "update_time": "2021-07-05T23:12:57Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/2F535603PS249601F", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/2F535603PS249601F/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZD2Q3A0909139J3152203", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:12:42Z", "update_time": "2021-07-05T23:12:57Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZD2Q3A0909139J3152203", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816807} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZDWI4V162188CW455701G", "intent": "sale", "state": "approved", "cart": "6D196635J17794229", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "31766547902", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "3DF69605L9958744R", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZDWI4V162188CW455701G", "create_time": "2021-07-05T23:12:40Z", "update_time": "2021-07-05T23:12:40Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/3DF69605L9958744R", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/3DF69605L9958744R/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZDWI4V162188CW455701G", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:12:25Z", "update_time": "2021-07-05T23:12:40Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZDWI4V162188CW455701G", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816810} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZB7Y8EC76174AW4613112", "intent": "sale", "state": "approved", "cart": "6FK6522150836620E", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "56028534885", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "0T320567TS5587836", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZB7Y8EC76174AW4613112", "create_time": "2021-07-05T23:09:04Z", "update_time": "2021-07-05T23:09:04Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/0T320567TS5587836", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/0T320567TS5587836/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZB7Y8EC76174AW4613112", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:08:47Z", "update_time": "2021-07-05T23:09:04Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZB7Y8EC76174AW4613112", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816813} +{"stream": "list_payments", "data": {"id": "PAYID-MDRZAUY1KV14872K8421472G", "intent": "sale", "state": "approved", "cart": "19W34946AD354311P", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "62173333941", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "6S892278N6406494Y", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRZAUY1KV14872K8421472G", "create_time": "2021-07-05T23:06:12Z", "update_time": "2021-07-05T23:06:12Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/6S892278N6406494Y", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/6S892278N6406494Y/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZAUY1KV14872K8421472G", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:05:55Z", "update_time": "2021-07-05T23:06:12Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRZAUY1KV14872K8421472G", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816816} +{"stream": "list_payments", "data": {"id": "PAYID-MDRY7BY3W991940VB486043B", "intent": "sale", "state": "approved", "cart": "1B350164A39756210", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "23749371955", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "19C257131E850262B", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRY7BY3W991940VB486043B", "create_time": "2021-07-05T23:02:46Z", "update_time": "2021-07-05T23:02:46Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/19C257131E850262B", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/19C257131E850262B/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY7BY3W991940VB486043B", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:02:31Z", "update_time": "2021-07-05T23:02:46Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY7BY3W991940VB486043B", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816818} +{"stream": "list_payments", "data": {"id": "PAYID-MDRY6KI3Y0452638X128114D", "intent": "sale", "state": "approved", "cart": "19Y331413K071511U", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "41468340464", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "0M443597T0019954R", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRY6KI3Y0452638X128114D", "create_time": "2021-07-05T23:01:13Z", "update_time": "2021-07-05T23:01:13Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/0M443597T0019954R", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/0M443597T0019954R/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY6KI3Y0452638X128114D", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T23:00:57Z", "update_time": "2021-07-05T23:01:13Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY6KI3Y0452638X128114D", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816820} +{"stream": "list_payments", "data": {"id": "PAYID-MDRY4JY42461888J3529050W", "intent": "sale", "state": "approved", "cart": "3NN161683Y756203T", "payer": {"payment_method": "paypal", "status": "VERIFIED", "payer_info": {"email": "integration-test-buyer@airbyte.io", "first_name": "test", "last_name": "buyer", "payer_id": "ZE5533HZPGMC6", "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}, "phone": "4086104434", "country_code": "US"}}, "transactions": [{"amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payee": {"merchant_id": "MDXWPD67GEP5W", "email": "integration-test-facilitator@airbyte.io"}, "description": "This is the payment transaction description.", "custom": "EBAY_EMS_90048630020055", "invoice_number": "65095789448", "item_list": {"items": [{"name": "hat", "sku": "1", "description": "Brown color hat", "price": "3.00", "currency": "USD", "tax": "0.01", "quantity": 5}, {"name": "handbag", "sku": "product34", "description": "Black color hand bag", "price": "15.00", "currency": "USD", "tax": "0.02", "quantity": 1}], "shipping_address": {"recipient_name": "Hello World", "line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "postal_code": "95131", "country_code": "US"}}, "related_resources": [{"sale": {"id": "1FN09943JY662130R", "state": "completed", "amount": {"total": "30.11", "currency": "USD", "details": {"subtotal": "30.00", "tax": "0.07", "shipping": "0.03", "insurance": "0.01", "handling_fee": "1.00", "shipping_discount": "-1.00", "discount": "0.00"}}, "payment_mode": "INSTANT_TRANSFER", "protection_eligibility": "ELIGIBLE", "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", "transaction_fee": {"value": "1.17", "currency": "USD"}, "parent_payment": "PAYID-MDRY4JY42461888J3529050W", "create_time": "2021-07-05T22:56:54Z", "update_time": "2021-07-05T22:56:54Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/sale/1FN09943JY662130R", "rel": "self", "method": "GET"}, {"href": "https://api.sandbox.paypal.com/v1/payments/sale/1FN09943JY662130R/refund", "rel": "refund", "method": "POST"}, {"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY4JY42461888J3529050W", "rel": "parent_payment", "method": "GET"}]}}]}], "create_time": "2021-07-05T22:56:39Z", "update_time": "2021-07-05T22:56:54Z", "links": [{"href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MDRY4JY42461888J3529050W", "rel": "self", "method": "GET"}]}, "emitted_at": 1707239816823} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/invalid_config.json similarity index 100% rename from airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json rename to airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/invalid_config.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/sample_config.json new file mode 100644 index 000000000000..9cfcf4147ffe --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/sample_config.json @@ -0,0 +1,12 @@ +{ + "client_id": "PAYPAL_CLIENT_ID", + "client_secret": "PAYPAL_SECRET", + "start_date": "2024-01-20T00:00:00Z", + "end_date": "2024-02-01T23:59:00Z", + "dispute_start_date": "2024-02-01T00:00:00.000Z", + "dispute_end_date": "2024-02-05T23:59:00.000Z", + "is_sandbox": true, + "buyer_username": "BUYER_USERNAME@SOMETHING.com", + "buyer_password": "BUYER_PASSWORD", + "payer_id": "ACCOUNT_ID" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/sample_state.json similarity index 100% rename from airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json rename to airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/sample_state.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/state.json similarity index 100% rename from airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json rename to airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_files/state.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 51be49033dca..06823a4a71e5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_paypal_transaction import SourcePaypalTransaction +from source_paypal_transaction.run import run if __name__ == "__main__": - source = SourcePaypalTransaction() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml index 1821fdddaddc..a1974a83fbe7 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml @@ -1,6 +1,6 @@ data: ab_internal: - ql: 200 + ql: 400 sl: 200 allowedHosts: hosts: @@ -11,13 +11,17 @@ data: connectorSubtype: api connectorType: source definitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 - dockerImageTag: 2.2.0 + dockerImageTag: 2.4.0 dockerRepository: airbyte/source-paypal-transaction documentationUrl: https://docs.airbyte.com/integrations/sources/paypal-transaction githubIssueLabel: source-paypal-transaction icon: paypal.svg license: MIT name: Paypal Transaction + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-paypal-transaction registries: cloud: enabled: true @@ -30,6 +34,11 @@ data: 2.1.0: message: 'Version 2.1.0 changes the format of the state. The format of the cursor changed from "2021-06-18T16:24:13+03:00" to "2021-06-18T16:24:13Z". The state key for the transactions stream changed to "transaction_updated_date" and the key for the balances stream change to "as_of_time". The upgrade is safe, but rolling back is not.' upgradeDeadline: "2023-09-18" + suggestedStreams: + streams: + - transactions + - balances + - list_payments supportLevel: certified tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-paypal-transaction/poetry.lock b/airbyte-integrations/connectors/source-paypal-transaction/poetry.lock new file mode 100644 index 000000000000..4200fc8953a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/poetry.lock @@ -0,0 +1,1216 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "outcome" +version = "1.3.0.post0" +description = "Capture the outcome of Python function calls." +optional = false +python-versions = ">=3.7" +files = [ + {file = "outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b"}, + {file = "outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8"}, +] + +[package.dependencies] +attrs = ">=19.2.0" + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pysocks" +version = "1.7.1" +description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"}, + {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"}, + {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, +] + +[[package]] +name = "pytest" +version = "8.0.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.1-py3-none-any.whl", hash = "sha256:3e4f16fe1c0a9dc9d9389161c127c3edc5d810c38d6793042fb81d9f48a59fca"}, + {file = "pytest-8.0.1.tar.gz", hash = "sha256:267f6563751877d772019b13aacbe4e860d73fe8f651f28112e9ac37de7513ae"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "selenium" +version = "4.18.1" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "selenium-4.18.1-py3-none-any.whl", hash = "sha256:b24a3cdd2d47c29832e81345bfcde0c12bb608738013e53c781b211b418df241"}, + {file = "selenium-4.18.1.tar.gz", hash = "sha256:a11f67afa8bfac6b77e148c987b33f6b14eb1cae4d352722a75de1f26e3f0ae2"}, +] + +[package.dependencies] +certifi = ">=2021.10.8" +trio = ">=0.17,<1.0" +trio-websocket = ">=0.9,<1.0" +typing_extensions = ">=4.9.0" +urllib3 = {version = ">=1.26,<3", extras = ["socks"]} + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "trio" +version = "0.24.0" +description = "A friendly Python library for async concurrency and I/O" +optional = false +python-versions = ">=3.8" +files = [ + {file = "trio-0.24.0-py3-none-any.whl", hash = "sha256:c3bd3a4e3e3025cd9a2241eae75637c43fe0b9e88b4c97b9161a55b9e54cd72c"}, + {file = "trio-0.24.0.tar.gz", hash = "sha256:ffa09a74a6bf81b84f8613909fb0beaee84757450183a7a2e0b47b455c0cac5d"}, +] + +[package.dependencies] +attrs = ">=20.1.0" +cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""} +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +idna = "*" +outcome = "*" +sniffio = ">=1.3.0" +sortedcontainers = "*" + +[[package]] +name = "trio-websocket" +version = "0.11.1" +description = "WebSocket library for Trio" +optional = false +python-versions = ">=3.7" +files = [ + {file = "trio-websocket-0.11.1.tar.gz", hash = "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f"}, + {file = "trio_websocket-0.11.1-py3-none-any.whl", hash = "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638"}, +] + +[package.dependencies] +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +trio = ">=0.11" +wsproto = ">=0.14" + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.dependencies] +pysocks = {version = ">=1.5.6,<1.5.7 || >1.5.7,<2.0", optional = true, markers = "extra == \"socks\""} + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "wsproto" +version = "1.2.0" +description = "WebSockets state-machine based protocol implementation" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"}, + {file = "wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065"}, +] + +[package.dependencies] +h11 = ">=0.9.0,<1" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "3e201a0e17ac40faf9a73f3e2b9e924e0d3993ee8982646436ea6517f8f6cd7b" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/pyproject.toml b/airbyte-integrations/connectors/source-paypal-transaction/pyproject.toml new file mode 100644 index 000000000000..36ea7e8161e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.4.0" +name = "source-paypal-transaction" +description = "Source implementation for Paypal Transaction." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/paypal-transaction" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_paypal_transaction" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.63.2" + + +[tool.poetry.scripts] +source-paypal-transaction = "source_paypal_transaction.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.0" +pytest-mock = "^3.12" +requests-mock = "^1.11.0" +selenium = "^4.17.2" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py deleted file mode 100644 index 3ecf3c37436c..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.51.44", -] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6", - "requests-mock", -] - -setup( - name="source_paypal_transaction", - description="Source implementation for Paypal Transaction.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py index 332549f3b617..af883e9c1c19 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py @@ -5,10 +5,14 @@ import base64 import logging from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Iterable, Mapping, MutableMapping, Optional import backoff import requests from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester +from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException logger = logging.getLogger("airbyte") @@ -42,6 +46,7 @@ def get_headers(self): @backoff.on_exception( backoff.expo, DefaultBackoffException, + max_tries=2, on_backoff=lambda details: logger.info( f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." ), @@ -49,14 +54,25 @@ def get_headers(self): ) def _get_refresh_access_token_response(self): try: - response = requests.request( - method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body(), headers=self.get_headers() - ) + request_url = self.get_token_refresh_endpoint() + request_headers = self.get_headers() + request_body = self.build_refresh_request_body() + + logger.info(f"Sending request to URL: {request_url}") + + response = requests.request(method="POST", url=request_url, data=request_body, headers=request_headers) + self._log_response(response) response.raise_for_status() + + response_json = response.json() + + self.access_token = response_json.get("access_token") + return response.json() + except requests.exceptions.RequestException as e: - if e.response.status_code == 429 or e.response.status_code >= 500: + if e.response and (e.response.status_code == 429 or e.response.status_code >= 500): raise DefaultBackoffException(request=e.response.request, response=e.response) raise except Exception as e: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml index 58f6d026171a..ddc8283179c2 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml @@ -1,11 +1,6 @@ version: 0.50.2 type: DeclarativeSource -check: - type: CheckStream - stream_names: - - balances - definitions: selector: type: RecordSelector @@ -16,7 +11,7 @@ definitions: requester: type: HttpRequester - url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' + url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/' path: "{{ parameters.path }}" http_method: GET request_headers: @@ -38,9 +33,15 @@ definitions: - type: DefaultErrorHandler backoff_strategies: - type: ConstantBackoffStrategy - backoff_time_in_seconds: 300 + backoff_time_in_seconds: 100 request_body_json: {} + #NOTE: The streams Payments, Orders and Subscriptions require a webhook so you can register + #the Ids of each event as these endpoints do not have a GET method to list the Ids and use it + #in other streams + + #Stream Transactions + #Paypal API only has V1 for this stream transactions_stream: type: DeclarativeStream primary_key: transaction_id @@ -67,6 +68,15 @@ definitions: $ref: "#/definitions/requester" request_parameters: fields: all + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + description: "Handle HTTP 400 with error message: Data for the given start date is not available. " + response_filters: + - http_codes: [400] + action: IGNORE + predicate: "{{ 'Data for the given start date is not available' in response['message']}}" transformations: - type: AddFields fields: @@ -89,26 +99,29 @@ definitions: start_datetime: type: MinMaxDatetime datetime: >- - {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} + {{ max( format_datetime(config.get('start_date'), '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: start_date inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: >- + {{ format_datetime(config.get('end_date') if config.get('end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }} end_time_option: type: RequestOption field_name: end_date inject_into: request_parameter - end_datetime: - type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" step: "P{{ config.get('time_window', 7) }}D" cursor_granularity: PT1S $parameters: - path: "transactions" + path: "v1/reporting/transactions" field_path: transaction_details + #Stream balances + #Paypal API only has V1 for this stream balances_stream: type: DeclarativeStream primary_key: as_of_time @@ -143,70 +156,244 @@ definitions: start_datetime: type: MinMaxDatetime datetime: >- - {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} + {{ max( format_datetime(config.get('start_date'), '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: as_of_time inject_into: request_parameter $parameters: - path: "balances" + path: "v1/reporting/balances" + + #New Stream - List Product + #Paypal API only has V1 for this stream + list_products_stream: + type: DeclarativeStream + primary_key: id + name: "list_products" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 20 + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: page_size + type: RequestOption + requester: + $ref: "#/definitions/requester" + http_method: GET + request_headers: + Content-Type: application/json + $parameters: + path: "v1/catalogs/products" + field_path: products + + # New Stream - Show Product Details + #Paypal API only has V1 for this stream + #This can't be incremental as there is no time filtering. If you need to have the updates, you need to Append in the full_sync + # This stream works, however has some challenges with performance. Whith a big catalog it can take up to 3 hrs. + show_product_details_stream: + type: DeclarativeStream + primary_key: id + name: "show_product_details" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: "/v1/catalogs/products/{{ stream_slice.id }}" + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: "id" + partition_field: "id" + stream: + $ref: "#/definitions/list_products_stream" + + #Stream List Disputes + #Paypal API only has V1 for this stream + list_disputes_stream: + type: DeclarativeStream + primary_key: dispute_id + name: "list_disputes" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: next_page_token + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: page_size + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 50 + requester: + $ref: "#/definitions/requester" + http_method: GET + transformations: + - type: AddFields + fields: + - path: + - updated_time_cut + value: >- + {{ record['update_time'] }} + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_time_cut + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: "{{ format_datetime(config.get('dispute_start_date') if config.get('dispute_start_date') else (now_utc() - duration('P179D')), '%Y-%m-%dT%H:%M:%S.%fZ')[:23] + 'Z' }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + end_datetime: + type: MinMaxDatetime + #Adding a time delta as the API has a problem with the slice being too close to the now_utc. Set to 30M + datetime: >- + {{ format_datetime(config.get('dispute_end_date') if config.get('dispute_end_date') else (now_utc() - duration('PT30M')), '%Y-%m-%dT%H:%M:%S.%fZ')[:23] + 'Z'}} + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_time_option: + type: RequestOption + field_name: update_time_after + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: update_time_before + inject_into: request_parameter + step: "P{{ config.get('time_window', 7) }}D" + cursor_granularity: PT1S + $parameters: + path: "v1/customer/disputes" + field_path: items + + #Stream Search Invoices + # Currently it does not support incremental sync as metadata does not contain last_update_date + search_invoices_stream: + type: DeclarativeStream + primary_key: id + name: "search_invoices" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: page_size + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + requester: + $ref: "#/definitions/requester" + http_method: POST + request_headers: + Content-Type: application/json + request_body_json: + creation_date_range: + start: "{{ config.get('start_date') }}" + end: >- + {{ format_datetime(config.get('end_date') if config.get('end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }} + $parameters: + field_path: items + path: "v2/invoicing/search-invoices" + + #Stream List Payments + #Currently uses V1 which is about to be derecated + #But there is no endpoint in v2 for listing payments + list_payments_stream: + type: DeclarativeStream + primary_key: id + name: "list_payments" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.next_id}}" + stop_condition: "{{ response.next_id == ''}}" + page_size: 20 + page_token_option: + type: RequestOption + field_name: start_id + inject_into: request_parameter + page_size_option: + type: RequestOption + field_name: count + inject_into: request_parameter + requester: + $ref: "#/definitions/requester" + request_parameters: + start_time: "{{ stream_interval.start_time.strftime('%Y-%m-%dT%H:%M:%SZ') }}" + end_time: "{{ stream_interval.end_time.strftime('%Y-%m-%dT%H:%M:%SZ') }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: update_time + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + #type: MinMaxDatetime + datetime: "{{ config.get('start_date') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: >- + {{ format_datetime(config.get('end_date') if config.get('end_date') else now_utc(), '%Y-%m-%dT%H:%M:%SZ') }} + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: start_time + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: end_time + inject_into: request_parameter + step: "P{{ config.get('time_window', 7) }}D" + cursor_granularity: PT1S + $parameters: + path: "v1/payments/payment" + field_path: payments streams: - "#/definitions/transactions_stream" - "#/definitions/balances_stream" + - "#/definitions/list_products_stream" + - "#/definitions/show_product_details_stream" + - "#/definitions/list_disputes_stream" + - "#/definitions/search_invoices_stream" + - "#/definitions/list_payments_stream" -spec: - type: Spec - documentation_url: https://docs.airbyte.com/integrations/sources/paypal-transactions - connection_specification: - $schema: http://json-schema.org/draft-07/schema# - type: object - additionalProperties: true - required: - - client_id - - client_secret - - start_date - - is_sandbox - properties: - client_id: - type: string - title: Client ID - description: "The Client ID of your Paypal developer application." - airbyte_secret: true - order: 0 - client_secret: - type: string - title: Client secret - description: "The Client Secret of your Paypal developer application." - airbyte_secret: true - order: 1 - start_date: - title: Start Date - description: >- - Start Date for data extraction in ISO - format. Date must be in range from 3 years till 12 hrs before - present time. - type: string - examples: ["2021-06-11T23:59:59", "2021-06-11T23:59:59+00:00"] - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(|Z|[+-][0-9]{2}:[0-9]{2})$ - format: "date-time" - order: 2 - is_sandbox: - title: "Sandbox" - description: "Determines whether to use the sandbox or production environment." - type: "boolean" - default: false - refresh_token: - type: "string" - title: "Refresh token" - description: "The key to refresh the expired access token." - airbyte_secret: true - time_window: - type: "integer" - title: "Number of days per request" - description: "The number of days per request. Must be a number between 1 and 31." - default: 7 - minimum: 1 - maximum: 31 +check: + stream_names: + - "balances" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py new file mode 100644 index 000000000000..1a6d4cc56c0e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paypal_transaction import SourcePaypalTransaction + + +def run(): + source = SourcePaypalTransaction() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_disputes.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_disputes.json new file mode 100644 index 000000000000..8d4c9ff57a97 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_disputes.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "dispute_id": { "type": ["null", "string"] }, + "create_time": { "type": "string", "format": "date-time" }, + "update_time": { "type": "string", "format": "date-time" }, + "updated_time_cut": { "type": "string", "format": "date-time" }, + "status": { "type": ["null", "string"] }, + "reason": { "type": ["null", "string"] }, + "dispute_state": { "type": ["null", "string"] }, + "dispute_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + }, + "links": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "href": { "type": ["null", "string"] }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_payments.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_payments.json new file mode 100644 index 000000000000..6ce37d9d6d3e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_payments.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "intent": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "cart": { "type": ["null", "string"] }, + "payer": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "payment_method": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "payer_info": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, + "payer_id": { "type": ["null", "string"] }, + "shipping_address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "recipient_name": { "type": ["null", "string"] }, + "line1": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] } + } + }, + "phone": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] } + } + } + } + }, + "transactions": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "reference_id": { "type": ["null", "string"] }, + "amount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "total": { "type": ["null", "string"] }, + "currency": { "type": ["null", "string"] }, + "details": { + "type": ["null", "object"], + "properties": { + "subtotal": { "type": ["null", "string"] }, + "shipping": { "type": ["null", "string"] }, + "insurance": { "type": ["null", "string"] }, + "handling_fee": { "type": ["null", "string"] }, + "shipping_discount": { "type": ["null", "string"] }, + "discount": { "type": ["null", "string"] } + } + } + } + }, + "payee": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "merchant_id": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] } + } + }, + "description": { "type": ["null", "string"] }, + "item_list": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "items": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "price": { "type": ["null", "string"] }, + "currency": { "type": ["null", "string"] }, + "tax": { "type": ["null", "string"] }, + "quantity": { "type": ["null", "integer"] }, + "image_url": { "type": ["null", "string"] } + } + } + }, + "shipping_address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "recipient_name": { "type": ["null", "string"] }, + "line1": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] } + } + } + } + }, + "related_resources": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "sale": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "amount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "total": { "type": ["null", "string"] }, + "currency": { "type": ["null", "string"] }, + "details": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "subtotal": { "type": ["null", "string"] }, + "shipping": { "type": ["null", "string"] }, + "insurance": { "type": ["null", "string"] }, + "handling_fee": { "type": ["null", "string"] }, + "shipping_discount": { "type": ["null", "string"] }, + "discount": { "type": ["null", "string"] } + } + } + } + }, + "payment_mode": { "type": ["null", "string"] }, + "protection_eligibility": { "type": ["null", "string"] }, + "protection_eligibility_type": { + "type": ["null", "string"] + }, + "transaction_fee": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "value": { "type": ["null", "string"] }, + "currency": { "type": ["null", "string"] } + } + }, + "purchase_unit_reference_id": { + "type": ["null", "string"] + }, + "parent_payment": { "type": ["null", "string"] }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "href": { "type": ["null", "string"] }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } + } + } + } + } + } + } + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "href": { "type": ["null", "string"] }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_products.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_products.json new file mode 100644 index 000000000000..b700519c4c72 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/list_products.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "links": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "href": { "type": ["null", "string"] }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/search_invoices.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/search_invoices.json new file mode 100644 index 000000000000..e0887a02b9e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/search_invoices.json @@ -0,0 +1,357 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "primary_recipients": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "billing_info": { + "type": ["null", "object"], + "properties": { + "business_name": { "type": ["null", "string"] }, + "name": { + "type": ["null", "object"], + "properties": { + "prefix": { "type": ["null", "string"] }, + "given_name": { "type": ["null", "string"] }, + "surname": { "type": ["null", "string"] }, + "middle_name": { "type": ["null", "string"] }, + "suffix": { "type": ["null", "string"] }, + "alternate_full_name": { "type": ["null", "string"] }, + "full_name": { "type": ["null", "string"] } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "address_line_1": { "type": ["null", "string"] }, + "address_line_2": { "type": ["null", "string"] }, + "address_line_3": { "type": ["null", "string"] }, + "address_line_4": { "type": ["null", "string"] }, + "admin_area_1": { "type": ["null", "string"] }, + "admin_area_2": { "type": ["null", "string"] }, + "admin_area_3": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] }, + "address_details": { "type": ["null", "object"] }, + "phones": { "type": ["null", "array"] }, + "additiona_info": { "type": ["null", "string"] }, + "email_address": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + } + } + }, + "shipping_info": { + "type": ["null", "object"], + "properties": { + "business_name": { "type": ["null", "string"] }, + "name": { + "type": ["null", "object"], + "properties": { + "prefix": { "type": ["null", "string"] }, + "given_name": { "type": ["null", "string"] }, + "surname": { "type": ["null", "string"] }, + "middle_name": { "type": ["null", "string"] }, + "suffix": { "type": ["null", "string"] }, + "alternate_full_name": { "type": ["null", "string"] }, + "full_name": { "type": ["null", "string"] } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "address_line_1": { "type": ["null", "string"] }, + "address_line_2": { "type": ["null", "string"] }, + "address_line_3": { "type": ["null", "string"] }, + "address_line_4": { "type": ["null", "string"] }, + "admin_area_1": { "type": ["null", "string"] }, + "admin_area_2": { "type": ["null", "string"] }, + "admin_area_3": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] }, + "address_details": { "type": ["null", "object"] } + } + } + } + } + } + } + }, + "additional_recipients": { "type": ["null", "array"] }, + "detail": { + "type": ["null", "object"], + "properties": { + "reference": { "type": ["null", "string"] }, + "note": { "type": ["null", "string"] }, + "terms_and_conditions": { "type": ["null", "string"] }, + "memo": { "type": ["null", "string"] }, + "attachments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "reference_url": { "type": ["null", "string"] }, + "content_type": { "type": ["null", "string"] }, + "size": { "type": ["null", "string"] }, + "create_time": { "type": ["null", "string"] } + } + } + }, + "currency_code": { "type": ["null", "string"] }, + "invoice_number": { "type": ["null", "string"] }, + "invoice_date": { "type": ["null", "string"], "format": "date" }, + "payment_term": { + "type": ["null", "object"], + "properties": { + "term_type": { "type": ["null", "string"] }, + "due_date": { "type": ["null", "string"], "format": "date" } + } + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "created_by": { "type": ["null", "string"] }, + "last_updated_by": { "type": ["null", "string"] }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "cancelled_by": { "type": ["null", "string"] }, + "last_seen_by": { "type": ["null", "string"] }, + "recipient_view_url": { "type": ["null", "string"] }, + "invoicer_view_url": { "type": ["null", "string"] }, + "cancel_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "first_sent_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_sent_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_by_flow": { "type": ["null", "string"] } + } + } + } + }, + "last_update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "invoicer": { + "type": ["null", "object"], + "properties": { + "business_name": { "type": ["null", "string"] }, + "name": { + "type": ["null", "object"], + "properties": { + "prefix": { "type": ["null", "string"] }, + "given_name": { "type": ["null", "string"] }, + "surname": { "type": ["null", "string"] }, + "middle_name": { "type": ["null", "string"] }, + "suffix": { "type": ["null", "string"] }, + "alternate_full_name": { "type": ["null", "string"] }, + "full_name": { "type": ["null", "string"] } + }, + "address": { + "type": ["null", "object"], + "properties": { + "address_line_1": { "type": ["null", "string"] }, + "address_line_2": { "type": ["null", "string"] }, + "address_line_3": { "type": ["null", "string"] }, + "admin_area_1": { "type": ["null", "string"] }, + "admin_area_2": { "type": ["null", "string"] }, + "admin_area_3": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] }, + "address_details": { "type": ["null", "object"] } + }, + "phones": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "country_code": { "type": ["null", "string"] }, + "national_number": { "type": ["null", "string"] }, + "extension_number": { "type": ["null", "string"] }, + "phone_type": { "type": ["null", "string"] } + } + } + }, + "website": { "type": ["null", "string"] }, + "tax_id": { "type": ["null", "string"] }, + "additional_notes": { "type": ["null", "string"] }, + "email_address": { "type": ["null", "string"] } + } + } + } + }, + "configuration": { + "type": ["null", "object"], + "properties": { + "tax_calculated_after_discount": { "type": ["null", "string"] }, + "tax_inclusive": { "type": ["null", "string"] }, + "allow_tip": { "type": ["null", "string"] }, + "partial_payment": { + "type": ["null", "object"], + "properties": { + "allow_partial_payment": { "type": ["null", "string"] }, + "minimum_amount_due": { "type": ["null", "object"] } + } + }, + "template_id": { "type": ["null", "string"] } + } + }, + "amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] }, + "breakdown": { + "type": ["null", "object"], + "properties": { + "item_total": { "type": ["null", "object"] }, + "discount": { "type": ["null", "object"] }, + "tax_total": { "type": ["null", "object"] }, + "shipping": { "type": ["null", "object"] }, + "custom": { "type": ["null", "object"] } + } + } + } + }, + "due_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + }, + "gratuity": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + }, + "payments": { + "transactions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "payment_id": { "type": ["null", "string"] }, + "note": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "payment_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "method": { "type": ["null", "string"] }, + "amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + }, + "shipping_info": { + "type": ["null", "object"], + "properties": { + "business_name": { "type": ["null", "string"] }, + "name": { + "type": ["null", "object"], + "properties": { + "prefix": { "type": ["null", "string"] }, + "given_name": { "type": ["null", "string"] }, + "surname": { "type": ["null", "string"] }, + "middle_name": { "type": ["null", "string"] }, + "suffix": { "type": ["null", "string"] }, + "alternate_full_name": { "type": ["null", "string"] }, + "full_name": { "type": ["null", "string"] } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "address_line_1": { "type": ["null", "string"] }, + "address_line_2": { "type": ["null", "string"] }, + "address_line_3": { "type": ["null", "string"] }, + "admin_area_1": { "type": ["null", "string"] }, + "admin_area_2": { "type": ["null", "string"] }, + "admin_area_3": { "type": ["null", "string"] }, + "postal_code": { "type": ["null", "string"] }, + "country_code": { "type": ["null", "string"] }, + "address_details": { "type": ["null", "object"] } + } + } + } + } + } + } + }, + "paid_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + }, + "refunds": { + "transactions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "refund_id": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "refund_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "method": { "type": ["null", "string"] }, + "amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + } + } + }, + "refund_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + }, + "links": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "href": { "type": ["null", "string"], "format": "uri" }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/show_product_details.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/show_product_details.json new file mode 100644 index 000000000000..822b85737f60 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/show_product_details.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "category": { "type": ["null", "string"] }, + "image_url": { "type": ["null", "string"] }, + "home_url": { "type": ["null", "string"] }, + "create_time": { "type": ["null", "string"], "format": "date-time" }, + "update_time": { "type": ["null", "string"], "format": "date-time" }, + "links": { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "href": { "type": ["null", "string"] }, + "rel": { "type": ["null", "string"] }, + "method": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index afd56a9278a1..4260da59befa 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource """ @@ -11,7 +12,6 @@ WARNING: Do not modify this file. """ - # Declarative Source class SourcePaypalTransaction(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.yaml b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.yaml new file mode 100644 index 000000000000..8f3379e51cf6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.yaml @@ -0,0 +1,78 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/paypal-transactions +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - client_id + - client_secret + - start_date + - is_sandbox + properties: + client_id: + type: string + title: Client ID + description: "The Client ID of your Paypal developer application." + airbyte_secret: true + order: 0 + client_secret: + type: string + title: Client secret + description: "The Client Secret of your Paypal developer application." + airbyte_secret: true + order: 1 + start_date: + title: Start Date + description: >- + Start Date for data extraction in ISO + format. Date must be in range from 3 years till 12 hrs before + present time. + type: string + examples: ["2021-06-11T23:59:59Z", "2021-06-11T23:59:59+00:00"] + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(|Z|[+-][0-9]{2}:[0-9]{2})$ + format: "date-time" + order: 2 + is_sandbox: + title: "Sandbox" + description: "Determines whether to use the sandbox or production environment." + type: "boolean" + default: false + dispute_start_date: + title: Dispute Start Date Range + description: >- + Start Date parameter for the list dispute endpoint in ISO + format. This Start Date must be in range within 180 days before + present time, and requires ONLY 3 miliseconds(mandatory). + If you don't use this option, it defaults to a start date set 180 days in the past. + type: string + examples: ["2021-06-11T23:59:59.000Z"] + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$ + format: "date-time" + order: 3 + end_date: + title: End Date + description: >- + End Date for data extraction in ISO + format. This can be help you select specific range of time, mainly for test purposes + or data integrity tests. When this is not used, now_utc() is used by the streams. + This does not apply to Disputes and Product streams. + type: string + examples: ["2021-06-11T23:59:59Z", "2021-06-11T23:59:59+00:00"] + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(|Z|[+-][0-9]{2}:[0-9]{2})$ + format: "date-time" + order: 4 + refresh_token: + type: "string" + title: "Refresh token" + description: "The key to refresh the expired access token." + airbyte_secret: true + time_window: + type: "integer" + title: "Number of days per request" + description: "The number of days per request. Must be a number between 1 and 31." + default: 7 + minimum: 1 + maximum: 31 diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py new file mode 100644 index 000000000000..dd19b6306e77 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py @@ -0,0 +1,88 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import time +from unittest.mock import patch + +import pytest +import requests +import requests_mock +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from source_paypal_transaction.components import PayPalOauth2Authenticator + + +@pytest.fixture +def mock_authenticator(): + return PayPalOauth2Authenticator( + config={}, + parameters={}, + client_id='test_client_id', + client_secret='test_client_secret', + token_refresh_endpoint='https://test.token.endpoint', + grant_type='test_grant_type' + ) + +def test_get_refresh_access_token_response(mock_authenticator): + expected_response_json = {'access_token': 'test_access_token', 'expires_in': 3600} + with requests_mock.Mocker() as mock_request: + mock_request.post('https://test.token.endpoint', json=expected_response_json, status_code=200) + # Call _get_refresh method + mock_authenticator._get_refresh_access_token_response() + + assert mock_authenticator.access_token == expected_response_json['access_token'] + +def test_token_expiration(mock_authenticator): + # Mock response for initial token request + initial_response_json = {'access_token': 'initial_access_token', 'expires_in': 1} + # Mock response for token refresh request + refresh_response_json = {'access_token': 'refreshed_access_token', 'expires_in': 3600} + with requests_mock.Mocker() as mock_request: + + mock_request.post('https://test.token.endpoint', json=initial_response_json, status_code=200) + mock_authenticator._get_refresh_access_token_response() + + # Assert that the initial access token is set correctly + assert mock_authenticator.access_token == initial_response_json['access_token'] + time.sleep(2) + + mock_request.post('https://test.token.endpoint', json=refresh_response_json, status_code=200) + mock_authenticator._get_refresh_access_token_response() + + # Assert that the access token is refreshed + assert mock_authenticator.access_token == refresh_response_json['access_token'] + + +def test_backoff_retry(mock_authenticator, caplog): + + mock_response = {'access_token': 'test_access_token', 'expires_in': 3600} + mock_reason = "Too Many Requests" + + with requests_mock.Mocker() as mock_request: + mock_request.post('https://test.token.endpoint', json=mock_response, status_code=429, reason=mock_reason) + with caplog.at_level(logging.INFO): + try: + mock_authenticator._get_refresh_access_token_response() + except requests.exceptions.HTTPError: + pass # Ignore the HTTPError + else: + pytest.fail("Expected DefaultBackoffException to be raised") + +@pytest.fixture +def authenticator_parameters(): + return { + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "config": {}, + "parameters": {}, + "token_refresh_endpoint": "https://test.token.endpoint", + "grant_type": "test_grant_type" + } + +def test_get_headers(authenticator_parameters): + expected_basic_auth = "Basic dGVzdF9jbGllbnRfaWQ6dGVzdF9jbGllbnRfc2VjcmV0" + authenticator = PayPalOauth2Authenticator(**authenticator_parameters) + headers = authenticator.get_headers() + assert headers == {"Authorization": expected_basic_auth} + + + diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py new file mode 100644 index 000000000000..06dd08dc74a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# conftest.py +import json +from datetime import datetime +from unittest.mock import patch + +import pytest +from source_paypal_transaction import SourcePaypalTransaction + + +@pytest.fixture(name="config") +def config_fixture(): + #From File test + # with open('../secrets/config.json') as f: + # return json.load(f) + #Mock test + return { + "client_id": "your_client_id", + "client_secret": "your_client_secret", + "start_date": "2024-01-30T00:00:00Z", + "end_date": "2024-02-01T00:00:00Z", + "dispute_start_date": "2024-02-01T00:00:00.000Z", + "dispute_end_date": "2024-02-05T23:59:00.000Z", + "buyer_username": "Your Buyer email", + "buyer_password": "Your Buyer Password", + "payer_id": "ypur ACCOUNT ID", + "is_sandbox": True + } + + +@pytest.fixture(name="source") +def source_fixture(): + return SourcePaypalTransaction() + +def validate_date_format(date_str, format): + try: + datetime.strptime(date_str, format) + return True + except ValueError: + return False + +def test_date_formats_in_config(config): + start_date_format = "%Y-%m-%dT%H:%M:%SZ" + dispute_date_format = "%Y-%m-%dT%H:%M:%S.%fZ" + assert validate_date_format(config['start_date'], start_date_format), "Start date format is incorrect" + assert validate_date_format(config['end_date'], start_date_format), "End date format is incorrect" + assert validate_date_format(config['dispute_start_date'], dispute_date_format), "Dispute start date format is incorrect" + assert validate_date_format(config['dispute_end_date'], dispute_date_format), "Dispute end date format is incorrect" + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture(): + return patch("source_paypal_transactions.source.AirbyteLogger") \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py new file mode 100644 index 000000000000..958db41262da --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py @@ -0,0 +1,146 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import os +from dataclasses import dataclass, field +from typing import Any, List, Mapping, Optional, Union + +import pytest +import requests +import requests_mock +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.types import Config + + +@dataclass +class CursorPaginationStrategy(PaginationStrategy): + """ + Pagination strategy that evaluates an interpolated string to define the next page token + Attributes: + page_size (Optional[int]): the number of records to request + cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value + config (Config): connection config + stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating + decoder (Decoder): decoder to decode the response + """ + cursor_value: Union[InterpolatedString, str] + config: Config + parameters: Mapping[str, Any] + page_size: Optional[int] = None + stop_condition: Optional[Union[InterpolatedBoolean, str]] = None + decoder: Decoder = field(default_factory=JsonDecoder) + + def __post_init__(self): + if isinstance(self.cursor_value, str): + self.cursor_value = InterpolatedString.create(self.cursor_value, parameters=self.parameters) + if isinstance(self.stop_condition, str): + self.stop_condition = InterpolatedBoolean(condition=self.stop_condition, parameters=self.parameters) + + @property + def initial_token(self) -> Optional[Any]: + return None + + + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + decoded_response = self.decoder.decode(response) + headers = response.headers + headers["link"] = response.links + + print("STOP CONDITION", self.stop_condition) + + if self.stop_condition: + should_stop = self.stop_condition.eval(self.config, response=decoded_response, headers=headers, last_records=last_records) + if should_stop: + print("Stopping...") + return None + + # Update cursor_value with the next_id from the response + self.cursor_value = InterpolatedString.create(decoded_response.get("next_id"), parameters=self.parameters) + token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response, headers=headers) + print("TOKEN", token) + return token if token else None + + def reset(self): + pass + + def get_page_size(self) -> Optional[int]: + return self.page_size + + +@pytest.fixture +def mock_responses(): + return [ + "token_page_init.json", + "token_PAY-0L38757939422510JMW5ZJVA.json", + "token_PAYID-MW5XXZY5YL87592N34454913.json" + ] + +@pytest.fixture +def cursor_pagination_strategy(mock_responses, stop_condition = None): + parameters = {} + decoder = JsonDecoder(parameters=parameters) + cursor_value = "start_id" # Initialize with a default value + + for response_file in mock_responses: + if cursor_value == "start_id": + cursor_value = load_mock_data(response_file).get("next_id") + else: + break # Stop after getting the next_id from the first response + + return CursorPaginationStrategy( + cursor_value=cursor_value, + config={}, + parameters=parameters, + page_size=3, + stop_condition=stop_condition, + decoder=decoder + ) + + +def load_mock_data(filename): + with open(os.path.join("./unit_tests/test_files", filename), "r") as file: + return json.load(file) + +def test_cursor_pagination(cursor_pagination_strategy, mock_responses): + with requests_mock.Mocker() as m: + base_url = "http://example.com/api/resource" + + # Mock responses + for i, response_file in enumerate(mock_responses): + print("") + print("####################################") + if i == 0: + url = f"{base_url}?count=3" + print("FIRST ITERATION:", response_file, i, url) + + if i > 0: + url += f"&start_id={next_id}" + print("NEXT ITERATIONS:", response_file, i, url) + m.get(url, json=load_mock_data(response_file), status_code=200) + # Get next_id from the response if it's not the last response + + if i < len(mock_responses) - 1: + next_id = load_mock_data(response_file)["next_id"] + print("FOUND NEXT ID:", next_id) + + else: + next_id = None + cursor_pagination_strategy(mock_responses, stop_condition = True) + + # Make API call and process response + response = requests.get(url) + print("GET RESPONSE:", response) + assert response.status_code == 200 + + decoded_response = response.json() + last_records = decoded_response["payments"] + next_id = cursor_pagination_strategy.next_page_token(response, last_records) + print("NEXT ID:", next_id) + + # Verify the pagination stopped + assert next_id is None + print("No more pages") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py new file mode 100644 index 000000000000..05b98d04f90a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +import re + +import pytest +import requests +import requests_mock +from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy + + +class MockPaginationStrategy(PaginationStrategy): + def __init__(self, page_size): + self.page_size = page_size + self.current_page = 1 + + @property + def initial_token(self): + return self.current_page + + def next_page_token(self, response, last_records): + self.current_page += 1 + return self.current_page if self.current_page <= 5 else None + + def reset(self): + self.current_page = 1 + + @property + def get_page_size(self): + return self.page_size + +@pytest.fixture +def mock_pagination_strategy(): + return MockPaginationStrategy(page_size=500) + +@pytest.fixture +def paginator(): + pagination_strategy = MockPaginationStrategy(page_size=3) + return DefaultPaginator( + pagination_strategy=pagination_strategy, + config={}, + url_base="http://example.com/v1/reporting/transactions", + parameters={} + ) + +def load_mock_data(page): + with open(f"./unit_tests/test_files/page_{page}.json", "r") as file: + return file.read() + +# Test to verify pagination logic transitions from page 1 to page 2 +def test_pagination_logic(paginator): + page_1_data = load_mock_data(1) + page_2_data = load_mock_data(2) + + paginator_url_1 = f"{paginator.url_base.string}?page=1&page_size={paginator.pagination_strategy.get_page_size}" + paginator_url_2 = f"{paginator.url_base.string}?page=2&page_size={paginator.pagination_strategy.get_page_size}" + + with requests_mock.Mocker() as m: + m.get(paginator_url_1, text=page_1_data, status_code=200) + m.get(paginator_url_2, text=page_2_data, status_code=200) + + response_page_1 = requests.get(paginator_url_1) + response_page_1._content = str.encode(page_1_data) + response_page_2 = requests.get(paginator_url_2) + response_page_2._content = str.encode(page_2_data) + + + # Simulate getting the next page token from page 1's response + next_page_token_page_1 = paginator.next_page_token(response_page_1, []) + print("NEXT PAGE TOKEN", next_page_token_page_1) + + # Assert that the next page token indicates moving to page 2 + assert next_page_token_page_1['next_page_token'] == 2, "Failed to transition from page 1 to page 2" + + + # Check that the correct page size is used in requests and that we have the right number of pages + assert len(m.request_history) == 2 + assert "page_size=3" in m.request_history[0].url + assert "page_size=3" in m.request_history[1].url \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_1.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_1.json new file mode 100644 index 000000000000..8e567afe7c7d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_1.json @@ -0,0 +1,135 @@ +{ + "transaction_details": [ + { + "transaction_info": { + "transaction_id": "2N603077GD600560F", + "transaction_event_code": "T1503", + "transaction_initiation_date": "2024-02-10T00:01:56+0000", + "transaction_updated_date": "2024-02-10T00:01:56+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "-60.75" + }, + "transaction_status": "S", + "ending_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "available_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "protection_eligibility": "02" + }, + "payer_info": { + "address_status": "N", + "payer_name": {} + }, + "shipping_info": {}, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + }, + { + "transaction_info": { + "transaction_id": "0JN65120FU073310V", + "transaction_event_code": "T0001", + "transaction_initiation_date": "2024-02-10T00:01:57+0000", + "transaction_updated_date": "2024-02-10T00:01:57+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "-10.00" + }, + "fee_amount": { + "currency_code": "USD", + "value": "-0.25" + }, + "transaction_status": "P", + "transaction_subject": "You have a payout!", + "transaction_note": "Thanks for your patronage!", + "ending_balance": { + "currency_code": "USD", + "value": "2048008.05" + }, + "available_balance": { + "currency_code": "USD", + "value": "2048008.05" + }, + "custom_field": "201403140001", + "protection_eligibility": "02" + }, + "payer_info": { + "email_address": "Ernesto.Witting@yahoo.com", + "address_status": "N", + "payer_name": {} + }, + "shipping_info": { + "name": "John, Merchant" + }, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + }, + { + "transaction_info": { + "transaction_id": "4XL46752RR472362C", + "paypal_reference_id": "0JN65120FU073310V", + "paypal_reference_id_type": "TXN", + "transaction_event_code": "T1105", + "transaction_initiation_date": "2024-02-10T00:01:57+0000", + "transaction_updated_date": "2024-02-10T00:01:57+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "10.25" + }, + "transaction_status": "S", + "transaction_subject": "You have a payout!", + "transaction_note": "Thanks for your patronage!", + "ending_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "available_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "protection_eligibility": "02" + }, + "payer_info": { + "address_status": "N", + "payer_name": {} + }, + "shipping_info": {}, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + } + ], + "account_number": "C7CYMKZDG8D6E", + "start_date": "2024-02-10T00:00:00+0000", + "end_date": "2024-02-10T05:20:00+0000", + "last_refreshed_datetime": "2024-02-14T01:59:59+0000", + "page": 1, + "total_items": 352, + "total_pages": 118, + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&end_date=2024-02-10T05%3A20%3A00Z&start_date=2024-02-10T00%3A00%3A00Z&page_size=3&page=118", + "rel": "last", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&end_date=2024-02-10T05%3A20%3A00Z&start_date=2024-02-10T00%3A00%3A00Z&page_size=3&page=2", + "rel": "next", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&end_date=2024-02-10T05%3A20%3A00Z&start_date=2024-02-10T00%3A00%3A00Z&page_size=3&page=1", + "rel": "self", + "method": "GET" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_2.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_2.json new file mode 100644 index 000000000000..e27274a5d72f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/page_2.json @@ -0,0 +1,160 @@ +{ + "transaction_details": [ + { + "transaction_info": { + "transaction_id": "794702034R094742N", + "transaction_event_code": "T0001", + "transaction_initiation_date": "2024-02-10T00:01:57+0000", + "transaction_updated_date": "2024-02-10T00:01:57+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "-20.00" + }, + "fee_amount": { + "currency_code": "USD", + "value": "-0.25" + }, + "transaction_status": "P", + "transaction_subject": "You have a payout!", + "transaction_note": "Thanks for your support!", + "ending_balance": { + "currency_code": "USD", + "value": "2047998.05" + }, + "available_balance": { + "currency_code": "USD", + "value": "2047998.05" + }, + "custom_field": "201403140002", + "protection_eligibility": "02" + }, + "payer_info": { + "address_status": "N", + "payer_name": {} + }, + "shipping_info": { + "name": "John, Merchant" + }, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + }, + { + "transaction_info": { + "transaction_id": "74734003LC191551G", + "paypal_reference_id": "794702034R094742N", + "paypal_reference_id_type": "TXN", + "transaction_event_code": "T1105", + "transaction_initiation_date": "2024-02-10T00:01:57+0000", + "transaction_updated_date": "2024-02-10T00:01:57+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "20.25" + }, + "transaction_status": "S", + "transaction_subject": "You have a payout!", + "transaction_note": "Thanks for your support!", + "ending_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "available_balance": { + "currency_code": "USD", + "value": "2048018.30" + }, + "protection_eligibility": "02" + }, + "payer_info": { + "address_status": "N", + "payer_name": {} + }, + "shipping_info": {}, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + }, + { + "transaction_info": { + "paypal_account_id": "5DEJUG27PZB9J", + "transaction_id": "44R34480PG8833736", + "transaction_event_code": "T0001", + "transaction_initiation_date": "2024-02-10T00:01:57+0000", + "transaction_updated_date": "2024-02-10T00:01:57+0000", + "transaction_amount": { + "currency_code": "USD", + "value": "-30.00" + }, + "fee_amount": { + "currency_code": "USD", + "value": "-0.25" + }, + "transaction_status": "S", + "transaction_subject": "You have a payout!", + "transaction_note": "Thanks for your patronage!", + "ending_balance": { + "currency_code": "USD", + "value": "2047988.05" + }, + "available_balance": { + "currency_code": "USD", + "value": "2047988.05" + }, + "custom_field": "201403140003", + "protection_eligibility": "02" + }, + "payer_info": { + "account_id": "5DEJUG27PZB9J", + "email_address": "sb-59loi25655860@business.example.com", + "address_status": "N", + "payer_status": "N", + "payer_name": { + "alternate_full_name": "Test Store" + }, + "country_code": "US" + }, + "shipping_info": { + "name": "John, Merchant" + }, + "cart_info": {}, + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + } + ], + "account_number": "C7CYMKZDG8D6E", + "start_date": "2024-02-10T00:00:00+0000", + "end_date": "2024-02-10T05:20:00+0000", + "last_refreshed_datetime": "2024-02-14T01:59:59+0000", + "page": 2, + "total_items": 352, + "total_pages": 118, + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&start_date=2024-02-10T00%3A00%3A00Z&end_date=2024-02-10T05%3A20%3A00Z&page_size=3&page=1", + "rel": "first", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&start_date=2024-02-10T00%3A00%3A00Z&end_date=2024-02-10T05%3A20%3A00Z&page_size=3&page=118", + "rel": "last", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&start_date=2024-02-10T00%3A00%3A00Z&end_date=2024-02-10T05%3A20%3A00Z&page_size=3&page=3", + "rel": "next", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&start_date=2024-02-10T00%3A00%3A00Z&end_date=2024-02-10T05%3A20%3A00Z&page_size=3&page=1", + "rel": "prev", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/reporting/transactions?fields=transaction_info%2Cpayer_info%2Cshipping_info%2Cauction_info%2Ccart_info%2Cincentive_info%2Cstore_info&start_date=2024-02-10T00%3A00%3A00Z&end_date=2024-02-10T05%3A20%3A00Z&page_size=3&page=2", + "rel": "self", + "method": "GET" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAY-0L38757939422510JMW5ZJVA.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAY-0L38757939422510JMW5ZJVA.json new file mode 100644 index 000000000000..b0a6520defec --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAY-0L38757939422510JMW5ZJVA.json @@ -0,0 +1,451 @@ +{ + "payments": [ + { + "id": "PAY-0L38757939422510JMW5ZJVA", + "intent": "authorize", + "state": "approved", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "mihai.streza1@mi-pay.com", + "first_name": "Mihai", + "last_name": "Streza", + "payer_id": "QHD3E8SRDDSQL", + "shipping_address": { + "recipient_name": "Mihai Streza" + }, + "phone": "07534201211", + "country_code": "GB" + } + }, + "transactions": [ + { + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E" + }, + "description": "topup", + "invoice_number": "100000000188897", + "soft_descriptor": "PAYPAL *JOHNMERCHAN", + "item_list": { + "items": [ + { + "name": "topup", + "price": "20.00", + "currency": "EUR", + "tax": "0.00", + "quantity": 1 + } + ], + "shipping_address": { + "recipient_name": "Mihai Streza" + } + }, + "related_resources": [ + { + "authorization": { + "id": "3S025738SW168153S", + "state": "captured", + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "billing_agreement_id": "B-42217126VD515152H", + "parent_payment": "PAY-0L38757939422510JMW5ZJVA", + "valid_until": "2024-03-01T12:55:48Z", + "create_time": "2024-02-01T12:55:48Z", + "update_time": "2024-02-01T12:55:51Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/capture", + "rel": "capture", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/void", + "rel": "void", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S/reauthorize", + "rel": "reauthorize", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA", + "rel": "parent_payment", + "method": "GET" + } + ] + } + }, + { + "capture": { + "id": "26U95072LD470800B", + "amount": { + "total": "20.00", + "currency": "EUR" + }, + "state": "completed", + "custom": "", + "transaction_fee": { + "value": "1.39", + "currency": "EUR" + }, + "parent_payment": "PAY-0L38757939422510JMW5ZJVA", + "invoice_number": "100000000188897", + "create_time": "2024-02-01T12:55:51Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/26U95072LD470800B", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/26U95072LD470800B/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/3S025738SW168153S", + "rel": "authorization", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA", + "rel": "parent_payment", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T12:55:48Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-0L38757939422510JMW5ZJVA", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAY-5UU821714H9319614MW5ZGTQ", + "intent": "authorize", + "state": "approved", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "mihai.streza1@mi-pay.com", + "first_name": "Mihai", + "last_name": "Streza", + "payer_id": "QHD3E8SRDDSQL", + "shipping_address": { + "recipient_name": "Mihai Streza" + }, + "phone": "07534201211", + "country_code": "GB" + } + }, + "transactions": [ + { + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E" + }, + "description": "topup", + "invoice_number": "100000000188890", + "soft_descriptor": "PAYPAL *JOHNMERCHAN", + "item_list": { + "items": [ + { + "name": "topup", + "price": "20.00", + "currency": "EUR", + "tax": "0.00", + "quantity": 1 + } + ], + "shipping_address": { + "recipient_name": "Mihai Streza" + } + }, + "related_resources": [ + { + "authorization": { + "id": "4MN954876H428782W", + "state": "captured", + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "billing_agreement_id": "B-9YM05135W51321351", + "parent_payment": "PAY-5UU821714H9319614MW5ZGTQ", + "valid_until": "2024-03-01T12:49:18Z", + "create_time": "2024-02-01T12:49:18Z", + "update_time": "2024-02-01T12:49:21Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/4MN954876H428782W", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/4MN954876H428782W/capture", + "rel": "capture", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/4MN954876H428782W/void", + "rel": "void", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/4MN954876H428782W/reauthorize", + "rel": "reauthorize", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-5UU821714H9319614MW5ZGTQ", + "rel": "parent_payment", + "method": "GET" + } + ] + } + }, + { + "capture": { + "id": "3LS31047RT411632Y", + "amount": { + "total": "20.00", + "currency": "EUR" + }, + "state": "completed", + "custom": "", + "transaction_fee": { + "value": "1.39", + "currency": "EUR" + }, + "parent_payment": "PAY-5UU821714H9319614MW5ZGTQ", + "invoice_number": "100000000188890", + "create_time": "2024-02-01T12:49:21Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/3LS31047RT411632Y", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/3LS31047RT411632Y/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/4MN954876H428782W", + "rel": "authorization", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-5UU821714H9319614MW5ZGTQ", + "rel": "parent_payment", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T12:49:18Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-5UU821714H9319614MW5ZGTQ", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAYID-MW5Y7VQ3XR69795B78311812", + "intent": "sale", + "state": "approved", + "cart": "21D62257BL170881B", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "ABpaypal@gmail.com", + "first_name": "kiran", + "last_name": "ingale", + "payer_id": "MEZSMT5X3R5HW", + "shipping_address": { + "recipient_name": "kiran ingale", + "line1": "3210 D Street", + "city": "SANFORD", + "state": "CA", + "postal_code": "27331", + "country_code": "US" + }, + "phone": "3333333333", + "country_code": "US" + } + }, + "transactions": [ + { + "reference_id": "default", + "amount": { + "total": "1434.45", + "currency": "USD", + "details": { + "subtotal": "1434.45", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "soft_descriptor": "JOHNMERCHAN JOHNMERCHAN", + "item_list": { + "shipping_address": { + "recipient_name": "kiran ingale", + "line1": "3210 D Street", + "city": "SANFORD", + "state": "CA", + "postal_code": "27331", + "country_code": "US" + } + }, + "related_resources": [ + { + "sale": { + "id": "6UA65534T65937149", + "state": "refunded", + "amount": { + "total": "1434.45", + "currency": "USD", + "details": { + "subtotal": "1434.45", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "50.55", + "currency": "USD" + }, + "purchase_unit_reference_id": "default", + "parent_payment": "PAYID-MW5Y7VQ3XR69795B78311812", + "create_time": "2024-02-01T12:34:30Z", + "update_time": "2024-02-01T12:35:49Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/6UA65534T65937149", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/6UA65534T65937149/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5Y7VQ3XR69795B78311812", + "rel": "parent_payment", + "method": "GET" + } + ], + "soft_descriptor": "JOHNMERCHAN JOHNMERCHAN" + } + }, + { + "refund": { + "id": "9JS27953SE332473L", + "state": "completed", + "amount": { + "total": "-1434.45", + "currency": "USD" + }, + "parent_payment": "PAYID-MW5Y7VQ3XR69795B78311812", + "sale_id": "6UA65534T65937149", + "create_time": "2024-02-01T12:35:49Z", + "update_time": "2024-02-01T12:35:49Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/refund/9JS27953SE332473L", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5Y7VQ3XR69795B78311812", + "rel": "parent_payment", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/6UA65534T65937149", + "rel": "sale", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T12:34:30Z", + "update_time": "2024-02-01T12:35:49Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5Y7VQ3XR69795B78311812", + "rel": "self", + "method": "GET" + } + ] + } + ], + "count": 3, + "next_id": "PAYID-MW5XXZY5YL87592N34454913" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAYID-MW5XXZY5YL87592N34454913.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAYID-MW5XXZY5YL87592N34454913.json new file mode 100644 index 000000000000..dfee3371b093 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_PAYID-MW5XXZY5YL87592N34454913.json @@ -0,0 +1,391 @@ +{ + "payments": [ + { + "id": "PAYID-MW5XXZY5YL87592N34454913", + "intent": "sale", + "state": "approved", + "cart": "2P498024GJ403825S", + "payer": { + "payment_method": "paypal", + "status": "UNVERIFIED", + "payer_info": { + "email": "rahul21@yopmail.com", + "first_name": "d", + "last_name": "d", + "payer_id": "KZAHJGF7B2SBU", + "shipping_address": { + "recipient_name": "d d", + "line1": "gfd", + "line2": "gdgd", + "city": "dfgd", + "state": "CA", + "postal_code": "95388", + "country_code": "US" + }, + "phone": "8219746756", + "country_code": "US" + } + }, + "transactions": [ + { + "reference_id": "default", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "soft_descriptor": "PAYPAL *JOHNMERCHAN JOHNMERCHAN", + "item_list": { + "shipping_address": { + "recipient_name": "d d", + "line1": "gfd", + "line2": "gdgd", + "city": "dfgd", + "state": "CA", + "postal_code": "95388", + "country_code": "US" + } + }, + "related_resources": [ + { + "sale": { + "id": "6W206643Y5829092B", + "state": "completed", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "3.48", + "currency": "USD" + }, + "purchase_unit_reference_id": "default", + "receipt_id": "1732798430780793", + "parent_payment": "PAYID-MW5XXZY5YL87592N34454913", + "create_time": "2024-02-01T11:09:27Z", + "update_time": "2024-02-01T11:09:27Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/6W206643Y5829092B", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/6W206643Y5829092B/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5XXZY5YL87592N34454913", + "rel": "parent_payment", + "method": "GET" + } + ], + "soft_descriptor": "PAYPAL *JOHNMERCHAN JOHNMERCHAN" + } + } + ] + } + ], + "create_time": "2024-02-01T11:09:27Z", + "update_time": "2024-02-01T11:09:27Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5XXZY5YL87592N34454913", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAYID-MW5W3EQ07B7347381105242R", + "intent": "sale", + "state": "approved", + "cart": "2FM93347R66426228", + "payer": { + "payment_method": "paypal", + "status": "UNVERIFIED", + "payer_info": { + "email": "admin@admin.com", + "first_name": "d", + "last_name": "d", + "payer_id": "N4KZ3KK4C2DFQ", + "shipping_address": { + "recipient_name": "d d", + "line1": "ytu", + "line2": "tyut", + "city": "tyut", + "state": "CA", + "postal_code": "95388", + "country_code": "US" + }, + "phone": "8219746756", + "country_code": "US" + } + }, + "transactions": [ + { + "reference_id": "default", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "soft_descriptor": "PAYPAL *JOHNMERCHAN JOHNMERCHAN", + "item_list": { + "shipping_address": { + "recipient_name": "d d", + "line1": "ytu", + "line2": "tyut", + "city": "tyut", + "state": "CA", + "postal_code": "95388", + "country_code": "US" + } + }, + "related_resources": [ + { + "sale": { + "id": "17C62595GV9382350", + "state": "completed", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "3.48", + "currency": "USD" + }, + "purchase_unit_reference_id": "default", + "receipt_id": "2958849288346255", + "parent_payment": "PAYID-MW5W3EQ07B7347381105242R", + "create_time": "2024-02-01T10:08:18Z", + "update_time": "2024-02-01T10:08:18Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/17C62595GV9382350", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/17C62595GV9382350/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5W3EQ07B7347381105242R", + "rel": "parent_payment", + "method": "GET" + } + ], + "soft_descriptor": "PAYPAL *JOHNMERCHAN JOHNMERCHAN" + } + } + ] + } + ], + "create_time": "2024-02-01T10:08:18Z", + "update_time": "2024-02-01T10:08:18Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5W3EQ07B7347381105242R", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAYID-MW5WOVY25V45764MA349022Y", + "intent": "sale", + "state": "approved", + "cart": "7CP287511G6711412", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "ABpaypal@gmail.com", + "first_name": "kiran", + "last_name": "ingale", + "payer_id": "MEZSMT5X3R5HW", + "shipping_address": { + "recipient_name": "kiran ingale", + "line1": "3210 D Street", + "city": "SANFORD", + "state": "CA", + "postal_code": "27331", + "country_code": "US" + }, + "phone": "3333333333", + "country_code": "US" + } + }, + "transactions": [ + { + "reference_id": "default", + "amount": { + "total": "1434.45", + "currency": "USD", + "details": { + "subtotal": "1434.45", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "soft_descriptor": "JOHNMERCHAN JOHNMERCHAN", + "item_list": { + "shipping_address": { + "recipient_name": "kiran ingale", + "line1": "3210 D Street", + "city": "SANFORD", + "state": "CA", + "postal_code": "27331", + "country_code": "US" + } + }, + "related_resources": [ + { + "sale": { + "id": "5SF16542W66927019", + "state": "refunded", + "amount": { + "total": "1434.45", + "currency": "USD", + "details": { + "subtotal": "1434.45", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "50.55", + "currency": "USD" + }, + "purchase_unit_reference_id": "default", + "parent_payment": "PAYID-MW5WOVY25V45764MA349022Y", + "create_time": "2024-02-01T09:41:44Z", + "update_time": "2024-02-01T09:45:35Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/5SF16542W66927019", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/5SF16542W66927019/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5WOVY25V45764MA349022Y", + "rel": "parent_payment", + "method": "GET" + } + ], + "soft_descriptor": "JOHNMERCHAN JOHNMERCHAN" + } + }, + { + "refund": { + "id": "4P679266N7690881N", + "state": "completed", + "amount": { + "total": "-1434.45", + "currency": "USD" + }, + "parent_payment": "PAYID-MW5WOVY25V45764MA349022Y", + "sale_id": "5SF16542W66927019", + "create_time": "2024-02-01T09:45:35Z", + "update_time": "2024-02-01T09:45:35Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/refund/4P679266N7690881N", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5WOVY25V45764MA349022Y", + "rel": "parent_payment", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/5SF16542W66927019", + "rel": "sale", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T09:41:43Z", + "update_time": "2024-02-01T09:45:35Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW5WOVY25V45764MA349022Y", + "rel": "self", + "method": "GET" + } + ] + } + ], + "count": 3, + "next_id": "" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_page_init.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_page_init.json new file mode 100644 index 000000000000..025dfc513c85 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_files/token_page_init.json @@ -0,0 +1,399 @@ +{ + "payments": [ + { + "id": "PAYID-MW55RCA31D103955T218492B", + "intent": "sale", + "state": "approved", + "cart": "06J27273EH485262V", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "sb-vxpcr15413769@personal.example.com", + "first_name": "John", + "last_name": "Doe", + "payer_id": "TWL7BJVYNS7GU", + "shipping_address": { + "recipient_name": "John Doe", + "line1": "1 Main St", + "city": "San Jose", + "state": "CA", + "postal_code": "95131", + "country_code": "US" + }, + "phone": "4083068029", + "country_code": "US" + } + }, + "transactions": [ + { + "reference_id": "1000000000047", + "amount": { + "total": "343.80", + "currency": "USD", + "details": { + "subtotal": "343.80", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "item_list": { + "shipping_address": { + "recipient_name": "John Doe", + "line1": "1 Main St", + "city": "San Jose", + "state": "CA", + "postal_code": "95131", + "country_code": "US" + } + }, + "related_resources": [ + { + "sale": { + "id": "7PE037460E080360M", + "state": "completed", + "amount": { + "total": "343.80", + "currency": "USD", + "details": { + "subtotal": "343.80", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "12.49", + "currency": "USD" + }, + "purchase_unit_reference_id": "1000000000047", + "parent_payment": "PAYID-MW55RCA31D103955T218492B", + "create_time": "2024-02-01T17:44:40Z", + "update_time": "2024-02-01T17:44:40Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/7PE037460E080360M", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/7PE037460E080360M/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW55RCA31D103955T218492B", + "rel": "parent_payment", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T17:44:40Z", + "update_time": "2024-02-01T17:44:40Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW55RCA31D103955T218492B", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAYID-MW53UPA6UB45753B0034831X", + "intent": "sale", + "state": "approved", + "cart": "9A220393SG7753433", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "sb-g43l4x28821325@personal.example.com", + "first_name": "John", + "last_name": "Doe", + "payer_id": "889X39VDHV8QY", + "shipping_address": { + "recipient_name": "John Doe", + "line1": "Via Unit? d'Italia, 5783296", + "city": "Napoli", + "state": "Napoli", + "postal_code": "80127", + "country_code": "IT" + }, + "phone": "9393358454", + "country_code": "IT" + } + }, + "transactions": [ + { + "reference_id": "default", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E", + "email": "john_merchant@example.com" + }, + "description": "T-Shirt", + "item_list": { + "items": [ + { + "name": "T-Shirt", + "description": "Green XL", + "price": "100.00", + "currency": "USD", + "tax": "0.00", + "quantity": 1 + } + ], + "shipping_address": { + "recipient_name": "John Doe", + "line1": "Via Unit? d'Italia, 5783296", + "city": "Napoli", + "state": "Napoli", + "postal_code": "80127", + "country_code": "IT" + } + }, + "related_resources": [ + { + "sale": { + "id": "29N28023XB153584X", + "state": "completed", + "amount": { + "total": "100.00", + "currency": "USD", + "details": { + "subtotal": "100.00", + "shipping": "0.00", + "insurance": "0.00", + "handling_fee": "0.00", + "shipping_discount": "0.00", + "discount": "0.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "transaction_fee": { + "value": "5.48", + "currency": "USD" + }, + "receivable_amount": { + "value": "100.00", + "currency": "USD" + }, + "exchange_rate": "1.098848913950027", + "purchase_unit_reference_id": "default", + "parent_payment": "PAYID-MW53UPA6UB45753B0034831X", + "create_time": "2024-02-01T15:35:25Z", + "update_time": "2024-02-01T15:35:25Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/29N28023XB153584X", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/sale/29N28023XB153584X/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW53UPA6UB45753B0034831X", + "rel": "parent_payment", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T15:35:24Z", + "update_time": "2024-02-01T15:35:25Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAYID-MW53UPA6UB45753B0034831X", + "rel": "self", + "method": "GET" + } + ] + }, + { + "id": "PAY-81S181868H8011217MW526OI", + "intent": "authorize", + "state": "approved", + "payer": { + "payment_method": "paypal", + "status": "VERIFIED", + "payer_info": { + "email": "mihai.streza1@mi-pay.com", + "first_name": "Mihai", + "last_name": "Streza", + "payer_id": "QHD3E8SRDDSQL", + "shipping_address": { + "recipient_name": "Mihai Streza" + }, + "phone": "07534201211", + "country_code": "GB" + } + }, + "transactions": [ + { + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payee": { + "merchant_id": "C7CYMKZDG8D6E" + }, + "description": "topup", + "invoice_number": "100000000188917", + "soft_descriptor": "PAYPAL *JOHNMERCHAN", + "item_list": { + "items": [ + { + "name": "topup", + "price": "20.00", + "currency": "EUR", + "tax": "0.00", + "quantity": 1 + } + ], + "shipping_address": { + "recipient_name": "Mihai Streza" + } + }, + "related_resources": [ + { + "authorization": { + "id": "05D21713M12255848", + "state": "captured", + "amount": { + "total": "20.00", + "currency": "EUR", + "details": { + "subtotal": "20.00" + } + }, + "payment_mode": "INSTANT_TRANSFER", + "protection_eligibility": "ELIGIBLE", + "protection_eligibility_type": "ITEM_NOT_RECEIVED_ELIGIBLE,UNAUTHORIZED_PAYMENT_ELIGIBLE", + "billing_agreement_id": "B-2B029484VC167663Y", + "parent_payment": "PAY-81S181868H8011217MW526OI", + "valid_until": "2024-03-01T14:48:26Z", + "create_time": "2024-02-01T14:48:26Z", + "update_time": "2024-02-01T14:48:30Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/capture", + "rel": "capture", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/void", + "rel": "void", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848/reauthorize", + "rel": "reauthorize", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI", + "rel": "parent_payment", + "method": "GET" + } + ] + } + }, + { + "capture": { + "id": "546282867R0022639", + "amount": { + "total": "20.00", + "currency": "EUR" + }, + "state": "completed", + "custom": "", + "transaction_fee": { + "value": "1.39", + "currency": "EUR" + }, + "parent_payment": "PAY-81S181868H8011217MW526OI", + "invoice_number": "100000000188917", + "create_time": "2024-02-01T14:48:30Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/546282867R0022639", + "rel": "self", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/capture/546282867R0022639/refund", + "rel": "refund", + "method": "POST" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/authorization/05D21713M12255848", + "rel": "authorization", + "method": "GET" + }, + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI", + "rel": "parent_payment", + "method": "GET" + } + ] + } + } + ] + } + ], + "create_time": "2024-02-01T14:48:25Z", + "links": [ + { + "href": "https://api.sandbox.paypal.com/v1/payments/payment/PAY-81S181868H8011217MW526OI", + "rel": "self", + "method": "GET" + } + ] + } + ], + "count": 3, + "next_id": "PAY-0L38757939422510JMW5ZJVA" +} diff --git a/airbyte-integrations/connectors/source-paystack/main.py b/airbyte-integrations/connectors/source-paystack/main.py index 04c982f1ed90..8f792800f6de 100644 --- a/airbyte-integrations/connectors/source-paystack/main.py +++ b/airbyte-integrations/connectors/source-paystack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_paystack import SourcePaystack +from source_paystack.run import run if __name__ == "__main__": - source = SourcePaystack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-paystack/metadata.yaml b/airbyte-integrations/connectors/source-paystack/metadata.yaml index d65d615b940b..203dd02b642c 100644 --- a/airbyte-integrations/connectors/source-paystack/metadata.yaml +++ b/airbyte-integrations/connectors/source-paystack/metadata.yaml @@ -11,6 +11,10 @@ data: icon: paystack.svg license: MIT name: Paystack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-paystack registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 946504314000..8d37937dc667 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( + entry_points={ + "console_scripts": [ + "source-paystack=source_paystack.run:run", + ], + }, name="source_paystack", description="Source implementation for Paystack.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/run.py b/airbyte-integrations/connectors/source-paystack/source_paystack/run.py new file mode 100644 index 000000000000..001e5668317e --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paystack import SourcePaystack + + +def run(): + source = SourcePaystack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pendo/main.py b/airbyte-integrations/connectors/source-pendo/main.py index 6d17f4106c1d..882482d947f8 100644 --- a/airbyte-integrations/connectors/source-pendo/main.py +++ b/airbyte-integrations/connectors/source-pendo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pendo import SourcePendo +from source_pendo.run import run if __name__ == "__main__": - source = SourcePendo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pendo/metadata.yaml b/airbyte-integrations/connectors/source-pendo/metadata.yaml index c3f0ee28127a..be8343b55fa1 100644 --- a/airbyte-integrations/connectors/source-pendo/metadata.yaml +++ b/airbyte-integrations/connectors/source-pendo/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pendo.svg license: MIT name: Pendo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pendo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pendo/setup.py b/airbyte-integrations/connectors/source-pendo/setup.py index 8aa15183ac48..907aa867ff7b 100644 --- a/airbyte-integrations/connectors/source-pendo/setup.py +++ b/airbyte-integrations/connectors/source-pendo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pendo=source_pendo.run:run", + ], + }, name="source_pendo", description="Source implementation for Pendo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pendo/source_pendo/run.py b/airbyte-integrations/connectors/source-pendo/source_pendo/run.py new file mode 100644 index 000000000000..a98bdc7d16ff --- /dev/null +++ b/airbyte-integrations/connectors/source-pendo/source_pendo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pendo import SourcePendo + + +def run(): + source = SourcePendo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-persistiq/main.py b/airbyte-integrations/connectors/source-persistiq/main.py index 438306b66706..b6be0062f615 100644 --- a/airbyte-integrations/connectors/source-persistiq/main.py +++ b/airbyte-integrations/connectors/source-persistiq/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_persistiq import SourcePersistiq +from source_persistiq.run import run if __name__ == "__main__": - source = SourcePersistiq() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-persistiq/metadata.yaml b/airbyte-integrations/connectors/source-persistiq/metadata.yaml index ded7693cd2c9..90a020fc773b 100644 --- a/airbyte-integrations/connectors/source-persistiq/metadata.yaml +++ b/airbyte-integrations/connectors/source-persistiq/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.persistiq.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-persistiq registries: oss: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/persistiq tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py index 88f883fbdacf..509b4ddc1797 100644 --- a/airbyte-integrations/connectors/source-persistiq/setup.py +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-persistiq=source_persistiq.run:run", + ], + }, name="source_persistiq", description="Source implementation for Persistiq.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py new file mode 100644 index 000000000000..2f604abf6ae5 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_persistiq import SourcePersistiq + + +def run(): + source = SourcePersistiq() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pexels-api/main.py b/airbyte-integrations/connectors/source-pexels-api/main.py index f13d88a9f301..4fdc405e120e 100644 --- a/airbyte-integrations/connectors/source-pexels-api/main.py +++ b/airbyte-integrations/connectors/source-pexels-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pexels_api import SourcePexelsApi +from source_pexels_api.run import run if __name__ == "__main__": - source = SourcePexelsApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml index 5a90b9a93769..9e5e8cab8fa3 100644 --- a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pexels.svg license: MIT name: Pexels API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pexels-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pexels-api/setup.py b/airbyte-integrations/connectors/source-pexels-api/setup.py index e45f600af446..7525bf7113f5 100644 --- a/airbyte-integrations/connectors/source-pexels-api/setup.py +++ b/airbyte-integrations/connectors/source-pexels-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pexels-api=source_pexels_api.run:run", + ], + }, name="source_pexels_api", description="Source implementation for Pexels Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py new file mode 100644 index 000000000000..da128a40cea0 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pexels_api import SourcePexelsApi + + +def run(): + source = SourcePexelsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pinterest/.coveragerc b/airbyte-integrations/connectors/source-pinterest/.coveragerc new file mode 100644 index 000000000000..a0c6e7c485cb --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_pinterest/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/README.md b/airbyte-integrations/connectors/source-pinterest/README.md index e22d33de28b5..71c73a2027e5 100644 --- a/airbyte-integrations/connectors/source-pinterest/README.md +++ b/airbyte-integrations/connectors/source-pinterest/README.md @@ -1,69 +1,55 @@ -# Pinterest Source +# Pinterest source connector + This is the repository for the Pinterest source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pinterest). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pinterest). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pinterest) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pinterest/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pinterest) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pinterest/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pinterest test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-pinterest spec +poetry run source-pinterest check --config secrets/config.json +poetry run source-pinterest discover --config secrets/config.json +poetry run source-pinterest read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-pinterest build ``` -An image will be built with the tag `airbyte/source-pinterest:dev`. +An image will be available on your host with the tag `airbyte/source-pinterest:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-pinterest:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-pinterest:dev spec @@ -72,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pinterest:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pinterest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-pinterest test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pinterest test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pinterest.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/pinterest.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl index 2d6f69c6364d..010cf0912179 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl @@ -1,18 +1,18 @@ {"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470} -{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1699893121669} -{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699373013, "view_tracking_url": null, "lead_form_id": null}, "emitted_at": 1699393433303} -{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893196846} -{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699373013.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID"}, "emitted_at": 1699393433712} -{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893280169} -{"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019} +{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-12-04", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1708094337349} +{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699394846, "view_tracking_url": null, "lead_form_id": null, "grid_click_type": "DIRECT_TO_DESTINATION", "customizable_cta_type": null}, "emitted_at": 1708094430503} +{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-12-04", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1708094509254} +{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699394439.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID", "targeting_template_ids": null}, "emitted_at": 1708094572628} +{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-12-04", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1708094657011} +{"stream": "boards", "data": {"created_at": "2021-06-08T09:37:18", "privacy": "PUBLIC", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "collaborator_count": 0, "description": "", "follower_count": 3, "id": "666743988523388559", "pin_count": 1, "name": "business", "owner": {"username": "integrationtest0375"}}, "emitted_at": 1708094711013} {"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true,"link":"http://airbyte.io/", "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "is_standard": true}, "emitted_at": 1698398201666} {"stream": "board_sections", "data": {"name": "Airbyte_board_section_new", "id": "5195035116725909603"}, "emitted_at": 1699893323493} {"stream": "board_section_pins","data":{"id":"666743919837294988","dominant_color":"#cacafe","pin_metrics":null,"title":"Airbyte","creative_type":"REGULAR","link":"http://airbyte.io/","board_id":"666743988523388559","created_at":"2021-06-08T09:37:30","is_owner":true,"description":"Data Integration","note":"","alt_text":null,"board_section_id":"5195034916661798218","parent_pin_id":null,"product_tags":[],"board_owner":{"username":"integrationtest0375"},"is_standard":true,"has_been_promoted":true},"emitted_at":1699893364884} {"stream": "campaigns", "data": {"id": "626744128956", "ad_account_id": "549761668032", "name": "2021-06-09 | Traffic | Keywords | Data Integration", "status": "ACTIVE", "objective_type": "CONSIDERATION", "lifetime_spend_cap": 0, "daily_spend_cap": 3000000, "order_line_id": null, "tracking_urls": null, "created_time": 1623245885, "updated_time": 1691447502, "type": "campaign", "is_flexible_daily_budgets": false, "summary_status": "COMPLETED", "is_campaign_budget_optimization": true, "start_time": 1623196800, "end_time": 1624060800}, "emitted_at": 1699393571700} -{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699894065462} +{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-12-04", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1708094774057} {"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} {"stream": "campaign_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699894287823} -{"stream": "user_account_analytics", "data": {"date": "2023-11-09", "data_status": "READY", "metrics": {"SAVE": 2.0, "OUTBOUND_CLICK_RATE": 0.0043859649122807015, "IMPRESSION": 912.0, "VIDEO_START": 0, "SAVE_RATE": 0.0021929824561403508, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT": 22.0, "VIDEO_AVG_WATCH_TIME": 0.0, "ENGAGEMENT_RATE": 0.02412280701754386, "PIN_CLICK": 17, "VIDEO_10S_VIEW": 0, "FULL_SCREEN_PLAY": 0, "CLOSEUP_RATE": 0.017543859649122806, "FULL_SCREEN_PLAYTIME": 0, "VIDEO_V50_WATCH_TIME": 0, "VIDEO_MRC_VIEW": 0, "CLICKTHROUGH": 4.0, "CLICKTHROUGH_RATE": 0.0043859649122807015, "OUTBOUND_CLICK": 4, "CLOSEUP": 16.0, "PIN_CLICK_RATE": 0.01864035087719298}}, "emitted_at": 1699894362486} +{"stream": "user_account_analytics", "data": {"date": "2024-02-18", "data_status": "READY", "metrics": {"SAVE": 6.0, "VIDEO_MRC_VIEW": 0, "VIDEO_10S_VIEW": 0, "PIN_CLICK": 15, "OUTBOUND_CLICK_RATE": 0.0017123287671232876, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT_RATE": 0.03767123287671233, "VIDEO_START": 0, "IMPRESSION": 584.0, "OUTBOUND_CLICK": 1, "ENGAGEMENT": 22.0, "PIN_CLICK_RATE": 0.025684931506849314, "VIDEO_V50_WATCH_TIME": 0, "SAVE_RATE": 0.010273972602739725, "VIDEO_AVG_WATCH_TIME": 0.0}}, "emitted_at": 1708476886035} {"stream": "keywords", "data": {"archived": false, "id": "2886935172273", "parent_id": "2680068678965", "parent_type": "AD_GROUP", "type": "KEYWORD", "bid": null, "match_type": "BROAD", "value": "data science"}, "emitted_at": 1699393669235} {"stream": "audiences", "data": {"type": "audience", "id": "2542622254639", "name": "airbyte audience", "ad_account_id": "549761668032", "audience_type": "ENGAGEMENT", "description": "airbyte audience", "status": "TOO_SMALL", "rule": {"engager_type": 1}}, "emitted_at": 1699293090886} {"stream": "advertiser_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "EENGAGEMENT_RATE": 0.1, "ENGAGEMENT_2": 1.0, "IMPRESSION_2": 10.0, "REPIN_2": 1.0, "TOTAL_ENGAGEMENT": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 5.0, "TOTAL_IMPRESSION_USER": 2.0, "TOTAL_REPIN_RATE": 0.1, "DATE": "2023-02-10"}, "emitted_at": 1699894848024} diff --git a/airbyte-integrations/connectors/source-pinterest/main.py b/airbyte-integrations/connectors/source-pinterest/main.py index 377a6ad6dd76..aff013c70319 100644 --- a/airbyte-integrations/connectors/source-pinterest/main.py +++ b/airbyte-integrations/connectors/source-pinterest/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pinterest import SourcePinterest +from source_pinterest.run import run if __name__ == "__main__": - source = SourcePinterest() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index e496ea4154c6..e541750bdf51 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 1.1.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-pinterest connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c @@ -13,6 +13,10 @@ data: icon: pinterest.svg license: MIT name: Pinterest + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pinterest registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pinterest/poetry.lock b/airbyte-integrations/connectors/source-pinterest/poetry.lock new file mode 100644 index 000000000000..f3f3fee593a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/poetry.lock @@ -0,0 +1,1053 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.13.4" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "responses-0.13.4-py2.py3-none-any.whl", hash = "sha256:d8d0f655710c46fd3513b9202a7f0dcedd02ca0f8cf4976f27fa8ab5b81e656d"}, + {file = "responses-0.13.4.tar.gz", hash = "sha256:9476775d856d3c24ae660bbebe29fb6d789d4ad16acd723efbfb6ee20990b899"}, +] + +[package.dependencies] +requests = ">=2.0" +six = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=3.7.1,<6.0.0)", "flake8", "mypy", "pytest (>=4.6)", "pytest (>=4.6,<5.0)", "pytest-cov", "pytest-localserver", "types-mock", "types-requests", "types-six"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "07657dff3674cca2f376658bfa717048c9caff7fa3b6f23144133f1d6dbac33a" diff --git a/airbyte-integrations/connectors/source-pinterest/pyproject.toml b/airbyte-integrations/connectors/source-pinterest/pyproject.toml new file mode 100644 index 000000000000..486999671d09 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.2.0" +name = "source-pinterest" +description = "Source implementation for Pinterest." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/pinterest" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_pinterest" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +pendulum = "==2.1.2" +airbyte-cdk = "==0.63.2" + +[tool.poetry.scripts] +source-pinterest = "source_pinterest.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" +responses = "^0.13.3" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-pinterest/requirements.txt b/airbyte-integrations/connectors/source-pinterest/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-pinterest/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py deleted file mode 100644 index 5da646d8e719..000000000000 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum~=2.1.2"] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "responses~=0.13.3", - "requests-mock", -] - -setup( - name="source_pinterest", - description="Source implementation for Pinterest.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py new file mode 100644 index 000000000000..2acf5886cb16 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pinterest import SourcePinterest + + +def run(): + source = SourcePinterest() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pipedrive/Dockerfile b/airbyte-integrations/connectors/source-pipedrive/Dockerfile index f82230e8fdda..8698c9a3fad5 100644 --- a/airbyte-integrations/connectors/source-pipedrive/Dockerfile +++ b/airbyte-integrations/connectors/source-pipedrive/Dockerfile @@ -34,5 +34,5 @@ COPY source_pipedrive ./source_pipedrive ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.2.1 +LABEL io.airbyte.version=2.2.2 LABEL io.airbyte.name=airbyte/source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/main.py b/airbyte-integrations/connectors/source-pipedrive/main.py index fb481bc2e9b2..64fe456c34fd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/main.py +++ b/airbyte-integrations/connectors/source-pipedrive/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pipedrive import SourcePipedrive +from source_pipedrive.run import run if __name__ == "__main__": - source = SourcePipedrive() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index e38dd09b1c37..b52b392e7a85 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -5,6 +5,10 @@ data: allowedHosts: hosts: - api.pipedrive.com # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pipedrive registries: cloud: enabled: true @@ -13,7 +17,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 2.2.1 + dockerImageTag: 2.2.2 dockerRepository: airbyte/source-pipedrive documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive @@ -29,5 +33,5 @@ data: releaseStage: alpha supportLevel: community tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py index 7e72591c8797..388dc3e91906 100644 --- a/airbyte-integrations/connectors/source-pipedrive/setup.py +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pipedrive=source_pipedrive.run:run", + ], + }, name="source_pipedrive", description="Source implementation for Pipedrive.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py new file mode 100644 index 000000000000..2ff2b80c12a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pipedrive import SourcePipedrive + + +def run(): + source = SourcePipedrive() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/main.py b/airbyte-integrations/connectors/source-pivotal-tracker/main.py index 3115bba2f932..767718108296 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/main.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pivotal_tracker import SourcePivotalTracker +from source_pivotal_tracker.run import run if __name__ == "__main__": - source = SourcePivotalTracker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml index 013eac02808a..a41a1713b016 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml +++ b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pivotal-tracker.svg license: MIT name: Pivotal Tracker + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pivotal-tracker registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py index 63de73cbd36f..050e33c2a4f7 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pivotal-tracker=source_pivotal_tracker.run:run", + ], + }, name="source_pivotal_tracker", description="Source implementation for Pivotal Tracker.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py new file mode 100644 index 000000000000..a187ff169ac2 --- /dev/null +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pivotal_tracker import SourcePivotalTracker + + +def run(): + source = SourcePivotalTracker() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-plaid/main.py b/airbyte-integrations/connectors/source-plaid/main.py index e8f3663ddccc..be0011cc4860 100644 --- a/airbyte-integrations/connectors/source-plaid/main.py +++ b/airbyte-integrations/connectors/source-plaid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_plaid import SourcePlaid +from source_plaid.run import run if __name__ == "__main__": - source = SourcePlaid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-plaid/metadata.yaml b/airbyte-integrations/connectors/source-plaid/metadata.yaml index e05ecb3ad445..a8407c8ad2b1 100644 --- a/airbyte-integrations/connectors/source-plaid/metadata.yaml +++ b/airbyte-integrations/connectors/source-plaid/metadata.yaml @@ -8,6 +8,10 @@ data: icon: plaid.svg license: MIT name: Plaid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-plaid registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-plaid/setup.py b/airbyte-integrations/connectors/source-plaid/setup.py index bcbf0b34a80c..fc6368795555 100644 --- a/airbyte-integrations/connectors/source-plaid/setup.py +++ b/airbyte-integrations/connectors/source-plaid/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-plaid=source_plaid.run:run", + ], + }, name="source_plaid", description="Source implementation for Plaid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/run.py b/airbyte-integrations/connectors/source-plaid/source_plaid/run.py new file mode 100644 index 000000000000..c83935220787 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_plaid import SourcePlaid + + +def run(): + source = SourcePlaid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-plausible/main.py b/airbyte-integrations/connectors/source-plausible/main.py index 581156e36e0f..453a2f457795 100644 --- a/airbyte-integrations/connectors/source-plausible/main.py +++ b/airbyte-integrations/connectors/source-plausible/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_plausible import SourcePlausible +from source_plausible.run import run if __name__ == "__main__": - source = SourcePlausible() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-plausible/metadata.yaml b/airbyte-integrations/connectors/source-plausible/metadata.yaml index 042fa88dd49a..086d1c33a3c0 100644 --- a/airbyte-integrations/connectors/source-plausible/metadata.yaml +++ b/airbyte-integrations/connectors/source-plausible/metadata.yaml @@ -8,6 +8,10 @@ data: icon: plausible.svg license: MIT name: Plausible + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-plausible registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-plausible/setup.py b/airbyte-integrations/connectors/source-plausible/setup.py index d222145d848e..894526785513 100644 --- a/airbyte-integrations/connectors/source-plausible/setup.py +++ b/airbyte-integrations/connectors/source-plausible/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-plausible=source_plausible.run:run", + ], + }, name="source_plausible", description="Source implementation for Plausible.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plausible/source_plausible/run.py b/airbyte-integrations/connectors/source-plausible/source_plausible/run.py new file mode 100644 index 000000000000..57370a5f3748 --- /dev/null +++ b/airbyte-integrations/connectors/source-plausible/source_plausible/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_plausible import SourcePlausible + + +def run(): + source = SourcePlausible() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pocket/main.py b/airbyte-integrations/connectors/source-pocket/main.py index bf2c5d41e121..45d6104c34f8 100644 --- a/airbyte-integrations/connectors/source-pocket/main.py +++ b/airbyte-integrations/connectors/source-pocket/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pocket import SourcePocket +from source_pocket.run import run if __name__ == "__main__": - source = SourcePocket() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pocket/metadata.yaml b/airbyte-integrations/connectors/source-pocket/metadata.yaml index 145bf1b927e4..219e92e32df2 100644 --- a/airbyte-integrations/connectors/source-pocket/metadata.yaml +++ b/airbyte-integrations/connectors/source-pocket/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pocket.svg license: MIT name: Pocket + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pocket registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pocket/setup.py b/airbyte-integrations/connectors/source-pocket/setup.py index bdbf2b6549ab..d7238d34a09e 100644 --- a/airbyte-integrations/connectors/source-pocket/setup.py +++ b/airbyte-integrations/connectors/source-pocket/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pocket=source_pocket.run:run", + ], + }, name="source_pocket", description="Source implementation for Pocket.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pocket/source_pocket/run.py b/airbyte-integrations/connectors/source-pocket/source_pocket/run.py new file mode 100644 index 000000000000..75eaea5398f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pocket/source_pocket/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pocket import SourcePocket + + +def run(): + source = SourcePocket() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 38a510a3f2d7..f32ce6b38105 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,11 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys - -from airbyte_cdk.entrypoint import launch -from source_pokeapi import SourcePokeapi +from source_pokeapi.run import run if __name__ == "__main__": - source = SourcePokeapi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 2597fc4c5eda..2f6a410ea49b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pokeapi registries: oss: enabled: true @@ -16,10 +20,10 @@ data: icon: pokeapi.svg license: MIT name: PokeAPI - releaseDate: "2020-05-04" + releaseDate: "2020-05-14" releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index 2fa7839b58fc..27a5590ece4d 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pokeapi=source_pokeapi.run:run", + ], + }, name="source_pokeapi", description="Source implementation for Pokeapi.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py new file mode 100644 index 000000000000..2b573e693954 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pokeapi import SourcePokeapi + + +def run(): + source = SourcePokeapi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/main.py b/airbyte-integrations/connectors/source-polygon-stock-api/main.py index 7fc7ae2dd497..77c0f73430be 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/main.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_polygon_stock_api import SourcePolygonStockApi +from source_polygon_stock_api.run import run if __name__ == "__main__": - source = SourcePolygonStockApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml index 9be804fef482..897591536c31 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml @@ -11,6 +11,10 @@ data: icon: polygon.svg license: MIT name: Polygon Stock API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-polygon-stock-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py index ddd7463c9292..2b4930c184d3 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-polygon-stock-api=source_polygon_stock_api.run:run", + ], + }, name="source_polygon_stock_api", description="Source implementation for Polygon Stock Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py b/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py new file mode 100644 index 000000000000..ab2a75d41fe6 --- /dev/null +++ b/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_polygon_stock_api import SourcePolygonStockApi + + +def run(): + source = SourcePolygonStockApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 2b9d0e0e6e90..d68232e1e409 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -1,7 +1,6 @@ import org.jsonschema2pojo.SourceType plugins { - id 'application' id 'airbyte-java-connector' id "org.jsonschema2pojo" version "1.2.1" } @@ -13,49 +12,24 @@ java { } airbyteJavaConnector { - cdkVersionRequired = '0.7.1' - features = ['db-sources'] + cdkVersionRequired = '0.20.6' + features = ['db-sources', 'datastore-postgres'] useLocalCdk = false } - application { mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -// Add a configuration for our migrations tasks defined below to encapsulate their dependencies -configurations { - migrations.extendsFrom implementation -} - -configurations.all { - resolutionStrategy { - force 'org.jooq:jooq:3.13.4' - } -} - dependencies { - testImplementation libs.jooq - testImplementation libs.hikaricp - - migrations libs.testcontainers.postgresql - migrations sourceSets.main.output - - // Lombok - implementation libs.lombok - annotationProcessor libs.lombok + implementation 'commons-codec:commons-codec:1.16.0' + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'io.debezium:debezium-connector-postgres:2.4.0.Final' - implementation 'org.apache.commons:commons-lang3:3.11' - implementation libs.postgresql - implementation libs.bundles.datadog + testFixturesApi 'org.testcontainers:postgresql:1.19.0' testImplementation 'org.hamcrest:hamcrest-all:1.3' - testFixturesImplementation libs.testcontainers.jdbc - testFixturesImplementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.jdbc - testImplementation libs.testcontainers.postgresql - testImplementation libs.junit.jupiter.system.stubs } jsonSchema2Pojo { diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index 0b3d1bd43d4b..22a65152d31f 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.2.26 + dockerImageTag: 3.3.12 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres @@ -26,5 +26,4 @@ data: supportLevel: certified tags: - language:java - - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java index 898c250d5f65..476b4fefdeaa 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_LSN; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -12,7 +12,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.cdk.db.jdbc.JdbcDatabase; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -82,9 +82,9 @@ public static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); final JsonNode numberType = Jsons.jsonNode(ImmutableMap.of("type", "number")); - properties.set(DebeziumEventUtils.CDC_LSN, numberType); - properties.set(DebeziumEventUtils.CDC_UPDATED_AT, stringType); - properties.set(DebeziumEventUtils.CDC_DELETED_AT, stringType); + properties.set(DebeziumEventConverter.CDC_LSN, numberType); + properties.set(DebeziumEventConverter.CDC_UPDATED_AT, stringType); + properties.set(DebeziumEventConverter.CDC_DELETED_AT, stringType); return stream; } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 1e07cea42090..98519ba3f651 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -53,7 +53,6 @@ import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresReplicationConnection; import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils; @@ -61,7 +60,6 @@ import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; @@ -70,6 +68,7 @@ import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.ResultWithFailed; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; +import io.airbyte.integrations.source.postgres.cdc.PostgresReplicationConnection; import io.airbyte.integrations.source.postgres.ctid.CtidPerStreamStateManager; import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations; import io.airbyte.integrations.source.postgres.ctid.CtidStateManager; @@ -98,6 +97,8 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.sql.Connection; import java.sql.PreparedStatement; @@ -145,7 +146,6 @@ public class PostgresSource extends AbstractJdbcSource implements private Set publicizedTablesInCdc; private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); private int stateEmissionFrequency; - private XminStatus xminStatus; public static Source sshWrappedSource(PostgresSource source) { return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security"); @@ -175,7 +175,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { // https://github.com/airbytehq/airbyte/issues/24796 additionalParameters.add("prepareThreshold=0"); - final String encodedDatabaseName = HostPortResolver.encodeValue(config.get(JdbcUtils.DATABASE_KEY).asText()); + final String encodedDatabaseName = URLEncoder.encode(config.get(JdbcUtils.DATABASE_KEY).asText(), StandardCharsets.UTF_8); final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:postgresql://%s:%s/%s?", config.get(JdbcUtils.HOST_KEY).asText(), @@ -273,11 +273,6 @@ protected void logPreSyncDebugData(final JdbcDatabase database, final Configured indexInfo.close(); } - // Log and save the xmin status - this.xminStatus = PostgresQueryUtils.getXminStatus(database); - LOGGER.info(String.format("Xmin Status : {Number of wraparounds: %s, Xmin Transaction Value: %s, Xmin Raw Value: %s", - xminStatus.getNumWraparound(), xminStatus.getXminXidValue(), xminStatus.getXminRawValue())); - } @Override @@ -315,13 +310,15 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = getConnectionProperties(sourceConfig); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); @@ -481,6 +478,15 @@ public List> getIncrementalIterators(final } if (isAnyStreamIncrementalSyncMode(catalog) && PostgresUtils.isXmin(sourceConfig)) { + // Log and save the xmin status + final XminStatus xminStatus; + try { + xminStatus = PostgresQueryUtils.getXminStatus(database); + } catch (SQLException e) { + throw new RuntimeException(e); + } + LOGGER.info(String.format("Xmin Status : {Number of wraparounds: %s, Xmin Transaction Value: %s, Xmin Raw Value: %s", + xminStatus.getNumWraparound(), xminStatus.getXminXidValue(), xminStatus.getXminRawValue())); final StreamsCategorised streamsCategorised = categoriseStreams(stateManager, catalog, xminStatus); final ResultWithFailed> streamsUnderVacuum = streamsUnderVacuum(database, streamsCategorised.ctidStreams().streamsForCtidSync(), @@ -688,9 +694,6 @@ protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set @Override protected AirbyteStateType getSupportedStateType(final JsonNode config) { - if (!featureFlags.useStreamCapableState()) { - return AirbyteStateType.LEGACY; - } return PostgresUtils.isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSpecConstants.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSpecConstants.java new file mode 100644 index 000000000000..a2e89818ee13 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSpecConstants.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +// Constants defined in +// airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json. +public class PostgresSpecConstants { + + public static final String INVALID_CDC_CURSOR_POSITION_PROPERTY = "invalid_cdc_cursor_position_behavior"; + public static final String FAIL_SYNC_OPTION = "Fail sync"; + public static final String RESYNC_DATA_OPTION = "Re-sync data"; + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java index 707c72a2d127..bfd4903cef9f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java @@ -23,6 +23,7 @@ import static io.airbyte.integrations.source.postgres.PostgresType.VARCHAR; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.time.Duration; import java.util.List; @@ -50,6 +51,14 @@ public class PostgresUtils { private static final int MIN_QUEUE_SIZE = 1000; private static final int MAX_QUEUE_SIZE = 10000; + private static final String DROP_AGGREGATE_IF_EXISTS_STATEMENT = "DROP aggregate IF EXISTS EPHEMERAL_HEARTBEAT(float4)"; + private static final String CREATE_AGGREGATE_STATEMENT = "CREATE AGGREGATE EPHEMERAL_HEARTBEAT(float4) (SFUNC = float4pl, STYPE = float4)"; + private static final String DROP_AGGREGATE_STATEMENT = "DROP aggregate EPHEMERAL_HEARTBEAT(float4)"; + private static final List EPHEMERAL_HEARTBEAT_CREATE_STATEMENTS = + List.of(DROP_AGGREGATE_IF_EXISTS_STATEMENT, CREATE_AGGREGATE_STATEMENT, DROP_AGGREGATE_STATEMENT); + + private static final int POSTGRESQL_VERSION_15 = 15; + public static String getPluginValue(final JsonNode field) { return field.has("plugin") ? field.get("plugin").asText() : PGOUTPUT_PLUGIN; } @@ -186,4 +195,15 @@ public static String prettyPrintConfiguredAirbyteStreamList(final List "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); } + public static void advanceLsn(final JdbcDatabase database) { + try { + if (database.getMetaData().getDatabaseMajorVersion() < POSTGRESQL_VERSION_15) { + database.executeWithinTransaction(EPHEMERAL_HEARTBEAT_CREATE_STATEMENTS); + LOGGER.info("Succesfully forced LSN advancement by creating & dropping an ephemeral heartbeat aggregate"); + } + } catch (final Exception e) { + LOGGER.info("Failed to force LSN advancement by creating & dropping an ephemeral heartbeat aggregate."); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java index d5ce5cde32e3..c337542bf6aa 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.source.postgres.cdc; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java index da7ed8c7bd96..45c20156aab2 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java @@ -4,17 +4,20 @@ package io.airbyte.integrations.source.postgres.cdc; +import static io.airbyte.cdk.db.DbAnalyticsUtils.cdcCursorInvalidMessage; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.streamsUnderVacuum; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.FAIL_SYNC_OPTION; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; import static io.airbyte.integrations.source.postgres.PostgresUtils.isDebugMode; import static io.airbyte.integrations.source.postgres.PostgresUtils.prettyPrintConfiguredAirbyteStreamList; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumEventConverter; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; @@ -48,7 +51,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.OptionalLong; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -71,9 +73,9 @@ public static List> cdcCtidIteratorsCombin final JsonNode sourceConfig = database.getSourceConfig(); final Duration firstRecordWaitTime = PostgresUtils.getFirstRecordWaitTime(sourceConfig); final Duration subsequentRecordWaitTime = PostgresUtils.getSubsequentRecordWaitTime(sourceConfig); - final OptionalInt queueSize = OptionalInt.of(PostgresUtils.getQueueSize(sourceConfig)); + final int queueSize = PostgresUtils.getQueueSize(sourceConfig); LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); - LOGGER.info("Queue size: {}", queueSize.getAsInt()); + LOGGER.info("Queue size: {}", queueSize); if (isDebugMode(sourceConfig) && !PostgresUtils.shouldFlushAfterSync(sourceConfig)) { throw new ConfigErrorException("WARNING: The config indicates that we are clearing the WAL while reading data. This will mutate the WAL" + @@ -111,6 +113,12 @@ public static List> cdcCtidIteratorsCombin savedOffset); if (!savedOffsetAfterReplicationSlotLSN) { + AirbyteTraceMessageUtility.emitAnalyticsTrace(cdcCursorInvalidMessage()); + if (!sourceConfig.get("replication_method").has(INVALID_CDC_CURSOR_POSITION_PROPERTY) || sourceConfig.get("replication_method").get( + INVALID_CDC_CURSOR_POSITION_PROPERTY).asText().equals(FAIL_SYNC_OPTION)) { + throw new ConfigErrorException( + "Saved offset is before replication slot's confirmed lsn. Please reset the connection, and then increase WAL retention or reduce sync frequency to prevent this from happening in the future. See https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting#under-cdc-incremental-mode-there-are-still-full-refresh-syncs for more details."); + } LOGGER.warn("Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch"); } else if (!isDebugMode(sourceConfig) && PostgresUtils.shouldFlushAfterSync(sourceConfig)) { // We do not want to acknowledge the WAL logs in debug mode. @@ -172,19 +180,19 @@ public static List> cdcCtidIteratorsCombin // Gets the target position. final var targetPosition = PostgresCdcTargetPosition.targetPosition(database); + // Attempt to advance LSN past the target position. For versions of Postgres before PG15, this + // ensures that there is an event that debezium will + // receive that is after the target LSN. + PostgresUtils.advanceLsn(database); final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(sourceConfig, - targetPosition, false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize); + targetPosition, false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize, false); final PostgresCdcStateHandler postgresCdcStateHandler = new PostgresCdcStateHandler(stateManager); + final var propertiesManager = new RelationalDbDebeziumPropertiesManager( + PostgresCdcProperties.getDebeziumDefaultProperties(database), sourceConfig, catalog); + final var eventConverter = new RelationalDbDebeziumEventConverter(new PostgresCdcConnectorMetadataInjector(), emittedAt); final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators( - catalog, - new PostgresCdcSavedInfoFetcher(stateToBeUsed), - postgresCdcStateHandler, - new PostgresCdcConnectorMetadataInjector(), - PostgresCdcProperties.getDebeziumDefaultProperties(database), - DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, - emittedAt, - false); + propertiesManager, eventConverter, new PostgresCdcSavedInfoFetcher(stateToBeUsed), postgresCdcStateHandler); if (initialSyncCtidIterators.isEmpty()) { return Collections.singletonList(incrementalIteratorSupplier.get()); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java index 609bf9def3b9..745f92cfceaa 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresConverter; import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; import io.airbyte.integrations.source.postgres.PostgresSource; import io.airbyte.integrations.source.postgres.PostgresUtils; @@ -69,6 +68,11 @@ private static Properties commonProperties(final JdbcDatabase database) { : HEARTBEAT_INTERVAL; props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis())); + if (sourceConfig.get("replication_method").has("heartbeat_action_query") + && !sourceConfig.get("replication_method").get("heartbeat_action_query").asText().isEmpty()) { + props.setProperty("heartbeat.action.query", sourceConfig.get("replication_method").get("heartbeat_action_query").asText()); + } + if (PostgresUtils.shouldFlushAfterSync(sourceConfig)) { props.setProperty("flush.lsn.source", "false"); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcTargetPosition.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcTargetPosition.java index 206c870193db..d669db72755c 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcTargetPosition.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.postgres; +package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresConverter.java similarity index 99% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresConverter.java index 1cfdbbf8eda0..9dd44b1c5cef 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresConverter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.postgres; +package io.airbyte.integrations.source.postgres.cdc; import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToDate; import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToTime; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCustomLoader.java similarity index 92% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCustomLoader.java index 09b4dcb45240..bccfccb0cc6c 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCustomLoader.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.postgres; +package io.airbyte.integrations.source.postgres.cdc; import io.airbyte.commons.json.Jsons; import io.debezium.connector.postgresql.PostgresConnectorConfig; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresDebeziumStateUtil.java similarity index 95% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresDebeziumStateUtil.java index 174c03893fa2..4bff1cd79829 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresDebeziumStateUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.postgres; +package io.airbyte.integrations.source.postgres.cdc; import static io.debezium.connector.postgresql.PostgresOffsetContext.LAST_COMMIT_LSN_KEY; import static io.debezium.connector.postgresql.SourceInfo.LSN_KEY; @@ -83,10 +83,9 @@ public OptionalLong savedOffset(final Properties baseProperties, final ConfiguredAirbyteCatalog catalog, final JsonNode cdcState, final JsonNode config) { - final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog, - AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty()), - Optional.empty()); - final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + final var offsetManager = AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty()); + final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); return parseSavedOffset(debeziumProperties); } @@ -135,13 +134,6 @@ private ChainedLogicalStreamBuilder addSlotOption(final String publicationName, if (pgConnection.haveMinimumServerVersion(140000)) { streamBuilder = streamBuilder.withSlotOption("messages", true); } - } else if (plugin.equalsIgnoreCase("wal2json")) { - streamBuilder = streamBuilder - .withSlotOption("pretty-print", 1) - .withSlotOption("write-in-chunks", 1) - .withSlotOption("include-xids", 1) - .withSlotOption("include-timestamp", 1) - .withSlotOption("include-not-null", "true"); } else { throw new RuntimeException("Unknown plugin value : " + plugin); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresReplicationConnection.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresReplicationConnection.java index 85f10313db41..1490a4c8cb50 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresReplicationConnection.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals.postgres; +package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcUtils; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index f00b42a5507a..e1cc6ff8b367 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -289,6 +289,21 @@ ], "default": "After loading Data in the destination", "order": 7 + }, + "heartbeat_action_query": { + "type": "string", + "title": "Debezium heartbeat query (Advanced)", + "description": "Specifies a query that the connector executes on the source database when the connector sends a heartbeat message. Please see the setup guide for how and when to configure this setting.", + "default": "", + "order": 8 + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 9 } } }, diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java index 028aea409c15..26f6f319f1df 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -34,11 +32,6 @@ public abstract class AbstractPostgresSourceSSLCertificateAcceptanceTest extends protected PostgresTestDatabase testdb; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CERT) @@ -105,9 +98,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index bce259aa454d..db8552a90e09 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -13,8 +13,6 @@ import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; @@ -77,11 +75,6 @@ private static Database getDatabaseFromConfig(final JsonNode config) { public abstract SshTunnel.TunnelMethod getTunnelMethod(); - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - // todo (cgardens) - dynamically create data by generating a database with a random name instead of // requiring data to already be in place. @Override @@ -141,9 +134,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java index fab5aa18cff8..fb063718b8d5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; @@ -18,11 +16,6 @@ public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgr private static final String SCHEMA_NAME = "test"; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected Database setupDatabase() throws Exception { testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index 41f7485cae03..8e76d19c564f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -10,8 +10,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -42,11 +40,6 @@ public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAccep protected PostgresTestDatabase testdb; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { testdb = PostgresTestDatabase.in(getServerImage(), ContainerModifier.CONF) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java index 28bd8523eb93..c099d9bce930 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -66,11 +64,6 @@ protected void postSetup() throws Exception { } } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected Database setupDatabase() { testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java index 98884e5382d7..07ca597b64ee 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java @@ -43,9 +43,7 @@ public class CloudDeploymentPostgresSourceAcceptanceTest extends SourceAcceptanc @Override protected FeatureFlags featureFlags() { return FeatureFlagsWrapper.overridingDeploymentMode( - FeatureFlagsWrapper.overridingUseStreamCapableState( - super.featureFlags(), - true), + super.featureFlags(), AdaptiveSourceRunner.CLOUD_MODE); } @@ -123,9 +121,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index 29ab96ad4c39..e657a8886ecb 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -12,8 +12,6 @@ import io.airbyte.cdk.db.Database; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -43,11 +41,6 @@ public class PostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptan private PostgresTestDatabase testdb; private JsonNode config; - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { testdb = PostgresTestDatabase.in(getServerImage()); @@ -98,11 +91,6 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - @Test public void testFullRefreshWithRevokingSchemaPermissions() throws Exception { prepareEnvForUserWithoutPermissions(testdb.getDatabase()); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java index a02ac8352de2..12b50a733c29 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; @@ -15,11 +13,6 @@ public class PostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected Database setupDatabase() throws SQLException { testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java index 3af2e4df2b1f..2986f35a38e9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -40,11 +38,6 @@ protected JsonNode getConfig() throws Exception { .build(); } - @Override - protected FeatureFlags featureFlags() { - return FeatureFlagsWrapper.overridingUseStreamCapableState(super.featureFlags(), true); - } - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_12) @@ -100,9 +93,4 @@ protected JsonNode getState() throws Exception { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json index 375ea5024c99..1243163310b0 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json @@ -290,6 +290,21 @@ ], "default": "After loading Data in the destination", "order": 7 + }, + "heartbeat_action_query": { + "type": "string", + "title": "Debezium heartbeat query (Advanced)", + "description": "Specifies a query that the connector executes on the source database when the connector sends a heartbeat message. Please see the setup guide for how and when to configure this setting.", + "default": "", + "order": 8 + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 9 } } }, diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json index 8b09d54fd1b6..040446878181 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json @@ -289,6 +289,21 @@ ], "default": "After loading Data in the destination", "order": 7 + }, + "heartbeat_action_query": { + "type": "string", + "title": "Debezium heartbeat query (Advanced)", + "description": "Specifies a query that the connector executes on the source database when the connector sends a heartbeat message. Please see the setup guide for how and when to configure this setting.", + "default": "", + "order": 8 + }, + "invalid_cdc_cursor_position_behavior": { + "type": "string", + "title": "Invalid CDC position behavior (Advanced)", + "description": "Determines whether Airbyte should fail or re-sync data in case of an stale/invalid cursor value into the WAL. If 'Fail sync' is chosen, a user will have to manually reset the connection before being able to continue syncing data. If 'Re-sync data' is chosen, Airbyte will automatically trigger a refresh but could lead to higher cloud costs and data loss.", + "enum": ["Fail sync", "Re-sync data"], + "default": "Fail sync", + "order": 9 } } }, diff --git a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java index 3de43a56b014..64a21f76a100 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java @@ -29,9 +29,7 @@ protected JsonNode getConfig() { } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - } + protected void tearDown(final TestDestinationEnv testEnv) {} @Override protected String getImageName() { diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java index a6d7ecb4d970..53ae215c4da4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java @@ -4,13 +4,28 @@ package io.airbyte.integrations.source.postgres; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Order; @Order(2) public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { - protected static String getServerImageName() { - return "debezium/postgres:13-bullseye"; + @Override + protected void setBaseImage() { + this.postgresImage = BaseImage.POSTGRES_12; + } + + @Override + @Disabled("https://github.com/airbytehq/airbyte/issues/35267") + public void newTableSnapshotTest() { + + } + + @Override + @Disabled("https://github.com/airbytehq/airbyte/issues/35267") + public void syncShouldIncrementLSN() { + } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 71c0b03ca2a9..c51a2ad086d2 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -6,9 +6,11 @@ import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.FAIL_SYNC_OPTION; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.RESYNC_DATA_OPTION; import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.USE_TEST_CHUNK_SIZE; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,17 +35,15 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.debezium.CdcSourceTest; import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresReplicationConnection; import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; +import io.airbyte.integrations.source.postgres.cdc.PostgresCdcTargetPosition; +import io.airbyte.integrations.source.postgres.cdc.PostgresReplicationConnection; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -75,16 +75,21 @@ @Order(1) public class CdcPostgresSourceTest extends CdcSourceTest { + protected BaseImage postgresImage; + + protected void setBaseImage() { + this.postgresImage = getServerImage(); + } + @Override protected PostgresTestDatabase createTestDatabase() { - return PostgresTestDatabase.in(getServerImage(), ContainerModifier.CONF).withReplicationSlot(); + setBaseImage(); + return PostgresTestDatabase.in(this.postgresImage, ContainerModifier.CONF).withReplicationSlot(); } @Override protected PostgresSource source() { - final var source = new PostgresSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new PostgresSource(); } @Override @@ -92,8 +97,9 @@ protected JsonNode config() { return testdb.testConfigBuilder() .withSchemas(modelsSchema(), modelsSchema() + "_random") .withoutSsl() - .withCdcReplication("After loading Data in the destination") + .withCdcReplication("After loading Data in the destination", RESYNC_DATA_OPTION) .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .with("heartbeat_action_query", "") .build(); } @@ -104,12 +110,21 @@ protected void setup() { testdb.withPublicationForAllTables(); } + // For legacy Postgres we will call advanceLsn() after we retrieved target LSN, so that debezium + // would not drop any record. + // However, that might cause unexpected state and cause failure in the test. Thus we need to bypass + // some check if they are on legacy postgres + // versions. + private boolean isOnLegacyPostgres() { + return postgresImage.majorVersion < 15; + } + @Test void testDebugMode() { final JsonNode invalidDebugConfig = testdb.testConfigBuilder() .withSchemas(modelsSchema(), modelsSchema() + "_random") .withoutSsl() - .withCdcReplication("While reading Data") + .withCdcReplication("While reading Data", RESYNC_DATA_OPTION) .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) .with("debug_mode", true) .build(); @@ -199,7 +214,12 @@ private void assertStateTypes(final List stateMessages, fin if (Objects.isNull(sharedState)) { sharedState = global.getSharedState(); } else { - assertEquals(sharedState, global.getSharedState()); + // This validation is only true for versions on or after postgres 15. We execute + // EPHEMERAL_HEARTBEAT_CREATE_STATEMENTS for earlier versions of + // Postgres. See https://github.com/airbytehq/airbyte/pull/33605 for details. + if (!isOnLegacyPostgres()) { + assertEquals(sharedState, global.getSharedState()); + } } assertEquals(1, global.getStreamStates().size()); final AirbyteStreamState streamState = global.getStreamStates().get(0); @@ -327,7 +347,11 @@ public void testTwoStreamSync() throws Exception { if (Objects.isNull(sharedState)) { sharedState = global.getSharedState(); } else { - assertEquals(sharedState, global.getSharedState()); + // LSN will be advanced for postgres version before 15. See + // https://github.com/airbytehq/airbyte/pull/33605 + if (!isOnLegacyPostgres()) { + assertEquals(sharedState, global.getSharedState()); + } } if (Objects.isNull(firstStreamInState)) { @@ -582,6 +606,47 @@ private void createAndPopulateTimestampTable() { } } + @Test + void testSyncShouldFailPurgedLogs() throws Exception { + final int recordsToCreate = 20; + + final JsonNode config = testdb.testConfigBuilder() + .withSchemas(modelsSchema(), modelsSchema() + "_random") + .withoutSsl() + .withCdcReplication("While reading Data", FAIL_SYNC_OPTION) + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); + final AutoCloseableIterator firstBatchIterator = source() + .read(config, getConfiguredCatalog(), null); + final List dataFromFirstBatch = AutoCloseableIterators + .toListAndClose(firstBatchIterator); + final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); + assertExpectedStateMessages(stateAfterFirstBatch); + // second batch of records again 20 being created + bulkInsertRecords(recordsToCreate); + + // Extract the last state message + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); + final AutoCloseableIterator secondBatchIterator = source() + .read(config, getConfiguredCatalog(), state); + final List dataFromSecondBatch = AutoCloseableIterators + .toListAndClose(secondBatchIterator); + final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterSecondBatch); + + for (int recordsCreated = 0; recordsCreated < 1; recordsCreated++) { + final JsonNode record = + Jsons.jsonNode(ImmutableMap + .of(COL_ID, 400 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, + "H-" + recordsCreated)); + writeModelRecord(record); + } + + // Triggering sync with the first sync's state only which would mimic a scenario that the second + // sync failed on destination end, and we didn't save state + assertThrows(ConfigErrorException.class, () -> source().read(config, getConfiguredCatalog(), state)); + } + @Test protected void syncShouldHandlePurgedLogsGracefully() throws Exception { @@ -758,7 +823,11 @@ protected void assertLsnPositionForSyncShouldIncrementLSN(final Long lsnPosition if (syncNumber == 1) { assertEquals(1, lsnPosition2.compareTo(lsnPosition1)); } else if (syncNumber == 2) { - assertEquals(0, lsnPosition2.compareTo(lsnPosition1)); + // Earlier Postgres version will advance lsn even if there is no sync records. See + // https://github.com/airbytehq/airbyte/pull/33605. + if (!isOnLegacyPostgres()) { + assertEquals(0, lsnPosition2.compareTo(lsnPosition1)); + } } else { throw new RuntimeException("Unknown sync number " + syncNumber); } @@ -794,7 +863,9 @@ protected void verifyCheckpointStatesByRecords() throws Exception { .toListAndClose(secondBatchIterator); assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); final List stateMessagesCDC = extractStateMessages(dataFromSecondBatch); - assertTrue(stateMessagesCDC.size() > 1, "Generated only the final state."); + if (!isOnLegacyPostgres()) { + assertTrue(stateMessagesCDC.size() > 1, "Generated only the final state."); + } assertEquals(stateMessagesCDC.size(), stateMessagesCDC.stream().distinct().count(), "There are duplicated states."); } @@ -833,7 +904,9 @@ protected void verifyCheckpointStatesBySeconds() throws Exception { assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); final List stateMessagesCDC = extractStateMessages(dataFromSecondBatch); - assertTrue(stateMessagesCDC.size() > 1, "Generated only the final state."); + if (!isOnLegacyPostgres()) { + assertTrue(stateMessagesCDC.size() > 1, "Generated only the final state."); + } assertEquals(stateMessagesCDC.size(), stateMessagesCDC.stream().distinct().count(), "There are duplicated states."); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java index b43fd3ee0d8e..8a0b4f7059c5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java @@ -14,7 +14,6 @@ import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; -import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlagsWrapper; @@ -67,10 +66,7 @@ private Source source() { PostgresSource source = new PostgresSource(); source.setFeatureFlags( FeatureFlagsWrapper.overridingDeploymentMode( - FeatureFlagsWrapper.overridingUseStreamCapableState( - new EnvVariableFeatureFlags(), - true), - AdaptiveSourceRunner.CLOUD_MODE)); + new EnvVariableFeatureFlags(), AdaptiveSourceRunner.CLOUD_MODE)); return PostgresSource.sshWrappedSource(source); } @@ -126,8 +122,8 @@ private PostgresTestDatabase.PostgresConfigBuilder configBuilderWithSSLMode( final String sslMode, final boolean innerAddress) { final var containerAddress = innerAddress - ? SshHelpers.getInnerContainerAddress(db.getContainer()) - : SshHelpers.getOuterContainerAddress(db.getContainer()); + ? SshBastionContainer.getInnerContainerAddress(db.getContainer()) + : SshBastionContainer.getOuterContainerAddress(db.getContainer()); return db.configBuilder() .with(JdbcUtils.HOST_KEY, Objects.requireNonNull(containerAddress.left)) .with(JdbcUtils.PORT_KEY, containerAddress.right) diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java index beb0f207087a..b1cb46096aa8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventConverter.CDC_LSN; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebeziumStateUtilTest.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java rename to airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebeziumStateUtilTest.java index 280d0ac2709e..f46eec5bc2ee 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebeziumStateUtilTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.cdk.integrations.debezium.internals; +package io.airbyte.integrations.source.postgres; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -12,11 +12,11 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.integrations.source.postgres.cdc.PostgresDebeziumStateUtil; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.connector.postgresql.connection.Lsn; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java index e2dcd3ab84fe..0865c043a6cb 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java @@ -5,15 +5,12 @@ package io.airbyte.integrations.source.postgres; import io.airbyte.cdk.integrations.debug.DebugUtil; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; public class PostgresDebugger { @SuppressWarnings({"unchecked", "deprecation", "resource"}) public static void main(final String[] args) throws Exception { final PostgresSource postgresSource = new PostgresSource(); - postgresSource.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); DebugUtil.debug(postgresSource); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index e169f3430d1b..705cf416fdc4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -20,8 +20,6 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; @@ -83,9 +81,7 @@ protected JsonNode config() { @Override protected PostgresSource source() { - final var source = new PostgresSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new PostgresSource(); } @Override @@ -367,11 +363,6 @@ void incrementalTimestampCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected boolean supportsPerStream() { - return true; - } - /** * Postgres Source Error Codes: *

      @@ -447,7 +438,8 @@ public void testUserHasNoPermissionToDataBase() throws Exception { } @Test - void testReadMultipleTablesIncrementally() throws Exception { + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { final var config = config(); ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1); final String namespace = getDefaultNamespace(); @@ -692,7 +684,7 @@ protected List getExpectedAirbyteMessagesSecondSync(final String .withCursor("5") .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2)); return expectedMessages; } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 9215ba5e792f..bfb5bad07639 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -27,8 +27,6 @@ import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -155,9 +153,7 @@ void tearDown() { } public PostgresSource source() { - final var source = new PostgresSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); - return source; + return new PostgresSource(); } private static DSLContext getDslContextWithSpecifiedUser(final JsonNode config, final String username, final String password) { @@ -250,20 +246,19 @@ void testUserDoesntHasPrivilegesToSelectTable() throws Exception { return null; }); final JsonNode config = getConfig(); - try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_3", "132")) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); - ctx.fetch("CREATE VIEW id_and_name_3_view(id, name) as\n" - + "SELECT id_and_name_3.id,\n" - + " id_and_name_3.name\n" - + "FROM id_and_name_3;\n" - + "ALTER TABLE id_and_name_3_view\n" - + " owner TO test_user_3"); - ctx.fetch("INSERT INTO id_and_name_3 (id, name) VALUES (1,'Zed'), (2, 'Jack'), (3, 'Antuan');"); - return null; - }); - } + final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_3", "132"); + final Database database = new Database(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); + ctx.fetch("CREATE VIEW id_and_name_3_view(id, name) as\n" + + "SELECT id_and_name_3.id,\n" + + " id_and_name_3.name\n" + + "FROM id_and_name_3;\n" + + "ALTER TABLE id_and_name_3_view\n" + + " owner TO test_user_3"); + ctx.fetch("INSERT INTO id_and_name_3 (id, name) VALUES (1,'Zed'), (2, 'Jack'), (3, 'Antuan');"); + return null; + }); final JsonNode anotherUserConfig = getConfig("test_user_3", "132"); final Set actualMessages = MoreIterators.toSet(source().read(anotherUserConfig, CONFIGURED_CATALOG, null)); @@ -313,13 +308,12 @@ void testDiscoverRecursiveRolePermissions() throws Exception { }); final var config = getConfig(); - try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_4", "132")) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); - return null; - }); - } + final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_4", "132"); + final Database database = new Database(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); + return null; + }); AirbyteCatalog actual = source().discover(getConfig("test_user_4", "132")); Set tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); assertEquals(Sets.newHashSet("id_and_name", "id_and_name_7", "id_and_name_3"), tableNames); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java index 02a48bcb2792..7bf7f586918d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java @@ -18,8 +18,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.base.Source; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; @@ -125,7 +123,6 @@ protected JsonNode getXminConfig() { protected Source source() { PostgresSource source = new PostgresSource(); - source.setFeatureFlags(FeatureFlagsWrapper.overridingUseStreamCapableState(new EnvVariableFeatureFlags(), true)); return PostgresSource.sshWrappedSource(source); } diff --git a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java index b92c319d9eec..625af2d4aa1a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java +++ b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java @@ -12,19 +12,14 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; -public class PostgresContainerFactory implements ContainerFactory> { +public class PostgresContainerFactory extends ContainerFactory> { @Override - public PostgreSQLContainer createNewContainer(DockerImageName imageName) { + protected PostgreSQLContainer createNewContainer(DockerImageName imageName) { return new PostgreSQLContainer<>(imageName.asCompatibleSubstituteFor("postgres")); } - @Override - public Class getContainerClass() { - return PostgreSQLContainer.class; - } - /** * Apply the postgresql.conf file that we've packaged as a resource. */ diff --git a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java index 155b649e96a8..07146c33264e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java +++ b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.INVALID_CDC_CURSOR_POSITION_PROPERTY; +import static io.airbyte.integrations.source.postgres.PostgresSpecConstants.RESYNC_DATA_OPTION; + import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcUtils; @@ -21,15 +24,17 @@ public class PostgresTestDatabase extends public static enum BaseImage { - POSTGRES_16("postgres:16-bullseye"), - POSTGRES_12("postgres:12-bullseye"), - POSTGRES_9("postgres:9-alpine"), - POSTGRES_SSL_DEV("marcosmarxm/postgres-ssl:dev"); + POSTGRES_16("postgres:16-bullseye", 16), + POSTGRES_12("postgres:12-bullseye", 12), + POSTGRES_9("postgres:9-alpine", 9), + POSTGRES_SSL_DEV("marcosmarxm/postgres-ssl:dev", 16); - private final String reference; + public final String reference; + public final int majorVersion; - private BaseImage(String reference) { + private BaseImage(String reference, int majorVersion) { this.reference = reference; + this.majorVersion = majorVersion; }; } @@ -172,10 +177,10 @@ public PostgresConfigBuilder withStandardReplication() { } public PostgresConfigBuilder withCdcReplication() { - return withCdcReplication("While reading Data"); + return withCdcReplication("While reading Data", RESYNC_DATA_OPTION); } - public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) { + public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour, String cdcCursorFailBehaviour) { return this .with("is_test", true) .with("replication_method", Jsons.jsonNode(ImmutableMap.builder() @@ -184,6 +189,7 @@ public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) { .put("publication", testDatabase.getPublicationName()) .put("initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds()) .put("lsn_commit_behaviour", LsnCommitBehaviour) + .put(INVALID_CDC_CURSOR_POSITION_PROPERTY, cdcCursorFailBehaviour) .build())); } diff --git a/airbyte-integrations/connectors/source-posthog/main.py b/airbyte-integrations/connectors/source-posthog/main.py index c788205d69b6..f7e69357d999 100644 --- a/airbyte-integrations/connectors/source-posthog/main.py +++ b/airbyte-integrations/connectors/source-posthog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_posthog import SourcePosthog +from source_posthog.run import run if __name__ == "__main__": - source = SourcePosthog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-posthog/metadata.yaml b/airbyte-integrations/connectors/source-posthog/metadata.yaml index 7351a4d8180b..cfd5c0c74779 100644 --- a/airbyte-integrations/connectors/source-posthog/metadata.yaml +++ b/airbyte-integrations/connectors/source-posthog/metadata.yaml @@ -16,6 +16,10 @@ data: icon: posthog.svg license: MIT name: PostHog + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-posthog registries: cloud: enabled: true @@ -29,7 +33,7 @@ data: message: The `event` field in the `events` stream has been corrected to the proper data type. To apply this change, refresh the schema for the `events` stream and reset your data. For more information [visit](https://docs.airbyte.com/integrations/sources/posthog-migrations) - upgradeDeadline: "2023-12-20" + upgradeDeadline: "2024-01-15" tags: - language:low-code - language:python diff --git a/airbyte-integrations/connectors/source-posthog/setup.py b/airbyte-integrations/connectors/source-posthog/setup.py index 75fb93d9ac2c..5ebf94b4b079 100644 --- a/airbyte-integrations/connectors/source-posthog/setup.py +++ b/airbyte-integrations/connectors/source-posthog/setup.py @@ -14,12 +14,29 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-posthog=source_posthog.run:run", + ], + }, name="source_posthog", description="Source implementation for Posthog.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={"tests": TEST_REQUIREMENTS}, ) diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/run.py b/airbyte-integrations/connectors/source-posthog/source_posthog/run.py new file mode 100644 index 000000000000..711b2e4ae483 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_posthog import SourcePosthog + + +def run(): + source = SourcePosthog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-postmarkapp/main.py b/airbyte-integrations/connectors/source-postmarkapp/main.py index 8257a82e80c1..c6b256d74b3a 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/main.py +++ b/airbyte-integrations/connectors/source-postmarkapp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_postmarkapp import SourcePostmarkapp +from source_postmarkapp.run import run if __name__ == "__main__": - source = SourcePostmarkapp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml index bac99f12668c..58849623dd1c 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml @@ -8,6 +8,10 @@ data: icon: postmark.svg license: MIT name: Postmark App + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-postmarkapp registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-postmarkapp/setup.py b/airbyte-integrations/connectors/source-postmarkapp/setup.py index 08a989882942..3c5e3ab4e671 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/setup.py +++ b/airbyte-integrations/connectors/source-postmarkapp/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-postmarkapp=source_postmarkapp.run:run", + ], + }, name="source_postmarkapp", description="Source implementation for Postmarkapp.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py new file mode 100644 index 000000000000..b3308b251000 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_postmarkapp import SourcePostmarkapp + + +def run(): + source = SourcePostmarkapp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-prestashop/main.py b/airbyte-integrations/connectors/source-prestashop/main.py index 813f7411045f..e51c47a996aa 100644 --- a/airbyte-integrations/connectors/source-prestashop/main.py +++ b/airbyte-integrations/connectors/source-prestashop/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_prestashop import SourcePrestashop +from source_prestashop.run import run if __name__ == "__main__": - source = SourcePrestashop() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-prestashop/metadata.yaml b/airbyte-integrations/connectors/source-prestashop/metadata.yaml index 184347008685..dbbf6240c187 100644 --- a/airbyte-integrations/connectors/source-prestashop/metadata.yaml +++ b/airbyte-integrations/connectors/source-prestashop/metadata.yaml @@ -15,6 +15,10 @@ data: icon: prestashop.svg license: MIT name: PrestaShop + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-prestashop registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-prestashop/setup.py b/airbyte-integrations/connectors/source-prestashop/setup.py index 4762aa8a2eaf..e883c046c0af 100644 --- a/airbyte-integrations/connectors/source-prestashop/setup.py +++ b/airbyte-integrations/connectors/source-prestashop/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-prestashop=source_prestashop.run:run", + ], + }, name="source_prestashop", description="Source implementation for PrestaShop.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py new file mode 100644 index 000000000000..23f77c77bf55 --- /dev/null +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_prestashop import SourcePrestashop + + +def run(): + source = SourcePrestashop() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-primetric/main.py b/airbyte-integrations/connectors/source-primetric/main.py index 3cbd6368b2cf..3cb9977197cc 100644 --- a/airbyte-integrations/connectors/source-primetric/main.py +++ b/airbyte-integrations/connectors/source-primetric/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_primetric import SourcePrimetric +from source_primetric.run import run if __name__ == "__main__": - source = SourcePrimetric() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-primetric/metadata.yaml b/airbyte-integrations/connectors/source-primetric/metadata.yaml index 38b37950abad..83bd3b3ecda9 100644 --- a/airbyte-integrations/connectors/source-primetric/metadata.yaml +++ b/airbyte-integrations/connectors/source-primetric/metadata.yaml @@ -8,6 +8,10 @@ data: icon: primetric.svg license: MIT name: Primetric + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-primetric registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-primetric/setup.py b/airbyte-integrations/connectors/source-primetric/setup.py index 00a6f81dc6b3..d73e7ee77688 100644 --- a/airbyte-integrations/connectors/source-primetric/setup.py +++ b/airbyte-integrations/connectors/source-primetric/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-primetric=source_primetric.run:run", + ], + }, name="source_primetric", description="Source implementation for Primetric.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-primetric/source_primetric/run.py b/airbyte-integrations/connectors/source-primetric/source_primetric/run.py new file mode 100644 index 000000000000..729752166581 --- /dev/null +++ b/airbyte-integrations/connectors/source-primetric/source_primetric/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_primetric import SourcePrimetric + + +def run(): + source = SourcePrimetric() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-public-apis/main.py b/airbyte-integrations/connectors/source-public-apis/main.py index c5ad4ac06bd9..c9796a4aa4ef 100644 --- a/airbyte-integrations/connectors/source-public-apis/main.py +++ b/airbyte-integrations/connectors/source-public-apis/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_public_apis import SourcePublicApis +from source_public_apis.run import run if __name__ == "__main__": - source = SourcePublicApis() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-public-apis/metadata.yaml b/airbyte-integrations/connectors/source-public-apis/metadata.yaml index 0df0a0c7da8e..fe8481eaf1f1 100644 --- a/airbyte-integrations/connectors/source-public-apis/metadata.yaml +++ b/airbyte-integrations/connectors/source-public-apis/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-public-apis registries: cloud: enabled: false @@ -23,5 +27,5 @@ data: releaseStage: alpha supportLevel: community tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-public-apis/setup.py b/airbyte-integrations/connectors/source-public-apis/setup.py index 8e3f218e28f7..d454fb726a25 100644 --- a/airbyte-integrations/connectors/source-public-apis/setup.py +++ b/airbyte-integrations/connectors/source-public-apis/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-public-apis=source_public_apis.run:run", + ], + }, name="source_public_apis", description="Source implementation for Public Apis.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py new file mode 100644 index 000000000000..b4927fb5a5e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_public_apis import SourcePublicApis + + +def run(): + source = SourcePublicApis() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-punk-api/main.py b/airbyte-integrations/connectors/source-punk-api/main.py index c16f1cc798dc..bd3661bdabfc 100644 --- a/airbyte-integrations/connectors/source-punk-api/main.py +++ b/airbyte-integrations/connectors/source-punk-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_punk_api import SourcePunkApi +from source_punk_api.run import run if __name__ == "__main__": - source = SourcePunkApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-punk-api/metadata.yaml b/airbyte-integrations/connectors/source-punk-api/metadata.yaml index 942b617b200f..80d6d03f5a1a 100644 --- a/airbyte-integrations/connectors/source-punk-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-punk-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: punkapi.svg license: MIT name: Punk API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-punk-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-punk-api/setup.py b/airbyte-integrations/connectors/source-punk-api/setup.py index 6e0119b0fd17..834098b436fe 100644 --- a/airbyte-integrations/connectors/source-punk-api/setup.py +++ b/airbyte-integrations/connectors/source-punk-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-punk-api=source_punk_api.run:run", + ], + }, name="source_punk_api", description="Source implementation for Punk Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py b/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py new file mode 100644 index 000000000000..f1a31bfed3f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_punk_api import SourcePunkApi + + +def run(): + source = SourcePunkApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pypi/main.py b/airbyte-integrations/connectors/source-pypi/main.py index a61df9711f41..44f5f367986f 100644 --- a/airbyte-integrations/connectors/source-pypi/main.py +++ b/airbyte-integrations/connectors/source-pypi/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pypi import SourcePypi +from source_pypi.run import run if __name__ == "__main__": - source = SourcePypi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pypi/metadata.yaml b/airbyte-integrations/connectors/source-pypi/metadata.yaml index 90cd84ad6509..978205dca0d5 100644 --- a/airbyte-integrations/connectors/source-pypi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pypi/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pypi.svg license: MIT name: PyPI + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pypi registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pypi/setup.py b/airbyte-integrations/connectors/source-pypi/setup.py index ae9a89a654a5..b1198fecaead 100644 --- a/airbyte-integrations/connectors/source-pypi/setup.py +++ b/airbyte-integrations/connectors/source-pypi/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pypi=source_pypi.run:run", + ], + }, name="source_pypi", description="Source implementation for Pypi.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pypi/source_pypi/run.py b/airbyte-integrations/connectors/source-pypi/source_pypi/run.py new file mode 100644 index 000000000000..5fc71c441a59 --- /dev/null +++ b/airbyte-integrations/connectors/source-pypi/source_pypi/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pypi import SourcePypi + + +def run(): + source = SourcePypi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/main.py b/airbyte-integrations/connectors/source-python-http-tutorial/main.py index e13b685ed7d4..57dce4e0679c 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/main.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_python_http_tutorial import SourcePythonHttpTutorial +from source_python_http_tutorial.run import run if __name__ == "__main__": - source = SourcePythonHttpTutorial() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py index 473e22de73aa..35164f2108aa 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py @@ -5,11 +5,28 @@ from setuptools import find_packages, setup setup( + entry_points={ + "console_scripts": [ + "source-python-http-tutorial=source_python_http_tutorial.run:run", + ], + }, name="source_python_http_tutorial", description="Source implementation for Python Http Tutorial.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=["airbyte-cdk", "pytest"], - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, ) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py new file mode 100644 index 000000000000..60690aa68eff --- /dev/null +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_python_http_tutorial import SourcePythonHttpTutorial + + +def run(): + source = SourcePythonHttpTutorial() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-qonto/main.py b/airbyte-integrations/connectors/source-qonto/main.py index 749ae981779f..eb2e90e366c1 100644 --- a/airbyte-integrations/connectors/source-qonto/main.py +++ b/airbyte-integrations/connectors/source-qonto/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_qonto import SourceQonto +from source_qonto.run import run if __name__ == "__main__": - source = SourceQonto() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-qonto/metadata.yaml b/airbyte-integrations/connectors/source-qonto/metadata.yaml index 83239b6c7d13..05c0393aa6ec 100644 --- a/airbyte-integrations/connectors/source-qonto/metadata.yaml +++ b/airbyte-integrations/connectors/source-qonto/metadata.yaml @@ -13,5 +13,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/qonto tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qonto/setup.py b/airbyte-integrations/connectors/source-qonto/setup.py index 8847f9443761..ba340d0b242a 100644 --- a/airbyte-integrations/connectors/source-qonto/setup.py +++ b/airbyte-integrations/connectors/source-qonto/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-qonto=source_qonto.run:run", + ], + }, name="source_qonto", description="Source implementation for Qonto.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/run.py b/airbyte-integrations/connectors/source-qonto/source_qonto/run.py new file mode 100644 index 000000000000..0ea8c8470c1f --- /dev/null +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_qonto import SourceQonto + + +def run(): + source = SourceQonto() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-qualaroo/main.py b/airbyte-integrations/connectors/source-qualaroo/main.py index 149d1ab5a20f..7a7ed24a96bf 100644 --- a/airbyte-integrations/connectors/source-qualaroo/main.py +++ b/airbyte-integrations/connectors/source-qualaroo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_qualaroo import SourceQualaroo +from source_qualaroo.run import run if __name__ == "__main__": - source = SourceQualaroo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml index 7364d8abe5c3..f33f9ef7bdec 100644 --- a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-qualaroo registries: oss: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index 840126cac8fb..24bf8b993372 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-qualaroo=source_qualaroo.run:run", + ], + }, name="source_qualaroo", description="Source implementation for Qualaroo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py new file mode 100644 index 000000000000..c6b4c65009e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_qualaroo import SourceQualaroo + + +def run(): + source = SourceQualaroo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-quickbooks/Dockerfile b/airbyte-integrations/connectors/source-quickbooks/Dockerfile deleted file mode 100644 index 11a9b982877c..000000000000 --- a/airbyte-integrations/connectors/source-quickbooks/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_quickbooks ./source_quickbooks - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.0.0 -LABEL io.airbyte.name=airbyte/source-quickbooks diff --git a/airbyte-integrations/connectors/source-quickbooks/acceptance-test-config.yml b/airbyte-integrations/connectors/source-quickbooks/acceptance-test-config.yml index cee0f008c31e..d5f58965d1bf 100644 --- a/airbyte-integrations/connectors/source-quickbooks/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-quickbooks/acceptance-test-config.yml @@ -1,7 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-quickbooks:dev -test_strictness_level: "high" +test_strictness_level: "low" acceptance_tests: spec: tests: @@ -35,11 +35,11 @@ acceptance_tests: bypass_reason: "unable to populate" - name: departments bypass_reason: "unable to populate" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-quickbooks/main.py b/airbyte-integrations/connectors/source-quickbooks/main.py index c475c3cf97fd..abeed13585f5 100644 --- a/airbyte-integrations/connectors/source-quickbooks/main.py +++ b/airbyte-integrations/connectors/source-quickbooks/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_quickbooks import SourceQuickbooks +from source_quickbooks.run import run if __name__ == "__main__": - source = SourceQuickbooks() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml index 27c2f6d7cab4..ec82e2ce8d74 100644 --- a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml @@ -5,14 +5,20 @@ data: - quickbooks.api.intuit.com - oauth.platform.intuit.com connectorSubtype: api + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorType: source definitionId: cf9c4355-b171-4477-8f2d-6c5cc5fc8b7e - dockerImageTag: 3.0.0 + dockerImageTag: 3.0.2 dockerRepository: airbyte/source-quickbooks githubIssueLabel: source-quickbooks icon: quickbooks.svg license: MIT name: QuickBooks + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-quickbooks registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-quickbooks/poetry.lock b/airbyte-integrations/connectors/source-quickbooks/poetry.lock new file mode 100644 index 000000000000..980d09b8161f --- /dev/null +++ b/airbyte-integrations/connectors/source-quickbooks/poetry.lock @@ -0,0 +1,1252 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "vcrpy" +version = "4.1.1" +description = "Automatically mock your HTTP interactions to simplify and speed up testing" +optional = false +python-versions = ">=3.5" +files = [ + {file = "vcrpy-4.1.1-py2.py3-none-any.whl", hash = "sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162"}, + {file = "vcrpy-4.1.1.tar.gz", hash = "sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599"}, +] + +[package.dependencies] +PyYAML = "*" +six = ">=1.5" +wrapt = "*" +yarl = {version = "*", markers = "python_version >= \"3.6\""} + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "71dc812ad49d88776001ad46700cb74ed9a5f22ae1f82e76755f93d8a315b221" diff --git a/airbyte-integrations/connectors/source-quickbooks/pyproject.toml b/airbyte-integrations/connectors/source-quickbooks/pyproject.toml new file mode 100644 index 000000000000..33b9c403e84e --- /dev/null +++ b/airbyte-integrations/connectors/source-quickbooks/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.0.2" +name = "source-quickbooks" +description = "Source implementation for quickbooks." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/quickbooks" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_quickbooks" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.63.2" +vcrpy = "==4.1.1" +urllib3 = "==1.26.18" + +[tool.poetry.scripts] +source-quickbooks = "source_quickbooks.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" + diff --git a/airbyte-integrations/connectors/source-quickbooks/setup.py b/airbyte-integrations/connectors/source-quickbooks/setup.py deleted file mode 100644 index 025726239f79..000000000000 --- a/airbyte-integrations/connectors/source-quickbooks/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.44.0", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.2", - "pytest-mock~=3.6.1", -] - -setup( - name="source_quickbooks", - description="Source implementation for Quickbooks.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml index ac70de7bbc6a..267f04d857d0 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml @@ -26,7 +26,6 @@ definitions: client_id: "{{ config['credentials']['client_id'] }}" client_secret: "{{ config['credentials']['client_secret'] }}" refresh_token: "{{ config['credentials']['refresh_token'] }}" - refresh_token_updater: {} retriever: type: SimpleRetriever record_selector: diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py new file mode 100644 index 000000000000..4e5d18504e8d --- /dev/null +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_quickbooks import SourceQuickbooks + + +def run(): + source = SourceQuickbooks() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json index 91170b2b5fe8..a1fb3ccc9a0c 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "ClassRef": { "properties": { @@ -102,6 +103,11 @@ }, "DetailType": { "type": ["null", "string"] + }, + "SubTotalLineDetail": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json index 9d2eb205046b..12e9946f5105 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "BillEmail": { "properties": { @@ -190,6 +191,11 @@ "DetailType": { "type": ["null", "string"] }, + "SubTotalLineDetail": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, "Description": { "type": ["null", "string"] }, @@ -227,7 +233,8 @@ "type": ["null", "object"] } }, - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true }, "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json index 9185d986b532..656b0c5e3209 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json @@ -80,9 +80,13 @@ "properties": { "value": { "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] } }, - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true }, "AllowIPNPayment": { "type": ["null", "boolean"] @@ -157,6 +161,11 @@ "Line": { "items": { "properties": { + "SubTotalLineDetail": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, "LinkedTxn": { "items": { "properties": { @@ -254,7 +263,8 @@ "type": ["null", "string"] } }, - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true }, "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json index e7271b4955c4..2f333006aa61 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json @@ -38,6 +38,11 @@ "Line": { "items": { "properties": { + "SubTotalLineDetail": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, "Id": { "type": ["null", "string"] }, @@ -84,7 +89,8 @@ "type": ["null", "object"] } }, - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true }, "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json index b9ec030fbd03..e48e7e0a34f4 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json @@ -116,7 +116,13 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { + "SubTotalLineDetail": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, "DiscountLineDetail": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json index ead3ac5eddd6..bfb6a3669cf2 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json @@ -46,7 +46,7 @@ "airbyte_secret": true }, "access_token": { - "description": "Access token fot making authenticated requests.", + "description": "Access token for making authenticated requests.", "title": "Access Token", "type": "string", "airbyte_secret": true diff --git a/airbyte-integrations/connectors/source-railz/main.py b/airbyte-integrations/connectors/source-railz/main.py index 03bea5dbba8a..bfa6b9fadb2f 100644 --- a/airbyte-integrations/connectors/source-railz/main.py +++ b/airbyte-integrations/connectors/source-railz/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_railz import SourceRailz +from source_railz.run import run if __name__ == "__main__": - source = SourceRailz() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-railz/metadata.yaml b/airbyte-integrations/connectors/source-railz/metadata.yaml index ea4aa848d729..eb0870a1779a 100644 --- a/airbyte-integrations/connectors/source-railz/metadata.yaml +++ b/airbyte-integrations/connectors/source-railz/metadata.yaml @@ -8,6 +8,10 @@ data: icon: railz.svg license: MIT name: Railz + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-railz registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-railz/setup.py b/airbyte-integrations/connectors/source-railz/setup.py index 3b938dc45ff5..37478efb8ddc 100644 --- a/airbyte-integrations/connectors/source-railz/setup.py +++ b/airbyte-integrations/connectors/source-railz/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-railz=source_railz.run:run", + ], + }, name="source_railz", description="Source implementation for Railz.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-railz/source_railz/run.py b/airbyte-integrations/connectors/source-railz/source_railz/run.py new file mode 100644 index 000000000000..831b665a05da --- /dev/null +++ b/airbyte-integrations/connectors/source-railz/source_railz/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_railz import SourceRailz + + +def run(): + source = SourceRailz() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/main.py b/airbyte-integrations/connectors/source-rd-station-marketing/main.py index b5e333d8a220..5210a9fdc2b3 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/main.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rd_station_marketing import SourceRDStationMarketing +from source_rd_station_marketing.run import run if __name__ == "__main__": - source = SourceRDStationMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml index 42135ace43a0..dbfc299e01bb 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rdstation.svg license: MIT name: RD Station Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rd-station-marketing registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py index fc5481e12eb5..039dd82b7d54 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rd-station-marketing=source_rd_station_marketing.run:run", + ], + }, name="source_rd_station_marketing", description="Source implementation for RD Station Marketing.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py new file mode 100644 index 000000000000..c684a12ed29f --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rd_station_marketing import SourceRDStationMarketing + + +def run(): + source = SourceRDStationMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recharge/README.md b/airbyte-integrations/connectors/source-recharge/README.md index 53853dd9e62c..03ee5b5b276e 100644 --- a/airbyte-integrations/connectors/source-recharge/README.md +++ b/airbyte-integrations/connectors/source-recharge/README.md @@ -1,67 +1,55 @@ -# Recharge Source +# Recharge source connector + This is the repository for the Recharge source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/recharge). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/recharge). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recharge) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recharge/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/recharge) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recharge/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source recharge test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-recharge spec +poetry run source-recharge check --config secrets/config.json +poetry run source-recharge discover --config secrets/config.json +poetry run source-recharge read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-recharge build ``` -An image will be built with the tag `airbyte/source-recharge:dev`. +An image will be available on your host with the tag `airbyte/source-recharge:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-recharge:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-recharge:dev spec @@ -70,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recharge:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-recharge:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-recharge test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recharge test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recharge.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/recharge.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 028b6f90e143..085c9b7cfc55 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -9,10 +9,6 @@ acceptance_tests: bypass_reason: "volatile data" - name: onetimes bypass_reason: "no data from stream" - - name: orders - bypass_reason: "no data from stream" - - name: subscriptions - bypass_reason: "no data from stream" ignored_fields: shop: - name: shop/updated_at @@ -26,6 +22,27 @@ acceptance_tests: exact_order: no extra_records: yes fail_on_extra_columns: false + - config_path: secrets/config_order_modern_api.json + empty_streams: + - name: collections + bypass_reason: "volatile data" + - name: discounts + bypass_reason: "volatile data" + - name: onetimes + bypass_reason: "no data from stream" + ignored_fields: + shop: + - name: shop/updated_at + bypass_reason: "updated after login" + - name: store/updated_at + bypass_reason: "updated after login" + timeout_seconds: 7200 + expect_records: + path: "integration_tests/expected_records_orders_modern_api.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false connection: tests: - config_path: secrets/config.json @@ -35,7 +52,7 @@ acceptance_tests: discovery: tests: - backward_compatibility_tests_config: - disable_for_version: 0.2.10 + disable_for_version: 1.1.2 config_path: secrets/config.json full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl index 9b1b0db102b2..491a25b9836a 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl @@ -1,12 +1,14 @@ -{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1699016394454} -{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1699016395217} -{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1699016397112} -{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1699016397881} -{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1699016397882} -{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1699016402746} -{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1699016403662} -{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1699016408078} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1699016425107} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1699016425108} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1699016425109} -{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1699016427703} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1706644129288} +{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1706644130026} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1706644132446} +{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1706644133275} +{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644133278} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1706644139386} +{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1706644140190} +{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1706644151126} +{"stream": "orders", "data": {"address_id": 69282975, "address_is_active": 1, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "browser_ip": null, "charge_id": 580825303, "charge_status": "SUCCESS", "created_at": "2023-05-13T00:07:28", "currency": "USD", "customer": {"accepts_marketing": true, "email": "kozakevich_k@example.com", "first_name": "Kelly", "last_name": "Kozakevich", "phone": null, "send_email_welcome": false, "verified_email": true}, "customer_id": 64962974, "discount_codes": null, "email": "kozakevich_k@example.com", "error": null, "first_name": "Kelly", "hash": "f99bd4a6877257af", "id": 534919106, "is_prepaid": 0, "last_name": "Kozakevich", "line_items": [{"external_inventory_policy": "decrement_obeying_policy", "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": 24.3, "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684723835051", "sku": "T3", "subscription_id": 153601366, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "variant_title": "L / City Green"}], "note": null, "note_attributes": [], "payment_processor": "shopify_payments", "processed_at": "2023-05-13T00:07:33", "scheduled_at": "2023-05-13T00:00:00", "shipped_date": "2023-05-13T00:07:33", "shipping_address": {"address1": "1921 W Wilson St", "address2": "", "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_date": "2023-05-13T00:00:00", "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "title": "Economy"}], "shopify_cart_token": null, "shopify_customer_id": "5213433266347", "shopify_id": "5006149877931", "shopify_order_id": "5006149877931", "shopify_order_number": 1016, "status": "SUCCESS", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "total_discounts": 0.0, "total_duties": "0.0", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": "43114102955", "type": "RECURRING", "updated_at": "2023-05-13T00:16:51"}, "emitted_at": 1706644162075} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1706644170248} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1706644170251} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1706644170252} +{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1706644179022} +{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1706644181724} diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl new file mode 100644 index 000000000000..9962d32adf33 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl @@ -0,0 +1,14 @@ +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1706644270838} +{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1706644271610} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1706644274123} +{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1706644274939} +{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644274942} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1706644280530} +{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1706644281267} +{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1706644292270} +{"stream": "orders", "data": {"id": 534919106, "address_id": 69282975, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "charge": {"id": 580825303, "external_transaction_id": {"payment_processor": "43114102955"}, "payment_processor_name": "shopify_payments", "status": "success"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:28+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "external_cart_token": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_order_name": {"ecommerce": "#1016"}, "external_order_number": {"ecommerce": "1016"}, "is_prepaid": 0, "line_items": [{"purchase_item_id": 153601366, "external_inventory_policy": "decrement_obeying_policy", "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "processed_at": "2023-05-13T04:07:33+00:00", "scheduled_at": "2023-05-13T04:00:00+00:00", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "taxable": false, "total_discounts": 0.0, "total_duties": "0.00", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644303256} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1706644311039} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1706644311045} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1706644311046} +{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1706644319680} +{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1706644322400} diff --git a/airbyte-integrations/connectors/source-recharge/main.py b/airbyte-integrations/connectors/source-recharge/main.py index c61ef445b68d..d8ccf40b711e 100644 --- a/airbyte-integrations/connectors/source-recharge/main.py +++ b/airbyte-integrations/connectors/source-recharge/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recharge import SourceRecharge +from source_recharge.run import run if __name__ == "__main__": - source = SourceRecharge() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index af32cc639991..31dbd5c73fc4 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -7,12 +7,16 @@ data: connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 1.1.2 + dockerImageTag: 1.1.5 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg license: MIT name: Recharge + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recharge registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock new file mode 100644 index 000000000000..4135398b392f --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.60.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.60.1.tar.gz", hash = "sha256:fc5212b2962c1dc6aca9cc6f1c2000d7636b7509915846c126420c2b0c814317"}, + {file = "airbyte_cdk-0.60.1-py3-none-any.whl", hash = "sha256:94b33c0f6851d1e2546eac3cec54c67489239595d9e0a496ef57c3fc808e89e3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "da1dbc89f0a40d0a16baa47814c1e57b38c5afec44baa89789bc069fe9a7a7af" diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml new file mode 100644 index 000000000000..15ca7a7471c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.1.5" +name = "source-recharge" +description = "Source implementation for Recharge." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/recharge" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_recharge" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.60.1" + +[tool.poetry.scripts] +source-recharge = "source_recharge.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-recharge/requirements.txt b/airbyte-integrations/connectors/source-recharge/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-recharge/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-recharge/setup.py b/airbyte-integrations/connectors/source-recharge/setup.py deleted file mode 100644 index bb091a439b28..000000000000 --- a/airbyte-integrations/connectors/source-recharge/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] - -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "requests-mock", -] - -setup( - name="source_recharge", - description="Source implementation for Recharge.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 126aa6ebfad8..aaf2cb91cf2e 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -3,7 +3,7 @@ # from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum import requests @@ -212,12 +212,23 @@ class Onetimes(RechargeStreamModernAPI, IncrementalRechargeStream): """ -class Orders(RechargeStreamDeprecatedAPI, IncrementalRechargeStream): +class OrdersDeprecatedApi(RechargeStreamDeprecatedAPI, IncrementalRechargeStream): """ Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders Using old API version to avoid schema changes and loosing email, first_name, last_name columns, because in new version it not present """ + name = "orders" + + +class OrdersModernApi(RechargeStreamModernAPI, IncrementalRechargeStream): + """ + Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders + Using newer API version to fetch all the data, based on the Customer's UI toggle `use_deprecated_api: FALSE`. + """ + + name = "orders" + class Products(RechargeStreamDeprecatedAPI): """ diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/run.py b/airbyte-integrations/connectors/source-recharge/source_recharge/run.py new file mode 100644 index 000000000000..2a56566c41c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recharge import SourceRecharge + + +def run(): + source = SourceRecharge() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json index 26546ab2f554..0112494af378 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json @@ -75,6 +75,96 @@ "browser_ip": { "type": ["null", "string"] }, + "charge": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "external_transaction_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "payment_processor": { + "type": ["null", "string"] + } + } + }, + "payment_processor_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "client_details": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "browser_ip": { + "type": ["null", "string"] + }, + "user_agent": { + "type": ["null", "string"] + } + } + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "error": { + "type": ["null", "string"] + }, + "external_cart_token": { + "type": ["null", "string"] + }, + "external_order_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "external_order_name": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "external_order_number": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "order_attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "taxable": { + "type": ["null", "boolean"] + }, + "total_duties": { + "type": ["null", "string"] + }, + "total_weight_grams": { + "type": ["null", "integer"] + }, "charge_id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index 8f926baad949..1d1ea875f3e3 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -3,7 +3,7 @@ # -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Tuple, Union from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode @@ -11,7 +11,20 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .api import Addresses, Charges, Collections, Customers, Discounts, Metafields, Onetimes, Orders, Products, Shop, Subscriptions +from .api import ( + Addresses, + Charges, + Collections, + Customers, + Discounts, + Metafields, + Onetimes, + OrdersDeprecatedApi, + OrdersModernApi, + Products, + Shop, + Subscriptions, +) class RechargeTokenAuthenticator(TokenAuthenticator): @@ -30,6 +43,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> except Exception as error: return False, f"Unable to connect to Recharge API with the provided credentials - {repr(error)}" + def select_orders_stream(self, config: Mapping[str, Any], **kwargs) -> Union[OrdersDeprecatedApi, OrdersModernApi]: + if config.get("use_orders_deprecated_api"): + return OrdersDeprecatedApi(config, **kwargs) + else: + return OrdersModernApi(config, **kwargs) + def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = RechargeTokenAuthenticator(token=config["access_token"]) return [ @@ -40,7 +59,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Discounts(config, authenticator=auth), Metafields(config, authenticator=auth), Onetimes(config, authenticator=auth), - Orders(config, authenticator=auth), + # select the Orders stream class, based on the UI toggle "Use `Orders` Deprecated API" + self.select_orders_stream(config, authenticator=auth), Products(config, authenticator=auth), Shop(config, authenticator=auth), Subscriptions(config, authenticator=auth), diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/spec.json b/airbyte-integrations/connectors/source-recharge/source_recharge/spec.json index 5a147b0d71da..54f1ff8eac75 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/spec.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/spec.json @@ -20,6 +20,12 @@ "title": "Access Token", "description": "The value of the Access Token generated. See the docs for more information.", "airbyte_secret": true + }, + "use_orders_deprecated_api": { + "type": "boolean", + "title": "Use `Orders` Deprecated API", + "description": "Define whether or not the `Orders` stream should use the deprecated `2021-01` API version, or use `2021-11`, otherwise.", + "default": true } } } diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py index 19ac0ebf284a..3981d725c047 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py @@ -15,7 +15,8 @@ Discounts, Metafields, Onetimes, - Orders, + OrdersDeprecatedApi, + OrdersModernApi, Products, RechargeStreamDeprecatedAPI, RechargeStreamModernAPI, @@ -45,7 +46,8 @@ class TestCommon: (Discounts, "id"), (Metafields, "id"), (Onetimes, "id"), - (Orders, "id"), + (OrdersDeprecatedApi, "id"), + (OrdersModernApi, "id"), (Products, "id"), (Shop, ["shop", "store"]), (Subscriptions, "id"), @@ -64,7 +66,8 @@ def test_primary_key(self, stream_cls, expected): (Discounts, "incremental", "discounts"), (Metafields, "full-refresh", "metafields"), (Onetimes, "incremental", "onetimes"), - (Orders, "incremental", "orders"), + (OrdersDeprecatedApi, "incremental", "orders"), + (OrdersModernApi, "incremental", "orders"), (Products, "full-refresh", "products"), (Shop, "full-refresh", None), (Subscriptions, "incremental", "subscriptions"), @@ -87,7 +90,8 @@ def test_data_path(self, config, stream_cls, stream_type, expected): (Discounts, "incremental", "discounts"), (Metafields, "full-refresh", "metafields"), (Onetimes, "incremental", "onetimes"), - (Orders, "incremental", "orders"), + (OrdersDeprecatedApi, "incremental", "orders"), + (OrdersModernApi, "incremental", "orders"), (Products, "full-refresh", "products"), (Shop, "full-refresh", "shop"), (Subscriptions, "incremental", "subscriptions"), @@ -134,9 +138,10 @@ def generate_records(self, stream_name, count): [ (Collections, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), (Metafields, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (OrdersModernApi, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), (Products, {}, {"page": 2}), (Shop, {}, None), - (Orders, {}, {"page": 2}), + (OrdersDeprecatedApi, {}, {"page": 2}), ], ) def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): @@ -230,7 +235,8 @@ def generate_records(self, stream_name, count): (Customers, "updated_at"), (Discounts, "updated_at"), (Onetimes, "updated_at"), - (Orders, "updated_at"), + (OrdersDeprecatedApi, "updated_at"), + (OrdersModernApi, "updated_at"), (Subscriptions, "updated_at"), ], ) @@ -247,7 +253,8 @@ def test_cursor_field(self, config, stream_cls, expected): (Customers, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), (Discounts, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), (Onetimes, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Orders, {}, {"page": 2}), + (OrdersDeprecatedApi, {}, {"page": 2}), + (OrdersModernApi, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), (Subscriptions, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), ], ) @@ -299,7 +306,14 @@ def test_next_page_token(self, config, stream_cls, cursor_response, requests_moc {"limit": 250, "cursor": "123"}, ), ( - Orders, + OrdersDeprecatedApi, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + ( + OrdersModernApi, None, {}, {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, @@ -327,7 +341,8 @@ def test_request_params(self, config, stream_cls, next_page_token, stream_state, (Customers, {"updated_at": 3}, {"updated_at": 4}, {"updated_at": 4}), (Discounts, {}, {"updated_at": 2}, {"updated_at": 2}), (Onetimes, {}, {"updated_at": 2}, {"updated_at": 2}), - (Orders, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), + (OrdersDeprecatedApi, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), + (OrdersModernApi, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), (Subscriptions, {"updated_at": 6}, {"updated_at": 7}, {"updated_at": 7}), ], ) @@ -335,3 +350,15 @@ def test_get_updated_state(self, config, stream_cls, current_state, latest_recor stream = stream_cls(config, authenticator=None) result = stream.get_updated_state(current_state, latest_record) assert result == expected + + + @pytest.mark.parametrize( + "stream_cls, expected", + [ + (Addresses, {'start_date': '2021-08-15 00:00:01', 'end_date': '2021-09-14 00:00:01'}), + ], + ) + def test_stream_slices(self, config, stream_cls, expected): + stream = stream_cls(config, authenticator=None) + result = list(stream.stream_slices(sync_mode=None, cursor_field=stream.cursor_field, stream_state=None)) + assert result[0] == expected diff --git a/airbyte-integrations/connectors/source-recreation/main.py b/airbyte-integrations/connectors/source-recreation/main.py index f126b44607d6..6847e93aab46 100644 --- a/airbyte-integrations/connectors/source-recreation/main.py +++ b/airbyte-integrations/connectors/source-recreation/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recreation import SourceRecreation +from source_recreation.run import run if __name__ == "__main__": - source = SourceRecreation() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recreation/metadata.yaml b/airbyte-integrations/connectors/source-recreation/metadata.yaml index b217b6b65c04..821554d94e1d 100644 --- a/airbyte-integrations/connectors/source-recreation/metadata.yaml +++ b/airbyte-integrations/connectors/source-recreation/metadata.yaml @@ -8,6 +8,10 @@ data: icon: recreation.svg license: MIT name: Recreation + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recreation registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recreation/setup.py b/airbyte-integrations/connectors/source-recreation/setup.py index 26e4b5657f8a..9f41baeed4b1 100644 --- a/airbyte-integrations/connectors/source-recreation/setup.py +++ b/airbyte-integrations/connectors/source-recreation/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recreation=source_recreation.run:run", + ], + }, name="source_recreation", description="Source implementation for Recreation.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-recreation/source_recreation/run.py b/airbyte-integrations/connectors/source-recreation/source_recreation/run.py new file mode 100644 index 000000000000..2f8b496ad695 --- /dev/null +++ b/airbyte-integrations/connectors/source-recreation/source_recreation/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recreation import SourceRecreation + + +def run(): + source = SourceRecreation() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recruitee/main.py b/airbyte-integrations/connectors/source-recruitee/main.py index 8d918eec10e2..26802f658cda 100644 --- a/airbyte-integrations/connectors/source-recruitee/main.py +++ b/airbyte-integrations/connectors/source-recruitee/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recruitee import SourceRecruitee +from source_recruitee.run import run if __name__ == "__main__": - source = SourceRecruitee() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recruitee/metadata.yaml b/airbyte-integrations/connectors/source-recruitee/metadata.yaml index 42407ff998bf..71061b162cbd 100644 --- a/airbyte-integrations/connectors/source-recruitee/metadata.yaml +++ b/airbyte-integrations/connectors/source-recruitee/metadata.yaml @@ -8,6 +8,10 @@ data: icon: recruitee.svg license: MIT name: Recruitee + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recruitee registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recruitee/setup.py b/airbyte-integrations/connectors/source-recruitee/setup.py index d2b12eff3c68..c03d7631bd28 100644 --- a/airbyte-integrations/connectors/source-recruitee/setup.py +++ b/airbyte-integrations/connectors/source-recruitee/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recruitee=source_recruitee.run:run", + ], + }, name="source_recruitee", description="Source implementation for Recruitee.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py b/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py new file mode 100644 index 000000000000..3d4111901af2 --- /dev/null +++ b/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recruitee import SourceRecruitee + + +def run(): + source = SourceRecruitee() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recurly/Dockerfile b/airbyte-integrations/connectors/source-recurly/Dockerfile deleted file mode 100644 index 51646e6746f5..000000000000 --- a/airbyte-integrations/connectors/source-recurly/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_recurly ./source_recurly - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.4.1 -LABEL io.airbyte.name=airbyte/source-recurly diff --git a/airbyte-integrations/connectors/source-recurly/README.md b/airbyte-integrations/connectors/source-recurly/README.md index 0b6cedea3f2e..936201b1a143 100644 --- a/airbyte-integrations/connectors/source-recurly/README.md +++ b/airbyte-integrations/connectors/source-recurly/README.md @@ -1,97 +1,104 @@ -# Amazon Seller-Partner Source +# Recurly source connector -This is the repository for the Amazon Seller-Partner source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/amazon-seller-partner). +This is the repository for the Recurly source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/recurly). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector + +From this connector directory, run: + +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. +### Creating credentials -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-seller-partner) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_seller-partner/integration_tests/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recurly) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recurly/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source amazon-seller-partner test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source recurly test creds` and place them into `secrets/config.json`. ### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json + +```bash +poetry run source-recurly spec +poetry run source-recurly check --config secrets/config.json +poetry run source-recurly discover --config secrets/config.json +poetry run source-recurly read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image +### Running unit tests +To run unit tests locally, from the connector directory run: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name source-amazon-seller-partner build +poetry run pytest unit_tests ``` -An image will be built with the tag `airbyte/source-amazon-seller-partner:dev`. +### Building the docker image + +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: -**Via `docker build`:** ```bash -docker build -t airbyte/source-amazon-seller-partner:dev . +airbyte-ci connectors --name=source-recurly build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-recurly:dev`. + +### Running the docker container + Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-amazon-seller-partner:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-seller-partner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json + +```bash +docker run --rm airbyte/source-recurly:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recurly:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recurly:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-recurly:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite + You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-recurly test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +Customize the `acceptance-test-config.yml` file to configure acceptance tests. See our [Connector Acceptance Tests reference](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires you to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management + +All of your dependencies should be managed via Poetry. To add a new dependency, run: + +```bash +poetry add +``` + +Please commit the changes to the `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector -### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recurly test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version listed as `dockerImageTag` in `metadata.yaml`. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recurly.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. diff --git a/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml index 863cbc656d27..c1e9dfe6ee73 100644 --- a/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml @@ -1,35 +1,40 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-recurly:dev -tests: +acceptance_tests: spec: - - spec_path: "source_recurly/spec.json" + tests: + - spec_path: "source_recurly/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - [ - "account_coupon_redemptions", - "account_notes", - "add_ons", - "billing_infos", - "credit_payments", - "line_items", - "shipping_methods", - "unique_coupons", - "export_dates", - ] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "add_ons" + bypass_reason: "Cannot seed this stream with free sandbox account" + - name: "billing_infos" + bypass_reason: "Cannot seed this stream with free sandbox account" + - name: "credit_payments" + bypass_reason: "Cannot seed this stream with free sandbox account" + - name: "shipping_methods" + bypass_reason: "Cannot seed this stream with free sandbox account" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/future_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/future_state.json" + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json b/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json index 0cc9399e6fae..1c9442f7c6d3 100644 --- a/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json @@ -1,50 +1,114 @@ -{ - "accounts": { - "updated_at": "2022-07-19T22:21:37Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "accounts" } + } }, - "account_coupon_redemptions": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "account_coupon_redemptions" } + } }, - "account_notes": { - "created_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "account_notes" } + } }, - "add_ons": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "add_ons" } + } }, - "billing_infos": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "billing_infos" } + } }, - "coupons": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "coupons" } + } }, - "credit_payments": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "credit_payments" } + } }, - "invoices": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "invoices" } + } }, - "line_items": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "line_items" } + } }, - "measured_units": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "measured_units" } + } }, - "plans": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "plans" } + } }, - "shipping_addresses": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "shipping_addresses" } + } }, - "shipping_methods": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "shipping_methods" } + } }, - "subscriptions": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "subscriptions" } + } }, - "transactions": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "transactions" } + } }, - "unique_coupons": { - "updated_at": "2022-07-19T22:21:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2036-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "unique_coupons" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-recurly/main.py b/airbyte-integrations/connectors/source-recurly/main.py index 251a30f0dffe..ba5c26176fde 100644 --- a/airbyte-integrations/connectors/source-recurly/main.py +++ b/airbyte-integrations/connectors/source-recurly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recurly import SourceRecurly +from source_recurly.run import run if __name__ == "__main__": - source = SourceRecurly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recurly/metadata.yaml b/airbyte-integrations/connectors/source-recurly/metadata.yaml index d10aee87e084..0a8afd004e8d 100644 --- a/airbyte-integrations/connectors/source-recurly/metadata.yaml +++ b/airbyte-integrations/connectors/source-recurly/metadata.yaml @@ -2,10 +2,12 @@ data: ab_internal: ql: 200 sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 - dockerImageTag: 0.4.1 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-recurly documentationUrl: https://docs.airbyte.com/integrations/sources/recurly githubIssueLabel: source-recurly @@ -18,6 +20,10 @@ data: oss: enabled: true releaseStage: alpha + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recurly supportLevel: community tags: - language:python diff --git a/airbyte-integrations/connectors/source-recurly/poetry.lock b/airbyte-integrations/connectors/source-recurly/poetry.lock new file mode 100644 index 000000000000..144b7f527169 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/poetry.lock @@ -0,0 +1,1045 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.67.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.67.0.tar.gz", hash = "sha256:cbbff1b3895c89313764a721870bb293a396c74bad8dd6e5c36a0c3b0a2f6a10"}, + {file = "airbyte_cdk-0.67.0-py3-none-any.whl", hash = "sha256:2082c859536a2450c03b89dba1bbdab21bad314fbf5ef6d2e86fefc4ba935373"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "recurly" +version = "4.10.0" +description = "Recurly v4" +optional = false +python-versions = "*" +files = [ + {file = "recurly-4.10.0-py3-none-any.whl", hash = "sha256:b8e3b1ec58f7b1e1b91286f2db864f6ba4053837ad920d0c2868508020442aaf"}, + {file = "recurly-4.10.0.tar.gz", hash = "sha256:a8dddab76bb38f76a715644448f45499227bfd00529ef33f7945b3bcc5a8f3a2"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "d85022c0f25e080bc6664f5dd4f866b1ef1dbe94bf07b5ad77a8eb4d45987798" diff --git a/airbyte-integrations/connectors/source-recurly/pyproject.toml b/airbyte-integrations/connectors/source-recurly/pyproject.toml new file mode 100644 index 000000000000..7e1569523ee2 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.5.0" +name = "source-recurly" +description = "Source implementation for Recurly." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/recurly" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_recurly" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0.67.0" +recurly = "==4.10.0" + +[tool.poetry.scripts] +source-recurly = "source_recurly.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-recurly/requirements.txt b/airbyte-integrations/connectors/source-recurly/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-recurly/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py deleted file mode 100644 index 1d278d40ec9f..000000000000 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "recurly==4.10.0", "requests"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] - -setup( - name="source_recurly", - description="Source implementation for Recurly.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/run.py b/airbyte-integrations/connectors/source-recurly/source_recurly/run.py new file mode 100644 index 000000000000..746b6556605c --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recurly import SourceRecurly + + +def run(): + source = SourceRecurly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_coupon_redemptions.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_coupon_redemptions.json index f49ddee10ee6..d9e894d584d3 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_coupon_redemptions.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_coupon_redemptions.json @@ -6,35 +6,18 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "account": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"], - "maxLength": 13 - }, - "code": { - "type": ["null", "string"], - "maxLength": 50 - } - } + "$ref": "account_details.json" }, "subscription_id": { "type": ["null", "string"], "maxLength": 13 }, "coupon": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"], - "maxLength": 13 - }, - "code": { - "type": ["null", "string"], - "maxLength": 256 - } - } + "$ref": "coupons.json" }, "state": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_notes.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_notes.json index 830233d4d39e..ee68e82c8973 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_notes.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/account_notes.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -7,19 +7,15 @@ "maxLength": 13, "readOnly": true }, + "object": { + "type": ["null", "string"] + }, "account_id": { "type": "string", "maxLength": 13 }, "user": { - "type": "object", - "properties": { - "id": { - "type": "string", - "readOnly": true, - "maxLength": 13 - } - } + "$ref": "users.json" }, "message": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/accounts.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/accounts.json index 013509268cf4..c9f1c5b84953 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/accounts.json @@ -6,6 +6,12 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, + "hosted_login_token": { + "type": ["null", "string"] + }, "code": { "type": ["null", "string"], "maxLength": 256 @@ -101,11 +107,18 @@ "title": "Country", "description": "Country, 2-letter ISO 3166-1 alpha-2 code.", "maxLength": 2 + }, + "geo_code": { + "type": ["null", "string"] } } }, "custom_fields": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } }, "has_live_subscription": { "type": ["null", "boolean"] @@ -140,6 +153,30 @@ "deleted_at": { "type": ["null", "string"], "format": "date-time" + }, + "billing_info": { + "$ref": "billing_infos.json" + }, + "external_accounts": { + "type": ["null", "array"], + "items": { + "$ref": "external_accounts.json" + } + }, + "invoice_template_id": { + "type": ["null", "string"] + }, + "override_business_entity_id": { + "type": ["null", "string"] + }, + "preferred_time_zone": { + "type": ["null", "string"] + }, + "shipping_addresses": { + "type": ["null", "array"], + "items": { + "$ref": "shipping_addresses.json" + } } } } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/billing_infos.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/billing_infos.json index f2cdc6f0c8bd..5dd179d84f39 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/billing_infos.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/billing_infos.json @@ -1,196 +1,3 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "id": { - "type": "string", - "maxLength": 13, - "readOnly": true - }, - "account_id": { - "type": "string", - "maxLength": 13, - "readOnly": true - }, - "first_name": { - "type": "string", - "maxLength": 50 - }, - "last_name": { - "type": "string", - "maxLength": 50 - }, - "company": { - "type": "string", - "maxLength": 100 - }, - "address": { - "type": "object", - "properties": { - "phone": { - "type": "string", - "title": "Phone number", - "maxLength": 256 - }, - "street1": { - "type": "string", - "title": "Street 1", - "maxLength": 256 - }, - "street2": { - "type": "string", - "title": "Street 2", - "maxLength": 256 - }, - "city": { - "type": "string", - "title": "City", - "maxLength": 256 - }, - "region": { - "type": "string", - "title": "State/Province", - "description": "State or province.", - "maxLength": 256 - }, - "postal_code": { - "type": "string", - "title": "Zip/Postal code", - "description": "Zip or postal code.", - "maxLength": 256 - }, - "country": { - "type": "string", - "title": "Country", - "description": "Country, 2-letter ISO 3166-1 alpha-2 code.", - "maxLength": 2 - } - } - }, - "vat_number": { - "type": "string", - "description": "Customer's VAT number (to avoid having the VAT applied). This is only used for automatically collected invoices.", - "maxLength": 20 - }, - "valid": { - "type": "boolean", - "readOnly": true - }, - "payment_method": { - "type": "object", - "properties": { - "card_type": { - "description": "Visa, MasterCard, American Express, Discover, JCB, etc.", - "type": "string", - "maxLength": 256 - }, - "first_six": { - "type": "string", - "description": "Credit card number's first six digits.", - "maxLength": 6 - }, - "last_four": { - "type": "string", - "description": "Credit card number's last four digits. Will refer to bank account if payment method is ACH.", - "maxLength": 4 - }, - "last_two": { - "type": "string", - "description": "The IBAN bank account's last two digits.", - "maxLength": 2 - }, - "exp_month": { - "type": "integer", - "description": "Expiration month.", - "maxLength": 2 - }, - "exp_year": { - "type": "integer", - "description": "Expiration year.", - "maxLength": 4 - }, - "gateway_token": { - "type": "string", - "description": "A token used in place of a credit card in order to perform transactions.", - "maxLength": 50 - }, - "cc_bin_country": { - "type": "string", - "description": "The 2-letter ISO 3166-1 alpha-2 country code associated with the credit card BIN, if known by Recurly. Available on the BillingInfo object only. Available when the BIN country lookup feature is enabled.", - "maxLength": 256 - }, - "gateway_code": { - "type": "string", - "description": "An identifier for a specific payment gateway.", - "maxLength": 13 - }, - "billing_agreement_id": { - "type": "string", - "description": "Billing Agreement identifier. Only present for Amazon or Paypal payment methods.", - "maxLength": 256 - }, - "name_on_account": { - "type": "string", - "description": "The name associated with the bank account.", - "maxLength": 256 - }, - "account_type": { - "description": "The bank account type. Only present for ACH payment methods.", - "type": "string", - "maxLength": 256 - }, - "routing_number": { - "type": "string", - "description": "The bank account's routing number. Only present for ACH payment methods.", - "maxLength": 256 - }, - "routing_number_bank": { - "type": "string", - "description": "The bank name of this routing number.", - "maxLength": 256 - } - } - }, - "fraud": { - "type": "object", - "title": "Fraud information", - "description": "Most recent fraud result.", - "readOnly": true, - "properties": { - "score": { - "type": "integer", - "title": "Kount score" - }, - "decision": { - "title": "Kount decision", - "maxLength": 10, - "type": "string" - }, - "risk_rules_triggered": { - "type": "object", - "title": "Kount rules" - } - } - }, - "primary_payment_method": { - "type": "boolean", - "description": "The `primary_payment_method` field is used to indicate the primary billing info on the account. The first billing info created on an account will always become primary. This payment method will be used" - }, - "backup_payment_method": { - "type": "boolean", - "description": "The `backup_payment_method` field is used to indicate a billing info as a backup on the account that will be tried if the initial billing info used for an invoice is declined." - }, - "created_at": { - "type": "string", - "format": "date-time", - "description": "When the billing information was created.", - "readOnly": true - }, - "updated_at": { - "type": "string", - "format": "date-time", - "description": "When the billing information was last changed.", - "readOnly": true - } - } + "$ref": "billing_infos.json" } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/coupons.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/coupons.json index 9fc95eaf9416..9c262d6773c3 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/coupons.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/coupons.json @@ -1,188 +1,3 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"], - "maxLength": 13 - }, - "code": { - "type": ["null", "string"], - "maxLength": 256 - }, - "name": { - "type": ["null", "string"], - "maxLength": 256 - }, - "state": { - "type": ["null", "string"], - "maxLength": 256 - }, - "max_redemptions": { - "type": ["null", "number"] - }, - "max_redemptions_per_account": { - "type": ["null", "number"] - }, - "unique_coupon_codes_count": { - "type": ["null", "number"] - }, - "unique_code_template": { - "type": ["null", "string"], - "maxLength": 256 - }, - "duration": { - "type": ["null", "string"], - "maxLength": 256 - }, - "temporal_amount": { - "type": ["null", "number"] - }, - "temporal_unit": { - "type": ["null", "string"], - "maxLength": 256 - }, - "free_trial_unit": { - "type": ["null", "string"], - "maxLength": 256 - }, - "free_trial_amount": { - "type": ["null", "number"] - }, - "applies_to_all_plans": { - "type": ["null", "boolean"] - }, - "applies_to_all_items": { - "type": ["null", "boolean"] - }, - "applies_to_non_plan_charges": { - "type": ["null", "boolean"] - }, - "plans": { - "type": ["null", "array"], - "title": "Plans", - "description": "A list of plans for which this coupon applies. This will be `null` if `applies_to_all_plans=true`.", - "items": { - "type": "object", - "title": "Plan mini details", - "description": "Just the important parts.", - "properties": { - "id": { - "type": "string", - "title": "Plan ID", - "maxLength": 13, - "readOnly": true - }, - "code": { - "type": "string", - "title": "Plan code", - "description": "Unique code to identify the plan. This is used in Hosted Payment Page URLs and in the invoice exports.", - "maxLength": 13 - } - } - } - }, - "items": { - "type": ["null", "array"], - "title": "Items", - "description": "A list of items for which this coupon applies. This will be\n`null` if `applies_to_all_items=true`.\n", - "items": { - "type": ["null", "object"], - "title": "Item mini details", - "description": "Just the important parts.", - "properties": { - "id": { - "type": "string", - "title": "Item ID", - "maxLength": 13, - "readOnly": true - } - } - } - }, - "redemption_resource": { - "type": ["null", "string"], - "maxLength": 256 - }, - "discount": { - "type": ["null", "object"], - "description": "Details of the discount a coupon applies. Will contain a `type`\nproperty and one of the following properties: `percent`, `fixed`, `trial`.\n", - "properties": { - "type": { - "type": "string", - "maxLength": 256 - }, - "percent": { - "description": "This is only present when `type=percent`.", - "type": "integer" - }, - "currencies": { - "type": "array", - "description": "This is only present when `type=fixed`.", - "items": { - "type": ["null", "object"], - "properties": { - "currency": { - "type": "string", - "title": "Currency", - "description": "3-letter ISO 4217 currency code.", - "maxLength": 3 - }, - "amount": { - "type": "number", - "format": "float", - "title": "Discount Amount", - "description": "Value of the fixed discount that this coupon applies." - } - } - } - }, - "trial": { - "type": "object", - "description": "This is only present when `type=free_trial`.", - "properties": { - "unit": { - "title": "Trial unit", - "description": "Temporal unit of the free trial", - "type": "string", - "maxLength": 256 - }, - "length": { - "type": "integer", - "title": "Trial length", - "description": "Trial length measured in the units specified by the sibling `unit` property" - } - } - } - } - }, - "coupon_type": { - "type": ["null", "string"], - "maxLength": 256 - }, - "hosted_page_description": { - "type": ["null", "string"], - "maxLength": 1024 - }, - "invoice_description": { - "type": ["null", "string"], - "maxLength": 1024 - }, - "redeem_by": { - "type": ["null", "string"], - "maxLength": 256 - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "expired_at": { - "type": ["null", "string"], - "format": "date-time" - } - } + "$ref": "coupons.json" } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json index 64db09629b59..6e5f4732e079 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/invoices.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -8,6 +8,12 @@ "readOnly": true, "maxLength": 13 }, + "uuid": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, "type": { "title": "Invoice type", "description": "Invoices are either charge, credit, or legacy invoices.", @@ -26,20 +32,7 @@ "maxLength": 256 }, "account": { - "type": "object", - "title": "Account mini details", - "properties": { - "id": { - "type": ["null", "string"], - "maxLength": 13, - "readOnly": true - }, - "code": { - "type": ["null", "string"], - "description": "The unique identifier of the account.", - "maxLength": 50 - } - } + "$ref": "account_details.json" }, "billing_info_id": { "type": ["null", "string"], @@ -264,6 +257,9 @@ } } }, + "used_tax_service": { + "type": ["null", "boolean"] + }, "vat_number": { "type": ["null", "string"], "title": "VAT number", @@ -292,23 +288,12 @@ "type": ["null", "array"], "title": "Line Items", "items": { - "type": ["null", "object"], - "title": "Line item", - "properties": { - "id": { - "type": "string", - "title": "Line item ID", - "maxLength": 13 - }, - "uuid": { - "type": "string", - "title": "UUID", - "description": "The UUID is useful for matching data with the CSV exports and building URLs into Recurly's UI.", - "maxLength": 32 - } - } + "$ref": "line_items.json" } }, + "has_more_line_items": { + "type": ["null", "boolean"] + }, "transactions": { "type": ["null", "array"], "title": "Transactions", @@ -378,6 +363,15 @@ "title": "Dunning Campaign ID", "description": "Unique ID to identify the dunning campaign used when dunning the invoice. Available when the Dunning Campaigns feature is enabled. For sites without multiple dunning campaigns enabled, this will always be the default dunning campaign.", "maxLength": 256 + }, + "dunning_events_sent": { + "type": ["null", "integer"] + }, + "final_dunning_event": { + "type": ["null", "boolean"] + }, + "business_entity_id": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/line_items.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/line_items.json index c981890d97ab..85370cc75850 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/line_items.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/line_items.json @@ -1,270 +1,3 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "title": "Line item", - "properties": { - "id": { - "type": "string", - "title": "Line item ID", - "maxLength": 13 - }, - "uuid": { - "type": "string", - "title": "UUID", - "description": "The UUID is useful for matching data with the CSV exports and building URLs into Recurly's UI.", - "maxLength": 32 - }, - "type": { - "type": "string", - "title": "Line item type", - "description": "Charges are positive line items that debit the account. Credits are negative line items that credit the account.", - "maxLength": 256 - }, - "item_code": { - "type": ["null", "string"], - "title": "Item Code", - "description": "Unique code to identify an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", - "maxLength": 50 - }, - "item_id": { - "type": ["null", "string"], - "title": "Item ID", - "description": "System-generated unique identifier for an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", - "maxLength": 13 - }, - "external_sku": { - "type": ["null", "string"], - "title": "External SKU", - "description": "Optional Stock Keeping Unit assigned to an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", - "maxLength": 50 - }, - "revenue_schedule_type": { - "type": ["null", "string"], - "title": "Revenue schedule type", - "maxLength": 256 - }, - "state": { - "type": "string", - "title": "Current state of the line item", - "description": "Pending line items are charges or credits on an account that have not been applied to an invoice yet. Invoiced line items will always have an `invoice_id` value.", - "maxLength": 256 - }, - "legacy_category": { - "type": ["null", "string"], - "title": "Legacy category", - "description": "Category to describe the role of a line item on a legacy invoice:\n- \"charges\" refers to charges being billed for on this invoice.\n- \"credits\" refers to refund or proration credits. This portion of the invoice can be considered a credit memo.\n- \"applied_credits\" refers to previous credits applied to this invoice. See their original_line_item_id to determine where the credit first originated.\n- \"carryforwards\" can be ignored. They exist to consume any remaining credit balance. A new credit with the same amount will be created and placed back on the account.\n" - }, - "bill_for_account_id": { - "type": "string", - "title": "Bill For Account ID", - "maxLength": 13, - "description": "The UUID of the account responsible for originating the line item." - }, - "subscription_id": { - "type": ["null", "string"], - "title": "Subscription ID", - "description": "If the line item is a charge or credit for a subscription, this is its ID.", - "maxLength": 13 - }, - "plan_id": { - "type": ["null", "string"], - "title": "Plan ID", - "description": "If the line item is a charge or credit for a plan or add-on, this is the plan's ID.", - "maxLength": 13 - }, - "plan_code": { - "type": ["null", "string"], - "title": "Plan code", - "description": "If the line item is a charge or credit for a plan or add-on, this is the plan's code.", - "maxLength": 50 - }, - "add_on_id": { - "type": ["null", "string"], - "title": "Add-on ID", - "description": "If the line item is a charge or credit for an add-on this is its ID.", - "maxLength": 13 - }, - "add_on_code": { - "type": ["null", "string"], - "title": "Add-on code", - "description": "If the line item is a charge or credit for an add-on, this is its code.", - "maxLength": 50 - }, - "invoice_id": { - "type": ["null", "string"], - "title": "Invoice ID", - "description": "Once the line item has been invoiced this will be the invoice's ID.", - "maxLength": 13 - }, - "invoice_number": { - "type": ["null", "string"], - "title": "Invoice number", - "description": "Once the line item has been invoiced this will be the invoice's number. If VAT taxation and the Country Invoice Sequencing feature are enabled, invoices will have country-specific invoice numbers for invoices billed to EU countries (ex: FR1001). Non-EU invoices will continue to use the site-level invoice number sequence.", - "maxLength": 256 - }, - "previous_line_item_id": { - "type": ["null", "string"], - "title": "Previous line item ID", - "description": "Will only have a value if the line item is a credit created from a previous credit, or if the credit was created from a charge refund.", - "maxLength": 13 - }, - "original_line_item_invoice_id": { - "type": ["null", "string"], - "title": "Original line item's invoice ID", - "description": "The invoice where the credit originated. Will only have a value if the line item is a credit created from a previous credit, or if the credit was created from a charge refund.", - "maxLength": 13 - }, - "origin": { - "type": "string", - "title": "Origin of line item", - "description": "A credit created from an original charge will have the value of the charge's origin.", - "maxLength": 256 - }, - "accounting_code": { - "type": "string", - "title": "Accounting code", - "description": "Internal accounting code to help you reconcile your revenue to the correct ledger. Line items created as part of a subscription invoice will use the plan or add-on's accounting code, otherwise the value will only be present if you define an accounting code when creating the line item.", - "maxLength": 20 - }, - "product_code": { - "type": "string", - "title": "Product code", - "description": "For plan-related line items this will be the plan's code, for add-on related line items it will be the add-on's code. For item-related line items it will be the item's `external_sku`.", - "maxLength": 50 - }, - "credit_reason_code": { - "type": ["null", "string"], - "title": "Credit reason code", - "description": "The reason the credit was given when line item is `type=credit`.", - "default": "general", - "maxLength": 256 - }, - "currency": { - "type": "string", - "title": "Currency", - "description": "3-letter ISO 4217 currency code.", - "maxLength": 3 - }, - "amount": { - "type": "number", - "format": "float", - "title": "Total after discounts and taxes", - "description": "`(quantity * unit_amount) - (discount + tax)`" - }, - "description": { - "type": "string", - "title": "Description", - "description": "Description that appears on the invoice. For subscription related items this will be filled in automatically.", - "maxLength": 255 - }, - "quantity": { - "type": "integer", - "title": "Quantity", - "description": "This number will be multiplied by the unit amount to compute the subtotal before any discounts or taxes.", - "default": 1 - }, - "unit_amount": { - "type": "number", - "format": "float", - "title": "Unit amount", - "description": "Positive amount for a charge, negative amount for a credit." - }, - "unit_amount_decimal": { - "type": ["null", "string"], - "title": "Unit amount decimal", - "description": "Positive amount for a charge, negative amount for a credit." - }, - "subtotal": { - "type": "number", - "format": "float", - "title": "Total before discounts and taxes", - "description": "`quantity * unit_amount`" - }, - "discount": { - "type": ["null", "number"], - "format": "float", - "title": "Discount", - "description": "The discount applied to the line item." - }, - "tax": { - "type": ["null", "number"], - "format": "float", - "title": "Tax", - "description": "The tax amount for the line item." - }, - "taxable": { - "type": "boolean", - "title": "Taxable?", - "description": "`true` if the line item is taxable, `false` if it is not." - }, - "tax_exempt": { - "type": "boolean", - "title": "Tax exempt?", - "description": "`true` exempts tax on charges, `false` applies tax on charges. If not defined, then defaults to the Plan and Site settings. This attribute does not work for credits (negative line items). Credits are always applied post-tax. Pre-tax discounts should use the Coupons feature." - }, - "tax_code": { - "type": ["null", "string"], - "title": "Tax code", - "description": "Used by Avalara, Vertex, and Recurly\u2019s EU VAT tax feature. The tax code values are specific to each tax system. If you are using Recurly\u2019s EU VAT feature you can use `unknown`, `physical`, or `digital`.", - "maxLength": 50 - }, - "proration_rate": { - "type": ["null", "number"], - "format": "float", - "title": "Proration rate", - "description": "When a line item has been prorated, this is the rate of the proration. Proration rates were made available for line items created after March 30, 2017. For line items created prior to that date, the proration rate will be `null`, even if the line item was prorated.", - "minimum": 0, - "maximum": 1 - }, - "refund": { - "type": "boolean", - "title": "Refund?" - }, - "refunded_quantity": { - "type": ["null", "integer"], - "title": "Refunded Quantity", - "description": "For refund charges, the quantity being refunded. For non-refund charges, the total quantity refunded (possibly over multiple refunds)." - }, - "credit_applied": { - "type": ["null", "number"], - "format": "float", - "title": "Credit Applied", - "description": "The amount of credit from this line item that was applied to the invoice." - }, - "shipping_address": { - "type": ["null", "object"], - "properties": { - "id": { - "type": "string", - "title": "Shipping Address ID", - "maxLength": 13, - "readOnly": true - } - } - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time", - "title": "Start date", - "description": "If an end date is present, this is value indicates the beginning of a billing time range. If no end date is present it indicates billing for a specific date." - }, - "end_date": { - "type": ["null", "string"], - "format": "date-time", - "title": "End date", - "description": "If this date is provided, it indicates the end of a time range." - }, - "created_at": { - "type": "string", - "format": "date-time", - "title": "Created at", - "description": "When the line item was created." - }, - "updated_at": { - "type": "string", - "format": "date-time", - "title": "Last updated at", - "description": "When the line item was last changed." - } - } + "$ref": "line_items.json" } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/measured_units.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/measured_units.json index abc96f8afead..7865d44d3079 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/measured_units.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/measured_units.json @@ -6,6 +6,9 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"], "maxLength": 256 diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/plans.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/plans.json index ed7fa91d786f..aabac321be53 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/plans.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/plans.json @@ -6,6 +6,9 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "code": { "type": ["null", "string"], "maxLength": 256 @@ -45,6 +48,48 @@ "auto_renew": { "type": ["null", "boolean"] }, + "pricing_model": { + "type": ["null", "string"] + }, + "ramp_intervals": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "starting_billing_cycle": { + "type": ["null", "integer"] + }, + "currencies": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "unit_amount": { + "type": ["null", "number"] + } + } + } + } + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, "accounting_code": { "type": ["null", "string"], "maxLength": 256 diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/account_details.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/account_details.json new file mode 100644 index 000000000000..9d3dc5d71945 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/account_details.json @@ -0,0 +1,35 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": "string" + }, + "object": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "parent_account_id": { + "type": ["null", "string"] + }, + "bill_to": { + "type": ["null", "string"] + }, + "dunning_campaign_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/billing_infos.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/billing_infos.json new file mode 100644 index 000000000000..dbf207f589b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/billing_infos.json @@ -0,0 +1,213 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string", + "maxLength": 13, + "readOnly": true + }, + "object": { + "type": ["null", "string"] + }, + "account_id": { + "type": "string", + "maxLength": 13, + "readOnly": true + }, + "first_name": { + "type": ["null", "string"], + "maxLength": 50 + }, + "last_name": { + "type": ["null", "string"], + "maxLength": 50 + }, + "company": { + "type": ["null", "string"], + "maxLength": 100 + }, + "address": { + "type": "object", + "properties": { + "phone": { + "type": ["null", "string"], + "title": "Phone number", + "maxLength": 256 + }, + "street1": { + "type": ["null", "string"], + "title": "Street 1", + "maxLength": 256 + }, + "street2": { + "type": ["null", "string"], + "title": "Street 2", + "maxLength": 256 + }, + "city": { + "type": ["null", "string"], + "title": "City", + "maxLength": 256 + }, + "region": { + "type": ["null", "string"], + "title": "State/Province", + "description": "State or province.", + "maxLength": 256 + }, + "postal_code": { + "type": ["null", "string"], + "title": "Zip/Postal code", + "description": "Zip or postal code.", + "maxLength": 256 + }, + "country": { + "type": ["null", "string"], + "title": "Country", + "description": "Country, 2-letter ISO 3166-1 alpha-2 code.", + "maxLength": 2 + } + } + }, + "vat_number": { + "type": ["null", "string"], + "description": "Customer's VAT number (to avoid having the VAT applied). This is only used for automatically collected invoices.", + "maxLength": 20 + }, + "valid": { + "type": "boolean", + "readOnly": true + }, + "payment_method": { + "type": "object", + "properties": { + "card_type": { + "description": "Visa, MasterCard, American Express, Discover, JCB, etc.", + "type": ["null", "string"], + "maxLength": 256 + }, + "object": { + "type": ["null", "string"] + }, + "first_six": { + "type": ["null", "string"], + "description": "Credit card number's first six digits.", + "maxLength": 6 + }, + "last_four": { + "type": ["null", "string"], + "description": "Credit card number's last four digits. Will refer to bank account if payment method is ACH.", + "maxLength": 4 + }, + "last_two": { + "type": ["null", "string"], + "description": "The IBAN bank account's last two digits.", + "maxLength": 2 + }, + "exp_month": { + "type": ["null", "integer"], + "description": "Expiration month.", + "maxLength": 2 + }, + "exp_year": { + "type": ["null", "integer"], + "description": "Expiration year.", + "maxLength": 4 + }, + "gateway_token": { + "type": ["null", "string"], + "description": "A token used in place of a credit card in order to perform transactions.", + "maxLength": 50 + }, + "cc_bin_country": { + "type": ["null", "string"], + "description": "The 2-letter ISO 3166-1 alpha-2 country code associated with the credit card BIN, if known by Recurly. Available on the BillingInfo object only. Available when the BIN country lookup feature is enabled.", + "maxLength": 256 + }, + "gateway_code": { + "type": ["null", "string"], + "description": "An identifier for a specific payment gateway.", + "maxLength": 13 + }, + "billing_agreement_id": { + "type": ["null", "string"], + "description": "Billing Agreement identifier. Only present for Amazon or Paypal payment methods.", + "maxLength": 256 + }, + "name_on_account": { + "type": ["null", "string"], + "description": "The name associated with the bank account.", + "maxLength": 256 + }, + "account_type": { + "description": "The bank account type. Only present for ACH payment methods.", + "type": ["null", "string"], + "maxLength": 256 + }, + "routing_number": { + "type": ["null", "string"], + "description": "The bank account's routing number. Only present for ACH payment methods.", + "maxLength": 256 + }, + "routing_number_bank": { + "type": ["null", "string"], + "description": "The bank name of this routing number.", + "maxLength": 256 + } + } + }, + "fraud": { + "type": ["null", "object"], + "title": "Fraud information", + "description": "Most recent fraud result.", + "readOnly": true, + "properties": { + "score": { + "type": ["null", "integer"], + "title": "Kount score" + }, + "decision": { + "title": "Kount decision", + "maxLength": 10, + "type": ["null", "string"] + }, + "risk_rules_triggered": { + "type": "object", + "title": "Kount rules" + } + } + }, + "primary_payment_method": { + "type": "boolean", + "description": "The `primary_payment_method` field is used to indicate the primary billing info on the account. The first billing info created on an account will always become primary. This payment method will be used" + }, + "backup_payment_method": { + "type": "boolean", + "description": "The `backup_payment_method` field is used to indicate a billing info as a backup on the account that will be tried if the initial billing info used for an invoice is declined." + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "When the billing information was created.", + "readOnly": true + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "When the billing information was last changed.", + "readOnly": true + }, + "updated_by": { + "type": ["null", "object"], + "properties": { + "ip": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupon_redemptions.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupon_redemptions.json new file mode 100644 index 000000000000..e74f00e93f57 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupon_redemptions.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "discount": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "percent": { + "type": ["null", "integer"] + }, + "currencies": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"] + } + } + } + }, + "trial": { + "type": ["null", "object"], + "properties": { + "unit": { + "type": ["null", "string"] + }, + "length": { + "type": ["null", "integer"] + } + } + } + } + }, + "coupon_type": { + "type": ["null", "string"] + }, + "expired_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "state": { + "type": ["null", "string"] + }, + "discounted": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupons.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupons.json new file mode 100644 index 000000000000..50b91db59b4f --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/coupons.json @@ -0,0 +1,194 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"], + "maxLength": 13 + }, + "object": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"], + "maxLength": 256 + }, + "name": { + "type": ["null", "string"], + "maxLength": 256 + }, + "state": { + "type": ["null", "string"], + "maxLength": 256 + }, + "max_redemptions": { + "type": ["null", "number"] + }, + "max_redemptions_per_account": { + "type": ["null", "number"] + }, + "unique_coupon_codes_count": { + "type": ["null", "number"] + }, + "unique_code_template": { + "type": ["null", "string"], + "maxLength": 256 + }, + "unique_coupon_code": { + "$ref": "unique_coupons.json" + }, + "duration": { + "type": ["null", "string"], + "maxLength": 256 + }, + "temporal_amount": { + "type": ["null", "number"] + }, + "temporal_unit": { + "type": ["null", "string"], + "maxLength": 256 + }, + "free_trial_unit": { + "type": ["null", "string"], + "maxLength": 256 + }, + "free_trial_amount": { + "type": ["null", "number"] + }, + "applies_to_all_plans": { + "type": ["null", "boolean"] + }, + "applies_to_all_items": { + "type": ["null", "boolean"] + }, + "applies_to_non_plan_charges": { + "type": ["null", "boolean"] + }, + "plans": { + "type": ["null", "array"], + "title": "Plans", + "description": "A list of plans for which this coupon applies. This will be `null` if `applies_to_all_plans=true`.", + "items": { + "type": "object", + "title": "Plan mini details", + "description": "Just the important parts.", + "properties": { + "id": { + "type": "string", + "title": "Plan ID", + "maxLength": 13, + "readOnly": true + }, + "code": { + "type": "string", + "title": "Plan code", + "description": "Unique code to identify the plan. This is used in Hosted Payment Page URLs and in the invoice exports.", + "maxLength": 13 + } + } + } + }, + "items": { + "type": ["null", "array"], + "title": "Items", + "description": "A list of items for which this coupon applies. This will be\n`null` if `applies_to_all_items=true`.\n", + "items": { + "type": ["null", "object"], + "title": "Item mini details", + "description": "Just the important parts.", + "properties": { + "id": { + "type": "string", + "title": "Item ID", + "maxLength": 13, + "readOnly": true + } + } + } + }, + "redemption_resource": { + "type": ["null", "string"], + "maxLength": 256 + }, + "discount": { + "type": ["null", "object"], + "description": "Details of the discount a coupon applies. Will contain a `type`\nproperty and one of the following properties: `percent`, `fixed`, `trial`.\n", + "properties": { + "type": { + "type": "string", + "maxLength": 256 + }, + "percent": { + "description": "This is only present when `type=percent`.", + "type": "integer" + }, + "currencies": { + "type": "array", + "description": "This is only present when `type=fixed`.", + "items": { + "type": ["null", "object"], + "properties": { + "currency": { + "type": "string", + "title": "Currency", + "description": "3-letter ISO 4217 currency code.", + "maxLength": 3 + }, + "amount": { + "type": "number", + "format": "float", + "title": "Discount Amount", + "description": "Value of the fixed discount that this coupon applies." + } + } + } + }, + "trial": { + "type": "object", + "description": "This is only present when `type=free_trial`.", + "properties": { + "unit": { + "title": "Trial unit", + "description": "Temporal unit of the free trial", + "type": "string", + "maxLength": 256 + }, + "length": { + "type": "integer", + "title": "Trial length", + "description": "Trial length measured in the units specified by the sibling `unit` property" + } + } + } + } + }, + "coupon_type": { + "type": ["null", "string"], + "maxLength": 256 + }, + "hosted_page_description": { + "type": ["null", "string"], + "maxLength": 1024 + }, + "invoice_description": { + "type": ["null", "string"], + "maxLength": 1024 + }, + "redeem_by": { + "type": ["null", "string"], + "maxLength": 256 + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "expired_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/external_accounts.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/external_accounts.json new file mode 100644 index 000000000000..29e6292263b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/external_accounts.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "object": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "external_account_code": { + "type": ["null", "string"] + }, + "external_connection_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/line_items.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/line_items.json new file mode 100644 index 000000000000..33e1fb8809d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/line_items.json @@ -0,0 +1,293 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "title": "Line item", + "properties": { + "id": { + "type": "string", + "title": "Line item ID", + "maxLength": 13 + }, + "object": { + "type": ["null", "string"] + }, + "uuid": { + "type": "string", + "title": "UUID", + "description": "The UUID is useful for matching data with the CSV exports and building URLs into Recurly's UI.", + "maxLength": 32 + }, + "type": { + "type": "string", + "title": "Line item type", + "description": "Charges are positive line items that debit the account. Credits are negative line items that credit the account.", + "maxLength": 256 + }, + "item_code": { + "type": ["null", "string"], + "title": "Item Code", + "description": "Unique code to identify an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", + "maxLength": 50 + }, + "item_id": { + "type": ["null", "string"], + "title": "Item ID", + "description": "System-generated unique identifier for an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", + "maxLength": 13 + }, + "external_sku": { + "type": ["null", "string"], + "title": "External SKU", + "description": "Optional Stock Keeping Unit assigned to an item. Available when the Credit Invoices and Subscription Billing Terms features are enabled.", + "maxLength": 50 + }, + "revenue_schedule_type": { + "type": ["null", "string"], + "title": "Revenue schedule type", + "maxLength": 256 + }, + "state": { + "type": "string", + "title": "Current state of the line item", + "description": "Pending line items are charges or credits on an account that have not been applied to an invoice yet. Invoiced line items will always have an `invoice_id` value.", + "maxLength": 256 + }, + "legacy_category": { + "type": ["null", "string"], + "title": "Legacy category", + "description": "Category to describe the role of a line item on a legacy invoice:\n- \"charges\" refers to charges being billed for on this invoice.\n- \"credits\" refers to refund or proration credits. This portion of the invoice can be considered a credit memo.\n- \"applied_credits\" refers to previous credits applied to this invoice. See their original_line_item_id to determine where the credit first originated.\n- \"carryforwards\" can be ignored. They exist to consume any remaining credit balance. A new credit with the same amount will be created and placed back on the account.\n" + }, + "account": { + "$ref": "account_details.json" + }, + "bill_for_account_id": { + "type": "string", + "title": "Bill For Account ID", + "maxLength": 13, + "description": "The UUID of the account responsible for originating the line item." + }, + "subscription_id": { + "type": ["null", "string"], + "title": "Subscription ID", + "description": "If the line item is a charge or credit for a subscription, this is its ID.", + "maxLength": 13 + }, + "plan_id": { + "type": ["null", "string"], + "title": "Plan ID", + "description": "If the line item is a charge or credit for a plan or add-on, this is the plan's ID.", + "maxLength": 13 + }, + "plan_code": { + "type": ["null", "string"], + "title": "Plan code", + "description": "If the line item is a charge or credit for a plan or add-on, this is the plan's code.", + "maxLength": 50 + }, + "add_on_id": { + "type": ["null", "string"], + "title": "Add-on ID", + "description": "If the line item is a charge or credit for an add-on this is its ID.", + "maxLength": 13 + }, + "add_on_code": { + "type": ["null", "string"], + "title": "Add-on code", + "description": "If the line item is a charge or credit for an add-on, this is its code.", + "maxLength": 50 + }, + "invoice_id": { + "type": ["null", "string"], + "title": "Invoice ID", + "description": "Once the line item has been invoiced this will be the invoice's ID.", + "maxLength": 13 + }, + "invoice_number": { + "type": ["null", "string"], + "title": "Invoice number", + "description": "Once the line item has been invoiced this will be the invoice's number. If VAT taxation and the Country Invoice Sequencing feature are enabled, invoices will have country-specific invoice numbers for invoices billed to EU countries (ex: FR1001). Non-EU invoices will continue to use the site-level invoice number sequence.", + "maxLength": 256 + }, + "previous_line_item_id": { + "type": ["null", "string"], + "title": "Previous line item ID", + "description": "Will only have a value if the line item is a credit created from a previous credit, or if the credit was created from a charge refund.", + "maxLength": 13 + }, + "original_line_item_invoice_id": { + "type": ["null", "string"], + "title": "Original line item's invoice ID", + "description": "The invoice where the credit originated. Will only have a value if the line item is a credit created from a previous credit, or if the credit was created from a charge refund.", + "maxLength": 13 + }, + "origin": { + "type": "string", + "title": "Origin of line item", + "description": "A credit created from an original charge will have the value of the charge's origin.", + "maxLength": 256 + }, + "accounting_code": { + "type": "string", + "title": "Accounting code", + "description": "Internal accounting code to help you reconcile your revenue to the correct ledger. Line items created as part of a subscription invoice will use the plan or add-on's accounting code, otherwise the value will only be present if you define an accounting code when creating the line item.", + "maxLength": 20 + }, + "product_code": { + "type": "string", + "title": "Product code", + "description": "For plan-related line items this will be the plan's code, for add-on related line items it will be the add-on's code. For item-related line items it will be the item's `external_sku`.", + "maxLength": 50 + }, + "credit_reason_code": { + "type": ["null", "string"], + "title": "Credit reason code", + "description": "The reason the credit was given when line item is `type=credit`.", + "default": "general", + "maxLength": 256 + }, + "currency": { + "type": "string", + "title": "Currency", + "description": "3-letter ISO 4217 currency code.", + "maxLength": 3 + }, + "amount": { + "type": "number", + "format": "float", + "title": "Total after discounts and taxes", + "description": "`(quantity * unit_amount) - (discount + tax)`" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description that appears on the invoice. For subscription related items this will be filled in automatically.", + "maxLength": 255 + }, + "quantity": { + "type": "integer", + "title": "Quantity", + "description": "This number will be multiplied by the unit amount to compute the subtotal before any discounts or taxes.", + "default": 1 + }, + "unit_amount": { + "type": "number", + "format": "float", + "title": "Unit amount", + "description": "Positive amount for a charge, negative amount for a credit." + }, + "unit_amount_decimal": { + "type": ["null", "string"], + "title": "Unit amount decimal", + "description": "Positive amount for a charge, negative amount for a credit." + }, + "subtotal": { + "type": "number", + "format": "float", + "title": "Total before discounts and taxes", + "description": "`quantity * unit_amount`" + }, + "discount": { + "type": ["null", "number"], + "format": "float", + "title": "Discount", + "description": "The discount applied to the line item." + }, + "tax": { + "type": ["null", "number"], + "format": "float", + "title": "Tax", + "description": "The tax amount for the line item." + }, + "taxable": { + "type": "boolean", + "title": "Taxable?", + "description": "`true` if the line item is taxable, `false` if it is not." + }, + "tax_exempt": { + "type": "boolean", + "title": "Tax exempt?", + "description": "`true` exempts tax on charges, `false` applies tax on charges. If not defined, then defaults to the Plan and Site settings. This attribute does not work for credits (negative line items). Credits are always applied post-tax. Pre-tax discounts should use the Coupons feature." + }, + "tax_code": { + "type": ["null", "string"], + "title": "Tax code", + "description": "Used by Avalara, Vertex, and Recurly\u2019s EU VAT tax feature. The tax code values are specific to each tax system. If you are using Recurly\u2019s EU VAT feature you can use `unknown`, `physical`, or `digital`.", + "maxLength": 50 + }, + "tax_info": { + "$ref": "tax_info.json" + }, + "proration_rate": { + "type": ["null", "number"], + "format": "float", + "title": "Proration rate", + "description": "When a line item has been prorated, this is the rate of the proration. Proration rates were made available for line items created after March 30, 2017. For line items created prior to that date, the proration rate will be `null`, even if the line item was prorated.", + "minimum": 0, + "maximum": 1 + }, + "refund": { + "type": "boolean", + "title": "Refund?" + }, + "refunded_quantity": { + "type": ["null", "integer"], + "title": "Refunded Quantity", + "description": "For refund charges, the quantity being refunded. For non-refund charges, the total quantity refunded (possibly over multiple refunds)." + }, + "credit_applied": { + "type": ["null", "number"], + "format": "float", + "title": "Credit Applied", + "description": "The amount of credit from this line item that was applied to the invoice." + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "id": { + "type": "string", + "title": "Shipping Address ID", + "maxLength": 13, + "readOnly": true + } + } + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time", + "title": "Start date", + "description": "If an end date is present, this is value indicates the beginning of a billing time range. If no end date is present it indicates billing for a specific date." + }, + "end_date": { + "type": ["null", "string"], + "format": "date-time", + "title": "End date", + "description": "If this date is provided, it indicates the end of a time range." + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created at", + "description": "When the line item was created." + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Last updated at", + "description": "When the line item was last changed." + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/shipping_addresses.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/shipping_addresses.json new file mode 100644 index 000000000000..e0f8091cdbe6 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/shipping_addresses.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": "string", + "title": "Shipping Address ID", + "maxLength": 13, + "readOnly": true + }, + "object": { + "type": ["null", "string"] + }, + "account_id": { + "type": "string", + "title": "Account ID", + "maxLength": 13, + "readOnly": true + }, + "nickname": { + "type": "string", + "maxLength": 255 + }, + "first_name": { + "type": "string", + "maxLength": 255 + }, + "last_name": { + "type": "string", + "maxLength": 255 + }, + "company": { + "type": "string", + "maxLength": 255 + }, + "email": { + "type": "string", + "maxLength": 255 + }, + "vat_number": { + "type": "string", + "maxLength": 20 + }, + "phone": { + "type": "string", + "maxLength": 30 + }, + "street1": { + "type": "string", + "maxLength": 255 + }, + "street2": { + "type": "string", + "maxLength": 255 + }, + "city": { + "type": "string", + "maxLength": 255 + }, + "region": { + "type": "string", + "maxLength": 255, + "description": "State or province." + }, + "postal_code": { + "type": "string", + "maxLength": 20, + "description": "Zip or postal code." + }, + "country": { + "type": "string", + "maxLength": 50, + "description": "Country, 2-letter ISO 3166-1 alpha-2 code." + }, + "geo_code": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string", + "title": "Created at", + "format": "date-time", + "readOnly": true + }, + "updated_at": { + "type": "string", + "title": "Updated at", + "format": "date-time", + "readOnly": true + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/tax_info.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/tax_info.json new file mode 100644 index 000000000000..13502eb46241 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/tax_info.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "tax_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "tax": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "level": { + "type": ["null", "string"] + }, + "billable": { + "type": ["null", "boolean"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/unique_coupons.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/unique_coupons.json new file mode 100644 index 000000000000..1d2a0a3a117c --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/unique_coupons.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "description": "A unique coupon code for a bulk coupon.", + "properties": { + "id": { + "type": "string", + "title": "Unique Coupon Code ID", + "readOnly": true, + "maxLength": 13 + }, + "object": { + "type": "string" + }, + "code": { + "type": "string", + "title": "Coupon code", + "description": "The code the customer enters to redeem the coupon.", + "maxLength": 256 + }, + "state": { + "type": ["null", "string"], + "title": "State", + "description": "Indicates if the unique coupon code is redeemable or why not.", + "maxLength": 256 + }, + "bulk_coupon_id": { + "type": ["null", "string"], + "title": "Bulk Coupon ID", + "description": "The Coupon ID of the parent Bulk Coupon", + "readOnly": true, + "maxLength": 13 + }, + "bulk_coupon_code": { + "type": ["null", "string"], + "title": "Bulk Coupon code", + "description": "The Coupon code of the parent Bulk Coupon", + "maxLength": 256 + }, + "created_at": { + "type": "string", + "title": "Created at", + "format": "date-time", + "readOnly": true + }, + "updated_at": { + "type": "string", + "title": "Updated at", + "format": "date-time", + "readOnly": true + }, + "redeemed_at": { + "type": ["null", "string"], + "title": "Redeemed at", + "description": "The date and time the unique coupon code was redeemed.", + "format": "date-time", + "readOnly": true + }, + "expired_at": { + "type": ["null", "string"], + "title": "Expired at", + "description": "The date and time the coupon was expired early or reached its `max_redemptions`.", + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/users.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/users.json new file mode 100644 index 000000000000..70b7c49ac122 --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shared/users.json @@ -0,0 +1,33 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": "string" + }, + "object": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shipping_addresses.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shipping_addresses.json index 4860797be798..bcbb555d47a4 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shipping_addresses.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/shipping_addresses.json @@ -1,85 +1,3 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "id": { - "type": "string", - "title": "Shipping Address ID", - "maxLength": 13, - "readOnly": true - }, - "account_id": { - "type": "string", - "title": "Account ID", - "maxLength": 13, - "readOnly": true - }, - "nickname": { - "type": "string", - "maxLength": 255 - }, - "first_name": { - "type": "string", - "maxLength": 255 - }, - "last_name": { - "type": "string", - "maxLength": 255 - }, - "company": { - "type": "string", - "maxLength": 255 - }, - "email": { - "type": "string", - "maxLength": 255 - }, - "vat_number": { - "type": "string", - "maxLength": 20 - }, - "phone": { - "type": "string", - "maxLength": 30 - }, - "street1": { - "type": "string", - "maxLength": 255 - }, - "street2": { - "type": "string", - "maxLength": 255 - }, - "city": { - "type": "string", - "maxLength": 255 - }, - "region": { - "type": "string", - "maxLength": 255, - "description": "State or province." - }, - "postal_code": { - "type": "string", - "maxLength": 20, - "description": "Zip or postal code." - }, - "country": { - "type": "string", - "maxLength": 50, - "description": "Country, 2-letter ISO 3166-1 alpha-2 code." - }, - "created_at": { - "type": "string", - "title": "Created at", - "format": "date-time", - "readOnly": true - }, - "updated_at": { - "type": "string", - "title": "Updated at", - "format": "date-time", - "readOnly": true - } - } + "$ref": "shipping_addresses.json" } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/subscriptions.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/subscriptions.json index eeed47064a7c..27c3b0ad4ea1 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/subscriptions.json @@ -6,20 +6,48 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "uuid": { "type": ["null", "string"], "maxLength": 32 }, "account": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "code": { "type": ["null", "string"], "maxLength": 256 + }, + "email": { + "type": ["null", "string"], + "maxLength": 256 + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "parent_account_id": { + "type": ["null", "string"] + }, + "bill_to": { + "type": ["null", "string"] + }, + "dunning_campaign_id": { + "type": ["null", "string"] } } }, @@ -30,9 +58,15 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "code": { "type": ["null", "string"], "maxLength": 256 + }, + "name": { + "type": ["null", "string"] } } }, @@ -43,16 +77,11 @@ "shipping": { "type": ["null", "object"], "properties": { + "object": { + "type": ["null", "string"] + }, "address": { - "type": ["null", "object"], - "properties": { - "id": { - "type": "string", - "title": "Shipping Address ID", - "maxLength": 13, - "readOnly": true - } - } + "$ref": "shipping_addresses.json" }, "method": { "type": ["null", "object"], @@ -62,6 +91,15 @@ "title": "Shipping Method ID", "readOnly": true, "maxLength": 13 + }, + "object": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] } } }, @@ -70,6 +108,7 @@ } } }, + "coupon_redemptions": { "$ref": "coupon_redemptions.json" }, "pending_change": { "type": ["null", "object"], "title": "Subscription Change", @@ -153,6 +192,31 @@ "auto_renew": { "type": ["null", "boolean"] }, + "ramp_intervals": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "starting_billing_cycle": { + "type": ["null", "integer"] + }, + "remaining_billing_cycles": { + "type": ["null", "integer"] + }, + "starting_on": { + "type": ["null", "string"], + "format": "date-time" + }, + "ending_on": { + "type": ["null", "string"], + "format": "date-time" + }, + "unit_amount": { + "type": ["null", "number"] + } + } + } + }, "paused_at": { "type": ["null", "string"], "format": "date-time" @@ -171,6 +235,9 @@ "unit_amount": { "type": ["null", "number"] }, + "tax_inclusive": { + "type": ["null", "boolean"] + }, "quantity": { "type": ["null", "number"] }, @@ -206,61 +273,7 @@ "type": ["null", "number"] }, "tax_info": { - "type": ["null", "object"], - "title": "Tax info", - "properties": { - "type": { - "type": "string", - "title": "Type", - "description": "Provides the tax type as \"vat\" for EU VAT, \"usst\" for U.S. Sales Tax, or the 2 letter country code for country level tax types like Canada, Australia, New Zealand, Israel, and all non-EU European countries.", - "maxLength": 256 - }, - "region": { - "type": "string", - "title": "Region", - "description": "Provides the tax region applied on an invoice. For U.S. Sales Tax, this will be the 2 letter state code. For EU VAT this will be the 2 letter country code. For all country level tax types, this will display the regional tax, like VAT, GST, or PST.", - "maxLength": 256 - }, - "rate": { - "type": "number", - "format": "float", - "title": "Rate" - }, - "tax_details": { - "type": ["null", "array"], - "description": "Provides additional tax details for Canadian Sales Tax when there is tax applied at both the country and province levels. This will only be populated for the Invoice response when fetching a single invoice and not for the InvoiceList or LineItem.", - "items": { - "type": "object", - "title": "Tax detail", - "properties": { - "type": { - "type": "string", - "title": "Type", - "description": "Provides the tax type for the region. For Canadian Sales Tax, this will be GST, HST, QST or PST.", - "maxLength": 256 - }, - "region": { - "type": "string", - "title": "Region", - "description": "Provides the tax region applied on an invoice. For Canadian Sales Tax, this will be either the 2 letter province code or country code.", - "maxLength": 256 - }, - "rate": { - "type": "number", - "format": "float", - "title": "Rate", - "description": "Provides the tax rate for the region." - }, - "tax": { - "type": "number", - "format": "float", - "title": "Tax", - "description": "The total tax applied for this tax type." - } - } - } - } - } + "$ref": "tax_info.json" }, "total": { "type": ["null", "number"] @@ -276,6 +289,9 @@ "net_terms": { "type": ["null", "number"] }, + "net_terms_type": { + "type": ["null", "string"] + }, "terms_and_conditions": { "type": ["null", "string"], "maxLength": 16384 @@ -288,6 +304,20 @@ "type": ["null", "string"], "maxLength": 1024 }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, "created_at": { "type": ["null", "string"], "format": "date-time" @@ -319,6 +349,20 @@ "billing_info_id": { "type": ["null", "string"], "maxLength": 13 + }, + "active_invoice_id": { + "type": ["null", "string"] + }, + "started_with_gift": { + "type": ["null", "boolean"] + }, + "converted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "action_result": { + "type": ["null", "object"], + "additionalProperties": true } } } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/transactions.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/transactions.json index 660e28842f2f..7b06e0d76cb5 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/transactions.json @@ -1,11 +1,15 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "uuid": { "type": ["null", "string"], "maxLength": 32 @@ -15,17 +19,7 @@ "maxLength": 13 }, "account": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"], - "maxLength": 13 - }, - "code": { - "type": ["null", "string"], - "maxLength": 256 - } - } + "$ref": "account_details.json" }, "invoice": { "type": ["null", "object"], @@ -37,6 +31,15 @@ "number": { "type": ["null", "string"], "maxLength": 256 + }, + "business_entity_id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] } } }, @@ -47,9 +50,21 @@ "type": ["null", "string"], "maxLength": 13 }, + "object": { + "type": ["null", "string"] + }, "number": { "type": ["null", "string"], "maxLength": 256 + }, + "business_entity_id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] } } }, @@ -126,6 +141,9 @@ "country": { "type": ["null", "string"], "maxLength": 256 + }, + "geo_code": { + "type": ["null", "string"] } } }, @@ -136,6 +154,9 @@ "payment_method": { "type": "object", "properties": { + "object": { + "type": ["null", "string"] + }, "card_type": { "type": ["null", "string"], "maxLength": 256 @@ -162,6 +183,9 @@ "type": ["null", "string"], "maxLength": 256 }, + "cc_bin_country": { + "type": ["null", "string"] + }, "gateway_code": { "type": ["null", "string"], "maxLength": 256 @@ -185,6 +209,9 @@ "routing_number_bank": { "type": ["null", "string"], "maxLength": 256 + }, + "username": { + "type": ["null", "string"] } } }, @@ -218,6 +245,15 @@ "id": { "type": ["null", "string"], "maxLength": 13 + }, + "object": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] } } }, @@ -266,6 +302,44 @@ "collected_at": { "type": ["null", "string"], "format": "date-time" + }, + "action_result": { + "type": ["null", "object"], + "additionalProperties": true + }, + "vat_number": { + "type": ["null", "string"] + }, + "fraud_info": { + "type": ["null", "object"], + "properties": { + "object": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "integer"] + }, + "decision": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + }, + "risk_rules_triggered": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/unique_coupons.json b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/unique_coupons.json index 1d8d5522b27d..0458768570ff 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/unique_coupons.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/schemas/unique_coupons.json @@ -1,63 +1,3 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "description": "A unique coupon code for a bulk coupon.", - "properties": { - "id": { - "type": "string", - "title": "Unique Coupon Code ID", - "readOnly": true, - "maxLength": 13 - }, - "code": { - "type": "string", - "title": "Coupon code", - "description": "The code the customer enters to redeem the coupon.", - "maxLength": 256 - }, - "state": { - "type": "string", - "title": "State", - "description": "Indicates if the unique coupon code is redeemable or why not.", - "maxLength": 256 - }, - "bulk_coupon_id": { - "type": ["null", "string"], - "title": "Bulk Coupon ID", - "description": "The Coupon ID of the parent Bulk Coupon", - "readOnly": true, - "maxLength": 13 - }, - "bulk_coupon_code": { - "type": ["null", "string"], - "title": "Bulk Coupon code", - "description": "The Coupon code of the parent Bulk Coupon", - "maxLength": 256 - }, - "created_at": { - "type": "string", - "title": "Created at", - "format": "date-time", - "readOnly": true - }, - "updated_at": { - "type": "string", - "title": "Updated at", - "format": "date-time", - "readOnly": true - }, - "redeemed_at": { - "type": ["null", "string"], - "title": "Redeemed at", - "description": "The date and time the unique coupon code was redeemed.", - "format": "date-time", - "readOnly": true - }, - "expired_at": { - "type": ["null", "string"], - "title": "Expired at", - "description": "The date and time the coupon was expired early or reached its `max_redemptions`.", - "format": "date-time" - } - } + "$ref": "unique_coupons.json" } diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/spec.json b/airbyte-integrations/connectors/source-recurly/source_recurly/spec.json index 02a427f7378b..d2135eb06551 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/spec.json +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/spec.json @@ -5,7 +5,7 @@ "title": "Recurly Source Spec", "type": "object", "required": ["api_key"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "api_key": { "type": "string", diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml deleted file mode 100644 index b69725d9b4b6..000000000000 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml +++ /dev/null @@ -1,35 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-redshift:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - timeout_seconds: "1200" - config_path: "secrets/config.json" - connection: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - status: "succeed" - discovery: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - basic_read: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: "1200" - incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_inc.json" - timeout_seconds: "1200" diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index 408c45723776..fb8474465151 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -1,39 +1,21 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.redshift.RedshiftSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation 'com.amazon.redshift:redshift-jdbc42:1.2.43.1067' - testImplementation 'org.apache.commons:commons-text:1.10.0' - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' - testImplementation project.project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - + testImplementation 'org.hamcrest:hamcrest-all:1.3' + testImplementation "org.testcontainers:jdbc:1.19.4" } diff --git a/airbyte-integrations/connectors/source-redshift/metadata.yaml b/airbyte-integrations/connectors/source-redshift/metadata.yaml index 373c5157645a..b8908d9443b5 100644 --- a/airbyte-integrations/connectors/source-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/source-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.2 dockerRepository: airbyte/source-redshift documentationUrl: https://docs.airbyte.com/integrations/sources/redshift githubIssueLabel: source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index d80a2558ef1b..b497968bb891 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -109,6 +109,7 @@ public Set getExcludedInternalNameSpaces() { } @Override + @SuppressWarnings("unchecked") public Set getPrivilegesTableForCurrentUser(final JdbcDatabase database, final String schema) throws SQLException { return new HashSet<>(database.bufferedResultSetQuery( connection -> { diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java index 47ea14662c35..72f64b0069d4 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java @@ -5,38 +5,35 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.redshift.RedshiftSource; import java.nio.file.Path; -import java.sql.JDBCType; -import java.sql.SQLException; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; // Run as part of integration tests, instead of unit tests, because there is no test container for // Redshift. -class RedshiftJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class RedshiftJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private JsonNode config; - - private static JsonNode getStaticConfig() { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); - } + private static JsonNode config; @BeforeAll static void init() { + config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s GEOMETRY)"; INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(ST_Point(129.77099609375, 62.093299865722656))"; } - @BeforeEach - public void setup() throws Exception { - config = getStaticConfig(); - super.setup(); + @Override + protected RedshiftTestDatabase createTestDatabase() { + final RedshiftTestDatabase testDatabase = new RedshiftTestDatabase(source().toDatabaseConfig(Jsons.clone(config))).initialized(); + try { + for (final String schemaName : TEST_SCHEMAS) { + testDatabase.with(DROP_SCHEMA_QUERY, schemaName); + } + } catch (final Exception ignore) {} + return testDatabase; } @Override @@ -45,23 +42,13 @@ public boolean supportsSchemas() { } @Override - public AbstractJdbcSource getJdbcSource() { + protected RedshiftSource source() { return new RedshiftSource(); } @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return RedshiftSource.DRIVER_CLASS; - } - - @AfterEach - public void tearDownRedshift() throws SQLException { - super.tearDown(); + protected JsonNode config() { + return Jsons.clone(config); } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java index 856f6e9d1e6e..1ff7fc23b136 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.source.redshift.RedshiftSourceOperations; import java.nio.file.Path; import java.sql.SQLException; +import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; @@ -28,6 +29,7 @@ public class RedshiftSourceOperationsTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); private JdbcDatabase database; @BeforeEach @@ -39,7 +41,8 @@ void setup() { config.get("password").asText(), DatabaseDriver.REDSHIFT.getDriverClassName(), RedshiftSource.getJdbcUrl(config), - JdbcDataSourceUtils.getConnectionProperties(config)); + JdbcDataSourceUtils.getConnectionProperties(config), + CONNECTION_TIMEOUT); database = new DefaultJdbcDatabase(dataSource, new RedshiftSourceOperations()); } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java index 66e959d401ee..787cebc127f3 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java @@ -10,9 +10,14 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import java.time.Duration; +import org.junit.jupiter.api.Disabled; +@Disabled public class RedshiftSslSourceAcceptanceTest extends RedshiftSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Override protected JdbcDatabase createDatabase(final JsonNode config) { return new DefaultJdbcDatabase( @@ -25,7 +30,8 @@ protected JdbcDatabase createDatabase(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), JdbcUtils.parseJdbcParameters("ssl=true&" + - "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"))); + "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"), + CONNECTION_TIMEOUT)); } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java new file mode 100644 index 000000000000..18f7ca2ee73d --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.REDSHIFT; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class RedshiftTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + + protected RedshiftTestDatabase(final JsonNode redshiftConfig) { + super(new NonContainer(redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText(), + redshiftConfig.has(JdbcUtils.PASSWORD_KEY) ? redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + redshiftConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), REDSHIFT.getDriverClassName(), "")); + this.username = redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = redshiftConfig.has(JdbcUtils.PASSWORD_KEY) ? redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = redshiftConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return REDSHIFT; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public void close() {} + + static public class RedshiftConfigBuilder extends TestDatabase.ConfigBuilder { + + protected RedshiftConfigBuilder(RedshiftTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-reply-io/main.py b/airbyte-integrations/connectors/source-reply-io/main.py index 879d184cebcd..ebe3ff2bfe7b 100644 --- a/airbyte-integrations/connectors/source-reply-io/main.py +++ b/airbyte-integrations/connectors/source-reply-io/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_reply_io import SourceReplyIo +from source_reply_io.run import run if __name__ == "__main__": - source = SourceReplyIo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-reply-io/metadata.yaml b/airbyte-integrations/connectors/source-reply-io/metadata.yaml index 9ad7fcafafd6..d57e6cb62bdf 100644 --- a/airbyte-integrations/connectors/source-reply-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-reply-io/metadata.yaml @@ -8,6 +8,10 @@ data: icon: reply-io.svg license: MIT name: Reply.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-reply-io registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-reply-io/setup.py b/airbyte-integrations/connectors/source-reply-io/setup.py index bc4d841b50fc..4b8ad096a754 100644 --- a/airbyte-integrations/connectors/source-reply-io/setup.py +++ b/airbyte-integrations/connectors/source-reply-io/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-reply-io=source_reply_io.run:run", + ], + }, name="source_reply_io", description="Source implementation for Reply Io.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py b/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py new file mode 100644 index 000000000000..e740b708c45a --- /dev/null +++ b/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_reply_io import SourceReplyIo + + +def run(): + source = SourceReplyIo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-retently/main.py b/airbyte-integrations/connectors/source-retently/main.py index 4aafda0c119f..dec9d2267e7b 100644 --- a/airbyte-integrations/connectors/source-retently/main.py +++ b/airbyte-integrations/connectors/source-retently/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_retently import SourceRetently +from source_retently.run import run if __name__ == "__main__": - source = SourceRetently() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-retently/metadata.yaml b/airbyte-integrations/connectors/source-retently/metadata.yaml index 40818fb95bca..77e0ea8d3fd7 100644 --- a/airbyte-integrations/connectors/source-retently/metadata.yaml +++ b/airbyte-integrations/connectors/source-retently/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-retently registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-retently/setup.py b/airbyte-integrations/connectors/source-retently/setup.py index 707ffcee736f..daf9a0a93efc 100644 --- a/airbyte-integrations/connectors/source-retently/setup.py +++ b/airbyte-integrations/connectors/source-retently/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-retently=source_retently.run:run", + ], + }, name="source_retently", description="Source implementation for Retently.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-retently/source_retently/run.py b/airbyte-integrations/connectors/source-retently/source_retently/run.py new file mode 100644 index 000000000000..4e0687b5c6a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_retently import SourceRetently + + +def run(): + source = SourceRetently() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ringcentral/main.py b/airbyte-integrations/connectors/source-ringcentral/main.py index cb412fcab28f..12f2eb4285cc 100644 --- a/airbyte-integrations/connectors/source-ringcentral/main.py +++ b/airbyte-integrations/connectors/source-ringcentral/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ringcentral import SourceRingcentral +from source_ringcentral.run import run if __name__ == "__main__": - source = SourceRingcentral() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml index e92f20f242b9..62c18c12ec9d 100644 --- a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml +++ b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ringcentral.svg license: MIT name: Ringcentral + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ringcentral registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-ringcentral/setup.py b/airbyte-integrations/connectors/source-ringcentral/setup.py index a0a34d080b35..90d664f5d9f6 100644 --- a/airbyte-integrations/connectors/source-ringcentral/setup.py +++ b/airbyte-integrations/connectors/source-ringcentral/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ringcentral=source_ringcentral.run:run", + ], + }, name="source_ringcentral", description="Source implementation for Ringcentral.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py new file mode 100644 index 000000000000..f203a14ce2d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ringcentral import SourceRingcentral + + +def run(): + source = SourceRingcentral() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rki-covid/main.py b/airbyte-integrations/connectors/source-rki-covid/main.py index 7b345c79d6d5..a104106001cb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/main.py +++ b/airbyte-integrations/connectors/source-rki-covid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rki_covid import SourceRkiCovid +from source_rki_covid.run import run if __name__ == "__main__": - source = SourceRkiCovid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml index fb3acfd90923..d11317580f3d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml +++ b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rki.svg license: MIT name: RKI Covid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rki-covid registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py index 4c8abfd77674..382e0f4e84b4 100644 --- a/airbyte-integrations/connectors/source-rki-covid/setup.py +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "airbyte-cdk"] setup( + entry_points={ + "console_scripts": [ + "source-rki-covid=source_rki_covid.run:run", + ], + }, name="source_rki_covid", description="Source implementation for Rki Covid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py new file mode 100644 index 000000000000..dc5cdd95d152 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rki_covid import SourceRkiCovid + + +def run(): + source = SourceRkiCovid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rocket-chat/main.py b/airbyte-integrations/connectors/source-rocket-chat/main.py index befcababa0fe..3e9e4cdfb68e 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/main.py +++ b/airbyte-integrations/connectors/source-rocket-chat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rocket_chat import SourceRocketChat +from source_rocket_chat.run import run if __name__ == "__main__": - source = SourceRocketChat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml index b78e01397580..35229f131695 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rocket-chat.svg license: MIT name: Rocket.chat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rocket-chat registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-rocket-chat/setup.py b/airbyte-integrations/connectors/source-rocket-chat/setup.py index 86e7d41371ab..0dea0d1fba8f 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/setup.py +++ b/airbyte-integrations/connectors/source-rocket-chat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rocket-chat=source_rocket_chat.run:run", + ], + }, name="source_rocket_chat", description="Source implementation for Rocket Chat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py b/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py new file mode 100644 index 000000000000..1cb56e281978 --- /dev/null +++ b/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rocket_chat import SourceRocketChat + + +def run(): + source = SourceRocketChat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rss/main.py b/airbyte-integrations/connectors/source-rss/main.py index 967135ca3254..9e21c3c97793 100644 --- a/airbyte-integrations/connectors/source-rss/main.py +++ b/airbyte-integrations/connectors/source-rss/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rss import SourceRss +from source_rss.run import run if __name__ == "__main__": - source = SourceRss() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rss/metadata.yaml b/airbyte-integrations/connectors/source-rss/metadata.yaml index 50a3f9531e7b..8e7d7b8fdc91 100644 --- a/airbyte-integrations/connectors/source-rss/metadata.yaml +++ b/airbyte-integrations/connectors/source-rss/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rss.svg license: MIT name: RSS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rss registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-rss/setup.py b/airbyte-integrations/connectors/source-rss/setup.py index 7bd87cde621b..f49ef214cb6a 100644 --- a/airbyte-integrations/connectors/source-rss/setup.py +++ b/airbyte-integrations/connectors/source-rss/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rss=source_rss.run:run", + ], + }, name="source_rss", description="Source implementation for Rss.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rss/source_rss/run.py b/airbyte-integrations/connectors/source-rss/source_rss/run.py new file mode 100644 index 000000000000..90f8a101fcfa --- /dev/null +++ b/airbyte-integrations/connectors/source-rss/source_rss/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rss import SourceRss + + +def run(): + source = SourceRss() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-s3/.coveragerc b/airbyte-integrations/connectors/source-s3/.coveragerc index 0c7476c81073..4c1de9ec0853 100644 --- a/airbyte-integrations/connectors/source-s3/.coveragerc +++ b/airbyte-integrations/connectors/source-s3/.coveragerc @@ -3,4 +3,5 @@ omit = source_s3/exceptions.py source_s3/stream.py source_s3/utils.py - source_s3/source_files_abstract/source.py \ No newline at end of file + source_s3/source_files_abstract/source.py + source_s3/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/README.md b/airbyte-integrations/connectors/source-s3/README.md index a057b1b24b62..71cb2aa21b8c 100644 --- a/airbyte-integrations/connectors/source-s3/README.md +++ b/airbyte-integrations/connectors/source-s3/README.md @@ -1,119 +1,55 @@ -# S3 Source +# S3 source connector + This is the repository for the S3 source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/s3). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/s3). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/s3) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_s3/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/s3) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_s3/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source s3 test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-s3 spec +poetry run source-s3 check --config secrets/config.json +poetry run source-s3 discover --config secrets/config.json +poetry run source-s3 read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-s3 build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-s3:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-s3:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-s3:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-s3:dev . -# Running the spec command against your patched connector -docker run airbyte/source-s3:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-s3:dev spec @@ -122,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-s3:dev discover --conf docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-s3:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-s3 test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-s3 test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/s3.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/s3.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index 252460fff68d..fec20a1fcff6 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -4,112 +4,182 @@ acceptance_tests: - config_path: secrets/config.json expect_records: path: integration_tests/expected_records/csv.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" + - config_path: secrets/config_iam_role.json + expect_records: + path: integration_tests/expected_records/csv.jsonl + exact_order: false + timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_custom_encoding_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_custom_format_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_format.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_user_schema_config.json expect_records: path: integration_tests/expected_records/legacy_csv_user_schema.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_no_header.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_skip_rows_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_skip_rows_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_with_nulls_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_csv_with_null_bools_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_null_bools.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_parquet_config.json expect_records: path: integration_tests/expected_records/parquet.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/parquet_dataset_config.json expect_records: path: integration_tests/expected_records/parquet_dataset.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_parquet_decimal_config.json expect_records: path: integration_tests/expected_records/legacy_parquet_decimal.jsonl timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_avro_config.json expect_records: path: integration_tests/expected_records/avro.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_jsonl_config.json expect_records: path: integration_tests/expected_records/jsonl.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/v4_jsonl_newlines_config.json expect_records: path: integration_tests/expected_records/jsonl_newlines.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_csv.json expect_records: path: integration_tests/expected_records/zip_csv.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_csv_custom_encoding.json expect_records: path: integration_tests/expected_records/zip_csv_custom_encoding.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_jsonl.json expect_records: path: integration_tests/expected_records/zip_jsonl.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_avro.json expect_records: path: integration_tests/expected_records/zip_avro.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/zip_config_parquet.json expect_records: path: integration_tests/expected_records/zip_parquet.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 + file_types: + skip_test: true + bypass_reason: "To be testes with the last config" - config_path: secrets/unstructured_config.json expect_records: path: integration_tests/expected_records/unstructured.jsonl - exact_order: true + exact_order: false timeout_seconds: 1800 connection: tests: - config_path: secrets/config.json status: succeed + - config_path: secrets/config_iam_role.json + status: succeed - config_path: secrets/v4_csv_custom_encoding_config.json status: succeed - config_path: secrets/v4_csv_custom_format_config.json @@ -201,6 +271,9 @@ acceptance_tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalogs/csv.json timeout_seconds: 1800 + - config_path: secrets/config_iam_role.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + timeout_seconds: 1800 - config_path: secrets/v4_parquet_config.json configured_catalog_path: integration_tests/configured_catalogs/parquet.json timeout_seconds: 1800 @@ -236,6 +309,11 @@ acceptance_tests: future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 + - config_path: secrets/config_iam_role.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 - config_path: secrets/v4_parquet_config.json configured_catalog_path: integration_tests/configured_catalogs/parquet.json future_state: diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py index 6b0c294530cd..706e9eba88be 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py @@ -3,14 +3,46 @@ # +import json +import logging +from pathlib import Path from typing import Iterable import pytest +import yaml pytest_plugins = ("connector_acceptance_test.plugin",) +logger = logging.getLogger("airbyte") @pytest.fixture(scope="session", autouse=True) def connector_setup() -> Iterable[None]: - """This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is responsible for configuring AWS credentials that are used for assuming role during the IAM role based authentication.""" + config_file_path = "secrets/config_iam_role.json" + acceptance_test_config_file_path = "acceptance-test-config.yml" + + # Read environment variables from the JSON file + with open(config_file_path, "r") as file: + config = json.load(file) + + # Prepare environment variables to append to the YAML file + env_vars = { + "custom_environment_variables": { + "AWS_ASSUME_ROLE_EXTERNAL_ID": config["acceptance_test_aws_external_id"], + "AWS_ACCESS_KEY_ID": config["acceptance_test_aws_access_key_id"], + "AWS_SECRET_ACCESS_KEY": config["acceptance_test_aws_secret_access_key"], + } + } + + # Append environment variables to the YAML file + yaml_path = Path(acceptance_test_config_file_path) + if yaml_path.is_file(): + with open(acceptance_test_config_file_path, "r") as file: + existing_data = yaml.safe_load(file) or {} + existing_data.update(env_vars) + with open(acceptance_test_config_file_path, "w") as file: + yaml.safe_dump(existing_data, file) + else: + raise Exception(f"{acceptance_test_config_file_path} does not exist.") + yield diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json index 80d1ba4217ca..3d0f3a14fc07 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json @@ -58,7 +58,7 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", "type": "string", "airbyte_hidden": true }, @@ -288,16 +288,50 @@ "const": "unstructured", "type": "string" }, - "skip_unprocessable_file_types": { + "skip_unprocessable_files": { "type": "boolean", "default": true, - "title": "Skip Unprocessable File Types", - "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] } }, - "required": ["filetype"], - "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] } ] }, @@ -324,6 +358,12 @@ "order": 2, "type": "string" }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "order": 6, + "type": "string" + }, "aws_secret_access_key": { "title": "AWS Secret Access Key", "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", @@ -340,6 +380,12 @@ "order": 4, "type": "string" }, + "region_name": { + "title": "AWS Region", + "description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + "order": 5, + "type": "string" + }, "dataset": { "title": "Output Stream Name", "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", @@ -576,6 +622,13 @@ "order": 2, "type": "string" }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "always_show": true, + "order": 7, + "type": "string" + }, "path_prefix": { "title": "Path Prefix", "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", @@ -590,13 +643,19 @@ "order": 4, "type": "string" }, + "region_name": { + "title": "AWS Region", + "description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + "order": 5, + "type": "string" + }, "start_date": { "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.", "examples": ["2021-01-01T00:00:00Z"], "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 5, + "order": 6, "type": "string" } }, diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl index 7b1f5f14c68c..539daeeb36e4 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl @@ -1,2 +1,2 @@ -{"stream": "test", "data": {"document_key": "Testdoc.pdf", "content": "# Heading\n\nThis is the content which is not just a single word", "_ab_source_file_last_modified": "2023-10-20T12:52:38.000000Z", "_ab_source_file_url": "Testdoc.pdf"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"document_key": "Testdoc_OCR.pdf", "content": "This is a test", "_ab_source_file_last_modified": "2023-10-23T10:55:37.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf"}, "emitted_at": 162727468000} \ No newline at end of file +{"stream": "test", "data": {"document_key": "Testdoc.pdf", "content": "# Heading\n\nThis is the content which is not just a single word", "_ab_source_file_last_modified": "2023-10-20T12:52:38.000000Z", "_ab_source_file_url": "Testdoc.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 162727468000} +{"stream": "test", "data": {"document_key": "Testdoc_OCR.pdf", "content": "This is a test", "_ab_source_file_last_modified": "2023-10-23T10:55:37.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index 4ab87a5ec93f..8592fc5684ef 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -58,7 +58,7 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", "type": "string", "airbyte_hidden": true }, @@ -288,12 +288,46 @@ "const": "unstructured", "type": "string" }, - "skip_unprocessable_file_types": { + "skip_unprocessable_files": { "type": "boolean", "default": true, - "title": "Skip Unprocessable File Types", - "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] } }, "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", @@ -331,6 +365,12 @@ "order": 3, "type": "string" }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "order": 6, + "type": "string" + }, "endpoint": { "title": "Endpoint", "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", @@ -339,6 +379,12 @@ "order": 4, "type": "string" }, + "region_name": { + "title": "AWS Region", + "description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + "order": 5, + "type": "string" + }, "dataset": { "title": "Output Stream Name", "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", @@ -575,6 +621,13 @@ "order": 2, "type": "string" }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "always_show": true, + "order": 7, + "type": "string" + }, "path_prefix": { "title": "Path Prefix", "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", @@ -589,13 +642,19 @@ "order": 4, "type": "string" }, + "region_name": { + "title": "AWS Region", + "description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + "order": 5, + "type": "string" + }, "start_date": { "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.", "examples": ["2021-01-01T00:00:00Z"], "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 5, + "order": 6, "type": "string" } }, diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py index c3b6b0bc32ed..cb0007d5581b 100644 --- a/airbyte-integrations/connectors/source-s3/main.py +++ b/airbyte-integrations/connectors/source-s3/main.py @@ -3,40 +3,7 @@ # -import sys -import traceback -from datetime import datetime -from typing import List - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type -from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader - - -def get_source(args: List[str]): - catalog_path = AirbyteEntrypoint.extract_catalog(args) - try: - return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) - except Exception: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - return None - +from source_s3.run import run if __name__ == "__main__": - _args = sys.argv[1:] - source = get_source(_args) - - if source: - launch(source, _args) + run() diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 56686a0063d7..2b3b204f87be 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.2.4 + dockerImageTag: 4.5.7 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 icon: s3.svg license: ELv2 name: S3 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-s3 registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-s3/poetry.lock b/airbyte-integrations/connectors/source-s3/poetry.lock new file mode 100644 index 000000000000..b8b2e27ae350 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/poetry.lock @@ -0,0 +1,2591 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.67.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.67.0.tar.gz", hash = "sha256:cbbff1b3895c89313764a721870bb293a396c74bad8dd6e5c36a0c3b0a2f6a10"}, + {file = "airbyte_cdk-0.67.0-py3-none-any.whl", hash = "sha256:2082c859536a2450c03b89dba1bbdab21bad314fbf5ef6d2e86fefc4ba935373"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +avro = {version = ">=1.11.2,<1.12.0", optional = true, markers = "extra == \"file-based\""} +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +fastavro = {version = ">=1.8.0,<1.9.0", optional = true, markers = "extra == \"file-based\""} +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +markdown = {version = "*", optional = true, markers = "extra == \"file-based\""} +pdf2image = {version = "1.16.3", optional = true, markers = "extra == \"file-based\""} +"pdfminer.six" = {version = "20221105", optional = true, markers = "extra == \"file-based\""} +pendulum = "<3.0.0" +pyarrow = {version = "12.0.1", optional = true, markers = "extra == \"file-based\""} +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +pytesseract = {version = "0.3.10", optional = true, markers = "extra == \"file-based\""} +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +unstructured = [ + {version = "0.10.27", optional = true, markers = "extra == \"file-based\""}, + {version = "0.10.27", extras = ["docx", "pptx"], optional = true, markers = "extra == \"file-based\""}, +] +"unstructured.pytesseract" = {version = ">=0.3.12", optional = true, markers = "extra == \"file-based\""} +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "avro" +version = "1.11.3" +description = "Avro is a serialization and RPC framework." +optional = false +python-versions = ">=3.6" +files = [ + {file = "avro-1.11.3.tar.gz", hash = "sha256:3393bb5139f9cf0791d205756ce1e39a5b58586af5b153d6a3b5a199610e9d17"}, +] + +[package.extras] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "boto3" +version = "1.34.48" +description = "The AWS SDK for Python" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "boto3-1.34.48-py3-none-any.whl", hash = "sha256:adc785ff05aec9fc93f82d507420b320203cd4fd011c67eb369b3aa2b5aeb298"}, + {file = "boto3-1.34.48.tar.gz", hash = "sha256:f9873c3f03de546d7297475c6acd771840c385521caadb8c121a1ac38bc59cd4"}, +] + +[package.dependencies] +botocore = ">=1.34.48,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.34.48" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "botocore-1.34.48-py3-none-any.whl", hash = "sha256:f3e1c84fa75fd6921dfbfb4b2f803bcc424b9b866982fe80e08edbd26ca9861c"}, + {file = "botocore-1.34.48.tar.gz", hash = "sha256:eabdde36309274b76bb79ae9bdfa10c1fd91a2c9b3343cfa15b8a91f8e1ec224"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = [ + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, +] + +[package.extras] +crt = ["awscrt (==0.19.19)"] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449"}, + {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b"}, + {file = "cryptography-42.0.4-cp37-abi3-win32.whl", hash = "sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925"}, + {file = "cryptography-42.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923"}, + {file = "cryptography-42.0.4-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0"}, + {file = "cryptography-42.0.4-cp39-abi3-win32.whl", hash = "sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129"}, + {file = "cryptography-42.0.4-cp39-abi3-win_amd64.whl", hash = "sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660"}, + {file = "cryptography-42.0.4.tar.gz", hash = "sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "dataclasses-json" +version = "0.6.4" +description = "Easily serialize dataclasses to and from JSON." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dataclasses_json-0.6.4-py3-none-any.whl", hash = "sha256:f90578b8a3177f7552f4e1a6e535e84293cd5da421fcce0642d49c0d7bdf8df2"}, + {file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"}, +] + +[package.dependencies] +marshmallow = ">=3.18.0,<4.0.0" +typing-inspect = ">=0.4.0,<1" + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dill" +version = "0.3.4" +description = "serialize all of python" +optional = false +python-versions = ">=2.7, !=3.0.*" +files = [ + {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, + {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "docker" +version = "7.0.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.0.0-py3-none-any.whl", hash = "sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b"}, + {file = "docker-7.0.0.tar.gz", hash = "sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "emoji" +version = "2.10.1" +description = "Emoji for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "emoji-2.10.1-py2.py3-none-any.whl", hash = "sha256:11fb369ea79d20c14efa4362c732d67126df294a7959a2c98bfd7447c12a218e"}, + {file = "emoji-2.10.1.tar.gz", hash = "sha256:16287283518fb7141bde00198f9ffff4e1c1cb570efb68b2f1ec50975c3a581d"}, +] + +[package.extras] +dev = ["coverage", "coveralls", "pytest"] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastavro" +version = "1.8.4" +description = "Fast read/write of AVRO files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastavro-1.8.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:7afe1475e8a967c04e2b0ef4d33bc10bffa66b4fa6e08bd2ee9d91b6768cba2a"}, + {file = "fastavro-1.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5fd73609f3c1ac0d90ae3179d2fb9d788f842245db2656ff9225fce871fc5b7"}, + {file = "fastavro-1.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78fdf1ba47e43146af72ac48d7b2247a06c4f2d95dfdaad6129c481014b07a6b"}, + {file = "fastavro-1.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d950542b3263653f00b695cbc728b5c60ab9ea6df32a7017ad9a6a67235386e7"}, + {file = "fastavro-1.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ce2ccfa9aff8df6da683c48542b7b2a216dde6d3a4d1c505c5e1b8ca2ec0abbb"}, + {file = "fastavro-1.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:f12f9914d6196695d3208ea348145a80d0defefe16b8a226373fe8ce68f66139"}, + {file = "fastavro-1.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d353aec9c000b96c33ad285651a2cba0f87fe50fcdecc6120689996af427194d"}, + {file = "fastavro-1.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eaed91d6e1fb06c172e0aaf4b1ca1fd019c3f4a481e314bf783a4c74f6b7015"}, + {file = "fastavro-1.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9293b303955acd34a6f69dd4ef3465bd575dbde0cd3e3f00227a0ba5012430b4"}, + {file = "fastavro-1.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b79baefd61554d9f03c4beaebbe638ef175d0efc1fb01f25e88ee6ae97985ab3"}, + {file = "fastavro-1.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:14d7cd3df019d41c66733b8bf5d983af9e1f601d4cb162853a49238a4087d6b0"}, + {file = "fastavro-1.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:c8fb27001b7023910969f15bee2c9205c4e9f40713929d6c1dca8f470fc8fc80"}, + {file = "fastavro-1.8.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e331229acef15f858d9863ced7b629ebef4bd5f80766d367255e51cbf44f8dab"}, + {file = "fastavro-1.8.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04e26b3ba288bd423f25630a3b9bd70cc61b46c6f6161de35e398a6fc8f260f0"}, + {file = "fastavro-1.8.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6281f4555659ed658b195d1618a637504013e57b680d6cbad7c726e9a4e2cf0b"}, + {file = "fastavro-1.8.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3201880149e1fb807d616ab46b338a26788173a9f4e8a3396ae145e86af878a1"}, + {file = "fastavro-1.8.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:39771719fa04b8321eeebfb0813eaa2723c20e5bf570bcca3f53f1169099a0d7"}, + {file = "fastavro-1.8.4-cp312-cp312-win_amd64.whl", hash = "sha256:7095ae37a5c46dacb7ef430092e5f94650f576be281487b72050c1cf12e4ee20"}, + {file = "fastavro-1.8.4-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:eb76f5bfcde91cde240c93594dae47670cdf1a95d7e5d0dc3ccdef57c6c1c183"}, + {file = "fastavro-1.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71ebe1cf090f800ca7d4c64d50c81c2a88c56e6ef6aa5eb61ec425e7ae723617"}, + {file = "fastavro-1.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f0ef601943ea11cd02a59c57f5588cea3e300ac67608f53c904ec7aeddd232"}, + {file = "fastavro-1.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1060318f3ab31bcc4b2184cea3763305b773163381601e304000da81a2f7e11f"}, + {file = "fastavro-1.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:01c8c7f22172174f2c2c0922801b552fbca75758f84b0ad3cd6f3e505a76ed05"}, + {file = "fastavro-1.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:bc8a1af80b8face4a41d8526a34b6474a874f7367a900d0b14752eacebb7a2b8"}, + {file = "fastavro-1.8.4-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:687a2f8fa83a76906c4ec35c9d0500e13a567fc631845f0e47646c48233c7725"}, + {file = "fastavro-1.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b921c63fcfb9149a32c40a9cd27b0e900fcda602455cbce4d773300019b9ce2"}, + {file = "fastavro-1.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2610a8683b10be7aaa532ddddbcb719883ee2d6f09dafd4a4a7b46d5d719fc07"}, + {file = "fastavro-1.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:94448dc749d098f846f6a6d82d59f85483bd6fcdecfb6234daac5f4494ae4156"}, + {file = "fastavro-1.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2d39c6b5db7014a3722a7d206310874430486f4895161911b6b6574cb1a6c48f"}, + {file = "fastavro-1.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:3b73472e8da33bcbf08ec989996637aea04eaca71058bb6d45def6fa4168f541"}, + {file = "fastavro-1.8.4.tar.gz", hash = "sha256:dae6118da27e81abf5957dc79a6d778888fc1bbf67645f52959cb2faba95beff"}, +] + +[package.extras] +codecs = ["lz4", "python-snappy", "zstandard"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "filetype" +version = "1.2.0" +description = "Infer file type and MIME type of any file/buffer. No external dependencies." +optional = false +python-versions = "*" +files = [ + {file = "filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25"}, + {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, +] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "importlib-metadata" +version = "7.0.1" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"}, + {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "joblib" +version = "1.3.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, + {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, +] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "langdetect" +version = "1.0.9" +description = "Language detection library ported from Google's language-detection." +optional = false +python-versions = "*" +files = [ + {file = "langdetect-1.0.9-py2-none-any.whl", hash = "sha256:7cbc0746252f19e76f77c0b1690aadf01963be835ef0cd4b56dddf2a8f1dfc2a"}, + {file = "langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "lxml" +version = "5.1.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, + {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, + {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, + {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, + {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8"}, + {file = "lxml-5.1.0-cp312-cp312-win32.whl", hash = "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01"}, + {file = "lxml-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623"}, + {file = "lxml-5.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67"}, + {file = "lxml-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd"}, + {file = "lxml-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7"}, + {file = "lxml-5.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, + {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, + {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, + {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, + {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890"}, + {file = "lxml-5.1.0-cp39-cp39-win32.whl", hash = "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e"}, + {file = "lxml-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f"}, + {file = "lxml-5.1.0.tar.gz", hash = "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.7)"] + +[[package]] +name = "markdown" +version = "3.5.2" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Markdown-3.5.2-py3-none-any.whl", hash = "sha256:d43323865d89fc0cb9b20c75fc8ad313af307cc087e84b657d9eec768eddeadd"}, + {file = "Markdown-3.5.2.tar.gz", hash = "sha256:e1ac7b3dc550ee80e602e71c1d168002f062e49f1b11e26a36264dafd4df2ef8"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +testing = ["coverage", "pyyaml"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "marshmallow" +version = "3.20.2" +description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +optional = false +python-versions = ">=3.8" +files = [ + {file = "marshmallow-3.20.2-py3-none-any.whl", hash = "sha256:c21d4b98fee747c130e6bc8f45c4b3199ea66bc00c12ee1f639f0aeca034d5e9"}, + {file = "marshmallow-3.20.2.tar.gz", hash = "sha256:4c1daff273513dc5eb24b219a8035559dc573c8f322558ef85f5438ddd1236dd"}, +] + +[package.dependencies] +packaging = ">=17.0" + +[package.extras] +dev = ["pre-commit (>=2.4,<4.0)", "pytest", "pytz", "simplejson", "tox"] +docs = ["alabaster (==0.7.15)", "autodocsumm (==0.2.12)", "sphinx (==7.2.6)", "sphinx-issues (==3.0.1)", "sphinx-version-warning (==1.1.2)"] +lint = ["pre-commit (>=2.4,<4.0)"] +tests = ["pytest", "pytz", "simplejson"] + +[[package]] +name = "moto" +version = "4.2.14" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "moto-4.2.14-py2.py3-none-any.whl", hash = "sha256:6d242dbbabe925bb385ddb6958449e5c827670b13b8e153ed63f91dbdb50372c"}, + {file = "moto-4.2.14.tar.gz", hash = "sha256:8f9263ca70b646f091edcc93e97cda864a542e6d16ed04066b1370ed217bd190"}, +] + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.5.0)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.0)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.0)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "nltk" +version = "3.8.1" +description = "Natural Language Toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"}, + {file = "nltk-3.8.1.zip", hash = "sha256:1834da3d0682cba4f2cede2f9aad6b0fafb6461ba451db0efb6f9c39798d64d3"}, +] + +[package.dependencies] +click = "*" +joblib = "*" +regex = ">=2021.8.3" +tqdm = "*" + +[package.extras] +all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] +corenlp = ["requests"] +machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] +plot = ["matplotlib"] +tgrep = ["pyparsing"] +twitter = ["twython"] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, + {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, + {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, + {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, + {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, + {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, + {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, + {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, + {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, + {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, + {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] + +[[package]] +name = "pdf2image" +version = "1.16.3" +description = "A wrapper around the pdftoppm and pdftocairo command line tools to convert PDF to a PIL Image list." +optional = false +python-versions = "*" +files = [ + {file = "pdf2image-1.16.3-py3-none-any.whl", hash = "sha256:b6154164af3677211c22cbb38b2bd778b43aca02758e962fe1e231f6d3b0e380"}, + {file = "pdf2image-1.16.3.tar.gz", hash = "sha256:74208810c2cef4d9e347769b8e62a52303982ddb4f2dfd744c7ab4b940ae287e"}, +] + +[package.dependencies] +pillow = "*" + +[[package]] +name = "pdfminer-six" +version = "20221105" +description = "PDF parser and analyzer" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pdfminer.six-20221105-py3-none-any.whl", hash = "sha256:1eaddd712d5b2732f8ac8486824533514f8ba12a0787b3d5fe1e686cd826532d"}, + {file = "pdfminer.six-20221105.tar.gz", hash = "sha256:8448ab7b939d18b64820478ecac5394f482d7a79f5f7eaa7703c6c959c175e1d"}, +] + +[package.dependencies] +charset-normalizer = ">=2.0.0" +cryptography = ">=36.0.0" + +[package.extras] +dev = ["black", "mypy (==0.931)", "nox", "pytest"] +docs = ["sphinx", "sphinx-argparse"] +image = ["Pillow"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "pillow" +version = "10.2.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyarrow" +version = "12.0.1" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytesseract" +version = "0.3.10" +description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytesseract-0.3.10-py3-none-any.whl", hash = "sha256:8f22cc98f765bf13517ead0c70effedb46c153540d25783e04014f28b55a5fc6"}, + {file = "pytesseract-0.3.10.tar.gz", hash = "sha256:f1c3a8b0f07fd01a1085d451f5b8315be6eec1d5577a6796d46dc7a62bd4120f"}, +] + +[package.dependencies] +packaging = ">=21.3" +Pillow = ">=8.0.0" + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-docx" +version = "1.1.0" +description = "Create, read, and update Microsoft Word .docx files." +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-docx-1.1.0.tar.gz", hash = "sha256:5829b722141cf1ab79aedf0c34d9fe9924b29764584c0f2164eb2b02dcdf17c9"}, + {file = "python_docx-1.1.0-py3-none-any.whl", hash = "sha256:bac9773278098a1ddc43a52d84e22f5909c4a3080a624530b3ecb3771b07c6cd"}, +] + +[package.dependencies] +lxml = ">=3.1.0" +typing-extensions = "*" + +[[package]] +name = "python-iso639" +version = "2024.2.7" +description = "Look-up utilities for ISO 639 language codes and names" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-iso639-2024.2.7.tar.gz", hash = "sha256:c323233348c34d57c601e3e6d824088e492896bcb97a61a87f7d93401a305377"}, + {file = "python_iso639-2024.2.7-py3-none-any.whl", hash = "sha256:7b149623ff74230f4ee3061fb01d18e57a8d07c5fee2aa72907f39b7f6d16cbc"}, +] + +[package.extras] +dev = ["black (==24.1.1)", "build (==1.0.3)", "flake8 (==7.0.0)", "pytest (==8.0.0)", "twine (==4.0.2)"] + +[[package]] +name = "python-magic" +version = "0.4.27" +description = "File type identification using libmagic" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b"}, + {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, +] + +[[package]] +name = "python-pptx" +version = "0.6.21" +description = "Generate and manipulate Open XML PowerPoint (.pptx) files" +optional = false +python-versions = "*" +files = [ + {file = "python-pptx-0.6.21.tar.gz", hash = "sha256:7798a2aaf89563565b3c7120c0acfe9aff775db0db3580544e3bf4840c2e378f"}, +] + +[package.dependencies] +lxml = ">=3.1.0" +Pillow = ">=3.3.2" +XlsxWriter = ">=0.5.7" + +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +optional = false +python-versions = "*" +files = [ + {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, + {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, + {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, + {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, + {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, + {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, + {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, + {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, +] + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "rapidfuzz" +version = "3.6.1" +description = "rapid fuzzy string matching" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ac434fc71edda30d45db4a92ba5e7a42c7405e1a54cb4ec01d03cc668c6dcd40"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a791168e119cfddf4b5a40470620c872812042f0621e6a293983a2d52372db0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a2f3e9df346145c2be94e4d9eeffb82fab0cbfee85bd4a06810e834fe7c03fa"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23de71e7f05518b0bbeef55d67b5dbce3bcd3e2c81e7e533051a2e9401354eb0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d056e342989248d2bdd67f1955bb7c3b0ecfa239d8f67a8dfe6477b30872c607"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01835d02acd5d95c1071e1da1bb27fe213c84a013b899aba96380ca9962364bc"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed0f712e0bb5fea327e92aec8a937afd07ba8de4c529735d82e4c4124c10d5a0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96cd19934f76a1264e8ecfed9d9f5291fde04ecb667faef5f33bdbfd95fe2d1f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e06c4242a1354cf9d48ee01f6f4e6e19c511d50bb1e8d7d20bcadbb83a2aea90"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d73dcfe789d37c6c8b108bf1e203e027714a239e50ad55572ced3c004424ed3b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:06e98ff000e2619e7cfe552d086815671ed09b6899408c2c1b5103658261f6f3"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:08b6fb47dd889c69fbc0b915d782aaed43e025df6979b6b7f92084ba55edd526"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a1788ebb5f5b655a15777e654ea433d198f593230277e74d51a2a1e29a986283"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c65f92881753aa1098c77818e2b04a95048f30edbe9c3094dc3707d67df4598b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:4243a9c35667a349788461aae6471efde8d8800175b7db5148a6ab929628047f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_arm64.whl", hash = "sha256:f59d19078cc332dbdf3b7b210852ba1f5db8c0a2cd8cc4c0ed84cc00c76e6802"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fbc07e2e4ac696497c5f66ec35c21ddab3fc7a406640bffed64c26ab2f7ce6d6"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cced1a8852652813f30fb5d4b8f9b237112a0bbaeebb0f4cc3611502556764"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82300e5f8945d601c2daaaac139d5524d7c1fdf719aa799a9439927739917460"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf97c321fd641fea2793abce0e48fa4f91f3c202092672f8b5b4e781960b891"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7420e801b00dee4a344ae2ee10e837d603461eb180e41d063699fb7efe08faf0"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060bd7277dc794279fa95522af355034a29c90b42adcb7aa1da358fc839cdb11"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7e3375e4f2bfec77f907680328e4cd16cc64e137c84b1886d547ab340ba6928"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a490cd645ef9d8524090551016f05f052e416c8adb2d8b85d35c9baa9d0428ab"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2e03038bfa66d2d7cffa05d81c2f18fd6acbb25e7e3c068d52bb7469e07ff382"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b19795b26b979c845dba407fe79d66975d520947b74a8ab6cee1d22686f7967"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:064c1d66c40b3a0f488db1f319a6e75616b2e5fe5430a59f93a9a5e40a656d15"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3c772d04fb0ebeece3109d91f6122b1503023086a9591a0b63d6ee7326bd73d9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:841eafba6913c4dfd53045835545ba01a41e9644e60920c65b89c8f7e60c00a9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win32.whl", hash = "sha256:266dd630f12696ea7119f31d8b8e4959ef45ee2cbedae54417d71ae6f47b9848"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:d79aec8aeee02ab55d0ddb33cea3ecd7b69813a48e423c966a26d7aab025cdfe"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_arm64.whl", hash = "sha256:484759b5dbc5559e76fefaa9170147d1254468f555fd9649aea3bad46162a88b"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b2ef4c0fd3256e357b70591ffb9e8ed1d439fb1f481ba03016e751a55261d7c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:588c4b20fa2fae79d60a4e438cf7133d6773915df3cc0a7f1351da19eb90f720"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7142ee354e9c06e29a2636b9bbcb592bb00600a88f02aa5e70e4f230347b373e"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dfc557c0454ad22382373ec1b7df530b4bbd974335efe97a04caec936f2956a"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03f73b381bdeccb331a12c3c60f1e41943931461cdb52987f2ecf46bfc22f50d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b0ccc2ec1781c7e5370d96aef0573dd1f97335343e4982bdb3a44c133e27786"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da3e8c9f7e64bb17faefda085ff6862ecb3ad8b79b0f618a6cf4452028aa2222"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fde9b14302a31af7bdafbf5cfbb100201ba21519be2b9dedcf4f1048e4fbe65d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c1a23eee225dfb21c07f25c9fcf23eb055d0056b48e740fe241cbb4b22284379"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e49b9575d16c56c696bc7b06a06bf0c3d4ef01e89137b3ddd4e2ce709af9fe06"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:0a9fc714b8c290261669f22808913aad49553b686115ad0ee999d1cb3df0cd66"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a3ee4f8f076aa92184e80308fc1a079ac356b99c39408fa422bbd00145be9854"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f056ba42fd2f32e06b2c2ba2443594873cfccc0c90c8b6327904fc2ddf6d5799"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win32.whl", hash = "sha256:5d82b9651e3d34b23e4e8e201ecd3477c2baa17b638979deeabbb585bcb8ba74"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:dad55a514868dae4543ca48c4e1fc0fac704ead038dafedf8f1fc0cc263746c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_arm64.whl", hash = "sha256:3c84294f4470fcabd7830795d754d808133329e0a81d62fcc2e65886164be83b"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e19d519386e9db4a5335a4b29f25b8183a1c3f78cecb4c9c3112e7f86470e37f"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01eb03cd880a294d1bf1a583fdd00b87169b9cc9c9f52587411506658c864d73"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:be368573255f8fbb0125a78330a1a40c65e9ba3c5ad129a426ff4289099bfb41"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3e5af946f419c30f5cb98b69d40997fe8580efe78fc83c2f0f25b60d0e56efb"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f382f7ffe384ce34345e1c0b2065451267d3453cadde78946fbd99a59f0cc23c"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be156f51f3a4f369e758505ed4ae64ea88900dcb2f89d5aabb5752676d3f3d7e"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1936d134b6c513fbe934aeb668b0fee1ffd4729a3c9d8d373f3e404fbb0ce8a0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ff8eaf4a9399eb2bebd838f16e2d1ded0955230283b07376d68947bbc2d33d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ae598a172e3a95df3383634589660d6b170cc1336fe7578115c584a99e0ba64d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cd4ba4c18b149da11e7f1b3584813159f189dc20833709de5f3df8b1342a9759"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:0402f1629e91a4b2e4aee68043a30191e5e1b7cd2aa8dacf50b1a1bcf6b7d3ab"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:1e12319c6b304cd4c32d5db00b7a1e36bdc66179c44c5707f6faa5a889a317c0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0bbfae35ce4de4c574b386c43c78a0be176eeddfdae148cb2136f4605bebab89"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win32.whl", hash = "sha256:7fec74c234d3097612ea80f2a80c60720eec34947066d33d34dc07a3092e8105"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:a553cc1a80d97459d587529cc43a4c7c5ecf835f572b671107692fe9eddf3e24"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:757dfd7392ec6346bd004f8826afb3bf01d18a723c97cbe9958c733ab1a51791"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2963f4a3f763870a16ee076796be31a4a0958fbae133dbc43fc55c3968564cf5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d2f0274595cc5b2b929c80d4e71b35041104b577e118cf789b3fe0a77b37a4c5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f211e366e026de110a4246801d43a907cd1a10948082f47e8a4e6da76fef52"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a59472b43879012b90989603aa5a6937a869a72723b1bf2ff1a0d1edee2cc8e6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a03863714fa6936f90caa7b4b50ea59ea32bb498cc91f74dc25485b3f8fccfe9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd95b6b7bfb1584f806db89e1e0c8dbb9d25a30a4683880c195cc7f197eaf0c"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7183157edf0c982c0b8592686535c8b3e107f13904b36d85219c77be5cefd0d8"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ad9d74ef7c619b5b0577e909582a1928d93e07d271af18ba43e428dc3512c2a1"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b53137d81e770c82189e07a8f32722d9e4260f13a0aec9914029206ead38cac3"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:49b9ed2472394d306d5dc967a7de48b0aab599016aa4477127b20c2ed982dbf9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:dec307b57ec2d5054d77d03ee4f654afcd2c18aee00c48014cb70bfed79597d6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4381023fa1ff32fd5076f5d8321249a9aa62128eb3f21d7ee6a55373e672b261"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win32.whl", hash = "sha256:8d7a072f10ee57c8413c8ab9593086d42aaff6ee65df4aa6663eecdb7c398dca"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:ebcfb5bfd0a733514352cfc94224faad8791e576a80ffe2fd40b2177bf0e7198"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_arm64.whl", hash = "sha256:1c47d592e447738744905c18dda47ed155620204714e6df20eb1941bb1ba315e"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eef8b346ab331bec12bbc83ac75641249e6167fab3d84d8f5ca37fd8e6c7a08c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53251e256017e2b87f7000aee0353ba42392c442ae0bafd0f6b948593d3f68c6"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dede83a6b903e3ebcd7e8137e7ff46907ce9316e9d7e7f917d7e7cdc570ee05"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4da90e4c2b444d0a171d7444ea10152e07e95972bb40b834a13bdd6de1110c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ca3dfcf74f2b6962f411c33dd95b0adf3901266e770da6281bc96bb5a8b20de9"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bcc957c0a8bde8007f1a8a413a632a1a409890f31f73fe764ef4eac55f59ca87"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:692c9a50bea7a8537442834f9bc6b7d29d8729a5b6379df17c31b6ab4df948c2"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c23ceaea27e790ddd35ef88b84cf9d721806ca366199a76fd47cfc0457a81b"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b155e67fff215c09f130555002e42f7517d0ea72cbd58050abb83cb7c880cec"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3028ee8ecc48250607fa8a0adce37b56275ec3b1acaccd84aee1f68487c8557b"}, + {file = "rapidfuzz-3.6.1.tar.gz", hash = "sha256:35660bee3ce1204872574fa041c7ad7ec5175b3053a4cb6e181463fc07013de7"}, +] + +[package.extras] +full = ["numpy"] + +[[package]] +name = "regex" +version = "2023.12.25" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.7" +files = [ + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f"}, + {file = "regex-2023.12.25-cp310-cp310-win32.whl", hash = "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630"}, + {file = "regex-2023.12.25-cp310-cp310-win_amd64.whl", hash = "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4"}, + {file = "regex-2023.12.25-cp311-cp311-win32.whl", hash = "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87"}, + {file = "regex-2023.12.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d"}, + {file = "regex-2023.12.25-cp312-cp312-win32.whl", hash = "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5"}, + {file = "regex-2023.12.25-cp312-cp312-win_amd64.whl", hash = "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232"}, + {file = "regex-2023.12.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39"}, + {file = "regex-2023.12.25-cp37-cp37m-win32.whl", hash = "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c"}, + {file = "regex-2023.12.25-cp37-cp37m-win_amd64.whl", hash = "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2"}, + {file = "regex-2023.12.25-cp38-cp38-win32.whl", hash = "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb"}, + {file = "regex-2023.12.25-cp38-cp38-win_amd64.whl", hash = "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20"}, + {file = "regex-2023.12.25-cp39-cp39-win32.whl", hash = "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9"}, + {file = "regex-2023.12.25-cp39-cp39-win_amd64.whl", hash = "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91"}, + {file = "regex-2023.12.25.tar.gz", hash = "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "responses" +version = "0.25.0" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"}, + {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] + +[[package]] +name = "s3transfer" +version = "0.10.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3transfer-0.10.0-py3-none-any.whl", hash = "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e"}, + {file = "s3transfer-0.10.0.tar.gz", hash = "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "69.1.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, + {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smart-open" +version = "5.1.0" +description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +optional = false +python-versions = ">=3.6.*" +files = [ + {file = "smart_open-5.1.0-py3-none-any.whl", hash = "sha256:2059b07f530c8c9e2158e4e1575309aacb74bd813da2325c1f348015d04f3bd6"}, + {file = "smart_open-5.1.0.tar.gz", hash = "sha256:e4dc1350b240ef0759e343e4e2f361bfd4e5477bb2619866e97f80240652e92e"}, +] + +[package.dependencies] +boto3 = {version = "*", optional = true, markers = "extra == \"s3\""} + +[package.extras] +all = ["azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage", "requests"] +azure = ["azure-common", "azure-core", "azure-storage-blob"] +gcs = ["google-cloud-storage"] +http = ["requests"] +s3 = ["boto3"] +test = ["azure-common", "azure-core", "azure-storage-blob", "boto3", "google-cloud-storage", "moto[server] (==1.3.14)", "parameterizedtestcase", "paramiko", "pathlib2", "pytest", "pytest-rerunfailures", "requests", "responses"] +webhdfs = ["requests"] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "tqdm" +version = "4.66.2" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"}, + {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +description = "Runtime inspection utilities for typing module." +optional = false +python-versions = "*" +files = [ + {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, + {file = "typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"}, +] + +[package.dependencies] +mypy-extensions = ">=0.3.0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "unstructured" +version = "0.10.27" +description = "A library that prepares raw documents for downstream ML tasks." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "unstructured-0.10.27-py3-none-any.whl", hash = "sha256:3a8a8e44302388ddc39c184059e8b4458f1cdc58032540b9af7d85f6c3eca3be"}, + {file = "unstructured-0.10.27.tar.gz", hash = "sha256:f567b5c4385993a9ab48db5563dd7b413aac4f2002bb22e6250496ea8f440f5e"}, +] + +[package.dependencies] +backoff = "*" +beautifulsoup4 = "*" +chardet = "*" +dataclasses-json = "*" +emoji = "*" +filetype = "*" +langdetect = "*" +lxml = "*" +nltk = "*" +numpy = "*" +python-docx = {version = ">=1.0.1", optional = true, markers = "extra == \"docx\""} +python-iso639 = "*" +python-magic = "*" +python-pptx = {version = "<=0.6.21", optional = true, markers = "extra == \"pptx\""} +rapidfuzz = "*" +requests = "*" +tabulate = "*" +typing-extensions = "*" + +[package.extras] +airtable = ["pyairtable"] +all-docs = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +azure = ["adlfs", "fsspec (==2023.9.1)"] +azure-cognitive-search = ["azure-search-documents"] +bedrock = ["boto3", "langchain"] +biomed = ["bs4"] +box = ["boxfs", "fsspec (==2023.9.1)"] +confluence = ["atlassian-python-api"] +csv = ["pandas"] +delta-table = ["deltalake", "fsspec (==2023.9.1)"] +discord = ["discord-py"] +doc = ["python-docx (>=1.0.1)"] +docx = ["python-docx (>=1.0.1)"] +dropbox = ["dropboxdrivefs", "fsspec (==2023.9.1)"] +elasticsearch = ["elasticsearch", "jq"] +embed-huggingface = ["huggingface", "langchain", "sentence-transformers"] +epub = ["pypandoc"] +gcs = ["bs4", "fsspec (==2023.9.1)", "gcsfs"] +github = ["pygithub (>1.58.0)"] +gitlab = ["python-gitlab"] +google-drive = ["google-api-python-client"] +huggingface = ["langdetect", "sacremoses", "sentencepiece", "torch", "transformers"] +image = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)"] +jira = ["atlassian-python-api"] +local-inference = ["markdown", "msg-parser", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pypandoc", "python-docx (>=1.0.1)", "python-pptx (<=0.6.21)", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +md = ["markdown"] +msg = ["msg-parser"] +notion = ["htmlBuilder", "notion-client"] +odt = ["pypandoc", "python-docx (>=1.0.1)"] +onedrive = ["Office365-REST-Python-Client (<2.4.3)", "bs4", "msal"] +openai = ["langchain", "openai", "tiktoken"] +org = ["pypandoc"] +outlook = ["Office365-REST-Python-Client (<2.4.3)", "msal"] +paddleocr = ["unstructured.paddleocr (==2.6.1.3)"] +pdf = ["onnx", "pdf2image", "pdfminer.six", "unstructured-inference (==0.7.10)", "unstructured.pytesseract (>=0.3.12)"] +ppt = ["python-pptx (<=0.6.21)"] +pptx = ["python-pptx (<=0.6.21)"] +reddit = ["praw"] +rst = ["pypandoc"] +rtf = ["pypandoc"] +s3 = ["fsspec (==2023.9.1)", "s3fs"] +salesforce = ["simple-salesforce"] +sharepoint = ["Office365-REST-Python-Client (<2.4.3)", "msal"] +slack = ["slack-sdk"] +tsv = ["pandas"] +wikipedia = ["wikipedia"] +xlsx = ["networkx", "openpyxl", "pandas", "xlrd"] + +[[package]] +name = "unstructured-pytesseract" +version = "0.3.12" +description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +optional = false +python-versions = ">=3.8" +files = [ + {file = "unstructured.pytesseract-0.3.12-py3-none-any.whl", hash = "sha256:6ed42530fc697bb08d1ae4884cc517ee808620c1c1414efe8d5d90334da068d3"}, + {file = "unstructured.pytesseract-0.3.12.tar.gz", hash = "sha256:751a21d67b1f109036bf4daf796d3e04631697a355efd650f3373412b249de2e"}, +] + +[package.dependencies] +packaging = ">=21.3" +Pillow = ">=8.0.0" + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "urllib3" +version = "2.0.7" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, + {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "werkzeug" +version = "3.0.1" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, + {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xlsxwriter" +version = "3.2.0" +description = "A Python module for creating Excel XLSX files." +optional = false +python-versions = ">=3.6" +files = [ + {file = "XlsxWriter-3.2.0-py3-none-any.whl", hash = "sha256:ecfd5405b3e0e228219bcaf24c2ca0915e012ca9464a14048021d21a995d490e"}, + {file = "XlsxWriter-3.2.0.tar.gz", hash = "sha256:9977d0c661a72866a61f9f7a809e25ebbb0fb7036baa3b9fe74afcfca6b3cb8c"}, +] + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + +[[package]] +name = "zipp" +version = "3.17.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, + {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "e1d8b49218f24ab7ceb86c74a9374287891f17e6481215e1049163a61df28f85" diff --git a/airbyte-integrations/connectors/source-s3/pyproject.toml b/airbyte-integrations/connectors/source-s3/pyproject.toml new file mode 100644 index 000000000000..3127d3b94c76 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "4.5.7" +name = "source-s3" +description = "Source implementation for S3." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/s3" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_s3" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +pytz = "==2024.1" +wcmatch = "==8.4" +python-snappy = "==0.6.1" +dill = "==0.3.4" + +[tool.poetry.scripts] +source-s3 = "source_s3.run:run" + +[tool.poetry.dependencies.airbyte-cdk] +extras = [ "file-based",] +version = "^0" + +[tool.poetry.dependencies.smart-open] +extras = [ "s3",] +version = "==5.1.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +moto = "==4.2.14" +docker = "^7.0.0" +pytest-mock = "^3.6.1" +requests-mock = "^1.9.3" +pandas = "==2.0.3" diff --git a/airbyte-integrations/connectors/source-s3/requirements.txt b/airbyte-integrations/connectors/source-s3/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-s3/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py deleted file mode 100644 index 3c67f5a8e084..000000000000 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.55.5", - "smart-open[s3]==5.1.0", - "wcmatch==8.4", - "dill==0.3.4", - "pytz", - "python-snappy==0.6.1", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", - "pandas==2.0.3", - "docker", -] - -setup( - name="source_s3", - description="Source implementation for S3.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/run.py b/airbyte-integrations/connectors/source-s3/source_s3/run.py new file mode 100644 index 000000000000..b29f09febbda --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/run.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader + + +def get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + try: + return SourceS3( + SourceS3StreamReader(), + Config, + SourceS3.read_catalog(catalog_path) if catalog_path else None, + SourceS3.read_config(config_path) if config_path else None, + SourceS3.read_state(state_path) if state_path else None, + cursor_cls=Cursor, + ) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = get_source(_args) + + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source.py index 8621fe4bbb5a..3fe19247578f 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source.py @@ -1,8 +1,6 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - - from typing import Optional from pydantic import BaseModel, Field @@ -39,6 +37,14 @@ class Config: always_show=True, order=2, ) + role_arn: Optional[str] = Field( + title=f"AWS Role ARN", + default=None, + description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " + f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + always_show=True, + order=7, + ) path_prefix: str = Field( default="", description="By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, " @@ -48,13 +54,19 @@ class Config: ) endpoint: str = Field("", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4) + region_name: Optional[str] = Field( + title="AWS Region", + default=None, + description="AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + order=5, + ) start_date: Optional[str] = Field( title="Start Date", description="UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.", examples=["2021-01-01T00:00:00Z"], format="date-time", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - order=5, + order=6, ) provider: S3Provider diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index 5f0a891c6a61..349377e069ad 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -1,9 +1,9 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any, Dict, Optional -from typing import Optional - +import dpath.util from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.utils import is_cloud_environment from pydantic import AnyUrl, Field, ValidationError, root_validator @@ -30,6 +30,14 @@ def documentation_url(cls) -> AnyUrl: order=2, ) + role_arn: Optional[str] = Field( + title=f"AWS Role ARN", + default=None, + description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " + f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + order=6, + ) + aws_secret_access_key: Optional[str] = Field( title="AWS Secret Access Key", default=None, @@ -47,6 +55,13 @@ def documentation_url(cls) -> AnyUrl: order=4, ) + region_name: Optional[str] = Field( + title="AWS Region", + default=None, + description="AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.", + order=5, + ) + @root_validator def validate_optional_args(cls, values): aws_access_key_id = values.get("aws_access_key_id") @@ -63,3 +78,16 @@ def validate_optional_args(cls, values): raise ValidationError("The endpoint must be a secure HTTPS endpoint.", model=Config) return values + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py index bcb4ccdffef4..22a801597aaa 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -6,7 +6,7 @@ from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message from airbyte_cdk.models import ConnectorSpecification -from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.file_based_source import DEFAULT_CONCURRENCY, FileBasedSource from airbyte_cdk.utils import is_cloud_environment from source_s3.source import SourceS3Spec from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer @@ -21,14 +21,17 @@ class SourceS3(FileBasedSource): - def read_config(self, config_path: str) -> Mapping[str, Any]: + _concurrency_level = DEFAULT_CONCURRENCY + + @classmethod + def read_config(cls, config_path: str) -> Mapping[str, Any]: """ Used to override the default read_config so that when the new file-based S3 connector processes a config in the legacy format, it can be transformed into the new config. This happens in entrypoint before we validate the config against the new spec. """ config = super().read_config(config_path) - if not self._is_v4_config(config): + if not SourceS3._is_v4_config(config): parsed_legacy_config = SourceS3Spec(**config) converted_config = LegacyConfigTransformer.convert(parsed_legacy_config) emit_configuration_as_airbyte_control_message(converted_config) @@ -66,7 +69,8 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: connectionSpecification=s4_spec, ) - def _is_v4_config(self, config: Mapping[str, Any]) -> bool: + @staticmethod + def _is_v4_config(config: Mapping[str, Any]) -> bool: return "streams" in config @staticmethod diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py index d8bfbd5b16bc..c3e415df7b25 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -5,9 +5,11 @@ import logging from datetime import datetime from io import IOBase +from os import getenv from typing import Iterable, List, Optional, Set import boto3.session +import pendulum import pytz import smart_open from airbyte_cdk.models import FailureType @@ -16,15 +18,18 @@ from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient from botocore.client import Config as ClientConfig +from botocore.credentials import RefreshableCredentials from botocore.exceptions import ClientError +from botocore.session import get_session from source_s3.v4.config import Config from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler +AWS_EXTERNAL_ID = getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") + class SourceS3StreamReader(AbstractFileBasedStreamReader): def __init__(self): super().__init__() - self._s3_client = None @property def config(self) -> Config: @@ -50,15 +55,71 @@ def s3_client(self) -> BaseClient: # We shouldn't hit this; config should always get set before attempting to # list or read files. raise ValueError("Source config is missing; cannot create the S3 client.") - if self._s3_client is None: - client_kv_args = _get_s3_compatible_client_args(self.config) if self.config.endpoint else {} - self._s3_client = boto3.client( + + client_kv_args = _get_s3_compatible_client_args(self.config) if self.config.endpoint else {} + + # Set the region_name if it's provided in the config + if self.config.region_name: + client_kv_args["region_name"] = self.config.region_name + + if self.config.role_arn: + _s3_client = self._get_iam_s3_client(client_kv_args) + else: + _s3_client = boto3.client( "s3", aws_access_key_id=self.config.aws_access_key_id, aws_secret_access_key=self.config.aws_secret_access_key, **client_kv_args, ) - return self._s3_client + + return _s3_client + + def _get_iam_s3_client(self, client_kv_args: dict) -> BaseClient: + """ + Creates an S3 client using AWS Security Token Service (STS) with assumed role credentials. This method handles + the authentication process by assuming an IAM role, optionally using an external ID for enhanced security. + The obtained credentials are set to auto-refresh upon expiration, ensuring uninterrupted access to the S3 service. + + :param client_kv_args: A dictionary of key-value pairs for the boto3 S3 client constructor. + :return: An instance of a boto3 S3 client with the assumed role credentials. + + The method assumes a role specified in the `self.config.role_arn` and creates a session with the S3 service. + If `AWS_ASSUME_ROLE_EXTERNAL_ID` environment variable is set, it will be used during the role assumption for additional security. + """ + + def refresh(): + client = boto3.client("sts") + if AWS_EXTERNAL_ID: + role = client.assume_role( + RoleArn=self.config.role_arn, + RoleSessionName="airbyte-source-s3", + ExternalId=AWS_EXTERNAL_ID, + ) + else: + role = client.assume_role( + RoleArn=self.config.role_arn, + RoleSessionName="airbyte-source-s3", + ) + + creds = role.get("Credentials", {}) + return { + "access_key": creds["AccessKeyId"], + "secret_key": creds["SecretAccessKey"], + "token": creds["SessionToken"], + "expiry_time": creds["Expiration"].isoformat(), + } + + session_credentials = RefreshableCredentials.create_from_metadata( + metadata=refresh(), + refresh_using=refresh, + method="sts-assume-role", + ) + + session = get_session() + session._credentials = session_credentials + autorefresh_session = boto3.Session(botocore_session=session) + + return autorefresh_session.client("s3", **client_kv_args) def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: """ @@ -144,7 +205,11 @@ def _page( continue for remote_file in self._handle_file(file): - if self.file_matches_globs(remote_file, globs) and remote_file.uri not in seen: + if ( + self.file_matches_globs(remote_file, globs) + and self.is_modified_after_start_date(remote_file.last_modified) + and remote_file.uri not in seen + ): seen.add(remote_file.uri) yield remote_file else: @@ -156,6 +221,12 @@ def _page( logger.info(f"Finished listing objects from S3 for prefix={prefix}. Found {total_n_keys_for_prefix} objects.") break + def is_modified_after_start_date(self, last_modified_date: Optional[datetime]) -> bool: + """Returns True if given date higher or equal than start date or something is missing""" + if not (self.config.start_date and last_modified_date): + return True + return last_modified_date >= pendulum.parse(self.config.start_date).naive() + def _handle_file(self, file): if file["Key"].endswith(".zip"): yield from self._handle_zip_file(file) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py index f8848368d763..cf4a82f8d3d0 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py @@ -15,7 +15,13 @@ class SourceTest(unittest.TestCase): def setUp(self) -> None: self._stream_reader = Mock(spec=SourceS3StreamReader) - self._source = SourceS3(self._stream_reader, Config, str(TEST_FILES_FOLDER.joinpath("catalog.json"))) + self._source = SourceS3( + self._stream_reader, + Config, + SourceS3.read_catalog(str(TEST_FILES_FOLDER.joinpath("catalog.json"))), + SourceS3.read_config(str(TEST_FILES_FOLDER.joinpath("v3_config.json"))), + None, + ) @patch("source_s3.v4.source.emit_configuration_as_airbyte_control_message") def test_given_config_is_v3_when_read_config_then_emit_new_config(self, emit_config_mock) -> None: diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 05d7f7873be1..948021fd5bb0 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -5,10 +5,10 @@ import io import logging -from datetime import datetime +from datetime import datetime, timedelta from itertools import product from typing import Any, Dict, List, Optional, Set -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec @@ -16,6 +16,7 @@ from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.stub import Stubber +from moto import mock_s3, mock_sts from pydantic import AnyUrl from source_s3.v4.config import Config from source_s3.v4.stream_reader import SourceS3StreamReader @@ -123,10 +124,51 @@ def test_get_matching_files( except Exception as exc: raise exc - stub = set_stub(reader, mocked_response, multiple_pages) - files = list(reader.get_matching_files(globs, None, logger)) - stub.deactivate() - assert set(f.uri for f in files) == expected_uris + with patch.object(SourceS3StreamReader, 's3_client', new_callable=MagicMock) as mock_s3_client: + _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages) + files = list(reader.get_matching_files(globs, None, logger)) + assert set(f.uri for f in files) == expected_uris + + +def _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages): + responses = [] + if multiple_pages and len(mocked_response) > 1: + # Split the mocked_response for pagination simulation + first_half = mocked_response[:len(mocked_response) // 2] + second_half = mocked_response[len(mocked_response) // 2:] + + responses.append({ + "IsTruncated": True, + "Contents": first_half, + "KeyCount": len(first_half), + "NextContinuationToken": "token", + }) + + responses.append({ + "IsTruncated": False, + "Contents": second_half, + "KeyCount": len(second_half), + }) + else: + responses.append({ + "IsTruncated": False, + "Contents": mocked_response, + "KeyCount": len(mocked_response), + }) + + def list_objects_v2_side_effect(Bucket, Prefix=None, ContinuationToken=None, **kwargs): + if ContinuationToken == "token": + return responses[1] + return responses[0] + + mock_s3_client.list_objects_v2 = MagicMock(side_effect=list_objects_v2_side_effect) + + +def _split_mocked_response(mocked_response, multiple_pages): + if not multiple_pages: + return mocked_response, [] + split_index = len(mocked_response) // 2 + return mocked_response[:split_index], mocked_response[split_index:] @patch("boto3.client") @@ -195,9 +237,9 @@ def test_open_file_calls_any_open_with_the_right_encoding(smart_open_mock): with reader.open_file(RemoteFile(uri="", last_modified=datetime.now()), FileReadMode.READ, encoding, logger) as fp: fp.read() - smart_open_mock.assert_called_once_with( - "s3://test/", transport_params={"client": reader.s3_client}, mode=FileReadMode.READ.value, encoding=encoding - ) + assert smart_open_mock.call_args.args == ("s3://test/",) + assert smart_open_mock.call_args.kwargs["mode"] == FileReadMode.READ.value + assert smart_open_mock.call_args.kwargs["encoding"] == encoding def test_get_s3_client_without_config_raises_exception(): @@ -217,24 +259,66 @@ def documentation_url(cls) -> AnyUrl: stream_reader.config = other_config -def set_stub(reader: SourceS3StreamReader, contents: List[Dict[str, Any]], multiple_pages: bool) -> Stubber: - s3_stub = Stubber(reader.s3_client) - split_contents_idx = int(len(contents) / 2) if multiple_pages else -1 - page1, page2 = contents[:split_contents_idx], contents[split_contents_idx:] - resp = { - "KeyCount": len(page1), - "Contents": page1, +@mock_sts +@patch("source_s3.v4.stream_reader.boto3.client") +def test_get_iam_s3_client(boto3_client_mock): + # Mock the STS client assume_role method + boto3_client_mock.return_value.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "assumed_access_key_id", + "SecretAccessKey": "assumed_secret_access_key", + "SessionToken": "assumed_session_token", + "Expiration": datetime.now(), + } } - if page2: - resp["NextContinuationToken"] = "token" - s3_stub.add_response("list_objects_v2", resp) - if page2: - s3_stub.add_response( - "list_objects_v2", - { - "KeyCount": len(page2), - "Contents": page2, - }, + + # Instantiate your stream reader and set the config + reader = SourceS3StreamReader() + reader.config = Config( + bucket="test", + role_arn="arn:aws:iam::123456789012:role/my-role", + streams=[], + endpoint=None, + ) + + # Call _get_iam_s3_client + with Stubber(reader.s3_client): + s3_client = reader._get_iam_s3_client({}) + + # Assertions to validate the s3 client + assert s3_client is not None + +@pytest.mark.parametrize( + "start_date, last_modified_date, expected_result", + ( + # True when file is new or modified after given start_date + ( + datetime.now() - timedelta(days=180), + datetime.now(), + True + ), + ( + datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + True + ), + # False when file is older than given start_date + ( + datetime.now(), + datetime.now() - timedelta(days=180), + False ) - s3_stub.activate() - return s3_stub + ) +) +def test_filter_file_by_start_date(start_date: datetime, last_modified_date: datetime, expected_result: bool) -> None: + reader = SourceS3StreamReader() + + reader.config = Config( + bucket="test", + aws_access_key_id="test", + aws_secret_access_key="test", + streams=[], + start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ") + ) + + assert expected_result == reader.is_modified_after_start_date(last_modified_date) diff --git a/airbyte-integrations/connectors/source-s3/v4_main.py b/airbyte-integrations/connectors/source-s3/v4_main.py index decb39a68c88..0dbd2e970189 100644 --- a/airbyte-integrations/connectors/source-s3/v4_main.py +++ b/airbyte-integrations/connectors/source-s3/v4_main.py @@ -15,8 +15,17 @@ def get_source(args: List[str]): catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) try: - return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) + return SourceS3( + SourceS3StreamReader(), + Config, + SourceS3.read_catalog(catalog_path) if catalog_path else None, + SourceS3.read_config(config_path) if config_path else None, + SourceS3.read_state(state_path) if state_path else None, + cursor_cls=Cursor, + ) except Exception: print( AirbyteMessage( diff --git a/airbyte-integrations/connectors/source-salesforce/README.md b/airbyte-integrations/connectors/source-salesforce/README.md index 74351bb472bb..3c68cf4b526d 100644 --- a/airbyte-integrations/connectors/source-salesforce/README.md +++ b/airbyte-integrations/connectors/source-salesforce/README.md @@ -1,118 +1,55 @@ -# Salesforce Source +# Salesforce source connector + This is the repository for the Salesforce source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/salesforce). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/salesforce). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/salesforce) +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/salesforce) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesforce/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source salesforce test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-salesforce spec +poetry run source-salesforce check --config secrets/config.json +poetry run source-salesforce discover --config secrets/config.json +poetry run source-salesforce read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-salesforce build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-salesforce:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-salesforce:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-salesforce:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-salesforce:dev . -# Running the spec command against your patched connector -docker run airbyte/source-salesforce:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-salesforce:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesforce:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-salesforce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-salesforce test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-salesforce test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/salesforce.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/salesforce.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml index c3c2133f001c..a78ff7a81fc8 100644 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml @@ -48,6 +48,8 @@ acceptance_tests: future_state: future_state_path: "integration_tests/future_state.json" timeout_seconds: 7200 + # skip incremental tests as filter condition greater than or equal is used, so last record for any stream state is duplicated + skip_comprehensive_incremental_tests: true full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py index 3b99cc407e8a..598f1cdb513b 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py @@ -10,11 +10,14 @@ import pytest import requests_mock -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams import Stream from source_salesforce.source import SourceSalesforce HERE = Path(__file__).parent +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) +_ANY_CONFIG = {} +_ANY_STATE = {} @pytest.fixture(name="input_config") @@ -33,7 +36,7 @@ def get_stream(input_config: Mapping[str, Any], stream_name: str) -> Stream: stream_cls = type("a", (object,), {"name": stream_name}) configured_stream_cls = type("b", (object,), {"stream": stream_cls(), "sync_mode": "full_refresh"}) catalog_cls = type("c", (object,), {"streams": [configured_stream_cls()]}) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.catalog = catalog_cls() return source.streams(input_config)[0] @@ -44,12 +47,12 @@ def get_any_real_stream(input_config: Mapping[str, Any]) -> Stream: def test_not_queryable_stream(caplog, input_config): stream = get_any_real_stream(input_config) - url = f"{stream.sf_api.instance_url}/services/data/{stream.sf_api.version}/jobs/query" + url = f"{stream._legacy_stream.sf_api.instance_url}/services/data/{stream._legacy_stream.sf_api.version}/jobs/query" # test non queryable BULK streams query = "Select Id, Subject from ActivityHistory" with caplog.at_level(logging.WARNING): - assert stream.create_stream_job(query, url) is None, "this stream should be skipped" + assert stream._legacy_stream.create_stream_job(query, url) is None, "this stream should be skipped" # check logs assert "is not queryable" in caplog.records[-1].message diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl index 14552380db0e..e556cda7b5a1 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl @@ -10,9 +10,9 @@ {"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph4SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv2QAE", "ProfileId": "00e4W000002LjMoQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320258} {"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph5SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv3QAE", "ProfileId": "00e4W000002LjMqQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320258} {"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph6SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv4QAE", "ProfileId": "00e4W000002LjMrQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320259} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIZQAY", "Label": "Sales", "MasterLabel": "salesforce", "NamespacePrefix": "standard", "DeveloperName": "Sales", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "The world's most popular sales force automation (SFA) solution", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": false, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785550} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIdQAI", "Label": "Service", "MasterLabel": "supportforce", "NamespacePrefix": "standard", "DeveloperName": "Service", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Manage customer service with accounts, contacts, cases, and more", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785551} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIeQAI", "Label": "Marketing", "MasterLabel": "Marketing", "NamespacePrefix": "standard", "DeveloperName": "Marketing", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Best-in-class on-demand marketing automation", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785552} +{"stream":"AppDefinition","data":{"Id":"000000000000000AAA","DurableId":"06m4W000001ldIZQAY","Label":"Sales","MasterLabel":"salesforce","NamespacePrefix":"standard","DeveloperName":"Sales","LogoUrl":"/img/salesforce-noname-logo-v2.svg","Description":"The world's most popular sales force automation (SFA) solution","UiType":"Aloha","NavType":"Standard","UtilityBar":null,"HeaderColor":"#0070D2","IsOverrideOrgTheme":false,"IsSmallFormFactorSupported":false,"IsMediumFormFactorSupported":false,"IsLargeFormFactorSupported":false,"IsNavPersonalizationDisabled":false,"IsNavAutoTempTabsDisabled":false,"IsNavTabPersistenceDisabled":false},"emitted_at":1708425402368} +{"stream":"AppDefinition","data":{"Id":"000000000000000AAA","DurableId":"06m4W000001ldIdQAI","Label":"Service","MasterLabel":"supportforce","NamespacePrefix":"standard","DeveloperName":"Service","LogoUrl":"/img/salesforce-noname-logo-v2.svg","Description":"Manage customer service with accounts, contacts, cases, and more","UiType":"Aloha","NavType":"Standard","UtilityBar":null,"HeaderColor":"#0070D2","IsOverrideOrgTheme":false,"IsSmallFormFactorSupported":false,"IsMediumFormFactorSupported":false,"IsLargeFormFactorSupported":true,"IsNavPersonalizationDisabled":false,"IsNavAutoTempTabsDisabled":false,"IsNavTabPersistenceDisabled":false},"emitted_at":1708425402369} +{"stream":"AppDefinition","data":{"Id":"000000000000000AAA","DurableId":"06m4W000001ldIeQAI","Label":"Marketing CRM Classic","MasterLabel":"Marketing","NamespacePrefix":"standard","DeveloperName":"Marketing","LogoUrl":"/img/salesforce-noname-logo-v2.svg","Description":"Track sales and marketing efforts with CRM objects.","UiType":"Aloha","NavType":"Standard","UtilityBar":null,"HeaderColor":"#0070D2","IsOverrideOrgTheme":false,"IsSmallFormFactorSupported":false,"IsMediumFormFactorSupported":false,"IsLargeFormFactorSupported":true,"IsNavPersonalizationDisabled":false,"IsNavAutoTempTabsDisabled":false,"IsNavTabPersistenceDisabled":false},"emitted_at":1708425402369} {"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJspQAF"}, "Id": "02i4W00000EkJspQAF", "ContactId": null, "AccountId": "0014W00002DkoWNQAZ", "ParentId": null, "RootAssetId": "02i4W00000EkJspQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Radish - Black, Winter, Organic", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787097} {"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJsqQAF"}, "Id": "02i4W00000EkJsqQAF", "ContactId": null, "AccountId": "0014W00002DkoW0QAJ", "ParentId": null, "RootAssetId": "02i4W00000EkJsqQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Cheese - Valancey", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787099} {"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJsrQAF"}, "Id": "02i4W00000EkJsrQAF", "ContactId": null, "AccountId": "0014W00002DkoW5QAJ", "ParentId": null, "RootAssetId": "02i4W00000EkJsrQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Truffle Cups Green", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787100} diff --git a/airbyte-integrations/connectors/source-salesforce/main.py b/airbyte-integrations/connectors/source-salesforce/main.py index 2ce8a19b8a7e..67536217f497 100644 --- a/airbyte-integrations/connectors/source-salesforce/main.py +++ b/airbyte-integrations/connectors/source-salesforce/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_salesforce import SourceSalesforce +from source_salesforce.run import run if __name__ == "__main__": - source = SourceSalesforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-salesforce/metadata.yaml b/airbyte-integrations/connectors/source-salesforce/metadata.yaml index aae9e5f02089..ae18dcd63d4f 100644 --- a/airbyte-integrations/connectors/source-salesforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesforce/metadata.yaml @@ -6,17 +6,21 @@ data: hosts: - "*.salesforce.com" connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: b117307c-14b6-41aa-9422-947e34922962 - dockerImageTag: 2.1.6 + dockerImageTag: 2.3.2 dockerRepository: airbyte/source-salesforce documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce githubIssueLabel: source-salesforce icon: salesforce.svg license: ELv2 name: Salesforce + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-salesforce registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-salesforce/poetry.lock b/airbyte-integrations/connectors/source-salesforce/poetry.lock new file mode 100644 index 000000000000..37eb11ff1200 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/poetry.lock @@ -0,0 +1,1200 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.63.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.63.2.tar.gz", hash = "sha256:b2edc160f560352a816f3a266b5dfa6dfe37868add1e3a0a2628eb19ba771ed1"}, + {file = "airbyte_cdk-0.63.2-py3-none-any.whl", hash = "sha256:8698cb94514f35577123520954503cb2da407423af109dffd03644ba8b0093cd"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.2.0" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "pytest-timeout" +version = "2.2.0" +description = "pytest plugin to abort hanging tests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-timeout-2.2.0.tar.gz", hash = "sha256:3b0b95dabf3cb50bac9ef5ca912fa0cfc286526af17afc806824df20c2f72c90"}, + {file = "pytest_timeout-2.2.0-py3-none-any.whl", hash = "sha256:bde531e096466f49398a59f2dde76fa78429a09a12411466f88a07213e220de2"}, +] + +[package.dependencies] +pytest = ">=5.0.0" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "2e04a4463d839afab2cd36da228d97580143a8d4ef83c3b80fcb4cb78e836d2b" diff --git a/airbyte-integrations/connectors/source-salesforce/pyproject.toml b/airbyte-integrations/connectors/source-salesforce/pyproject.toml new file mode 100644 index 000000000000..f81862b62b1c --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.3.2" +name = "source-salesforce" +description = "Source implementation for Salesforce." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/salesforce" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_salesforce" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +pandas = "==2.2.0" +airbyte-cdk = "^0.63.2" + +[tool.poetry.scripts] +source-salesforce = "source_salesforce.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.6" +pytest = "^6.1" +pytest-timeout = "^2.2.0" +requests-mock = "^1.9.3" +freezegun = "^1.4.0" diff --git a/airbyte-integrations/connectors/source-salesforce/requirements.txt b/airbyte-integrations/connectors/source-salesforce/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py deleted file mode 100644 index f2789cdc2e87..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk==0.54.0", "pandas"] # Pinning airbyte-cdk until we update source-salesforce to use the ConcurrentSource - -TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock~=3.6", "requests-mock~=1.9.3", "pytest-timeout"] - -setup( - name="source_salesforce", - description="Source implementation for Salesforce.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index e88c4db1ff0b..eb0eed9ef70d 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -51,7 +51,6 @@ "AppTabMember", "CollaborationGroupRecord", "ColorDefinition", - "ContentDocumentLink", "ContentFolderItem", "ContentFolderMember", "DataStatistics", @@ -129,6 +128,19 @@ "UserRecordAccess", ] +PARENT_SALESFORCE_OBJECTS = { + # parent_name - name of parent stream + # field - in each parent record, which is needed for stream slice + # schema_minimal - required for getting proper class name full_refresh/incremental, rest/bulk for parent stream + "ContentDocumentLink": { + "parent_name": "ContentDocument", + "field": "Id", + "schema_minimal": { + "properties": {"Id": {"type": ["string", "null"]}, "SystemModstamp": {"type": ["string", "null"], "format": "date-time"}} + }, + } +} + # The following objects are not supported by the Bulk API. Listed objects are version specific. UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = [ "AcceptedEventRelation", @@ -184,6 +196,7 @@ UNSUPPORTED_FILTERING_STREAMS = [ "ApiEvent", "BulkApiResultEventStore", + "ContentDocumentLink", "EmbeddedServiceDetail", "EmbeddedServiceLabel", "FormulaFunction", diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py new file mode 100644 index 000000000000..07c8c7ce83ab --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_salesforce import SourceSalesforce + + +def _get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + try: + return SourceSalesforce( + SourceSalesforce.read_catalog(catalog_path) if catalog_path else None, + SourceSalesforce.read_config(config_path) if config_path else None, + SourceSalesforce.read_state(state_path) if state_path else None, + ) + except Exception as error: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = _get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index 4d276d443692..cb2293731d01 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -6,25 +6,40 @@ from datetime import datetime from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union +import pendulum import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.logger import AirbyteLogFormatter from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Level, SyncMode -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade -from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor +from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, NoopCursor from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.sources.utils.schema_helpers import InternalConfig from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import FailureType from dateutil.relativedelta import relativedelta +from pendulum.parsing.exceptions import ParserError from requests import codes, exceptions # type: ignore[import] -from .api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce -from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, Describe, IncrementalRestSalesforceStream, RestSalesforceStream +from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce +from .streams import ( + BulkIncrementalSalesforceStream, + BulkSalesforceStream, + BulkSalesforceSubStream, + Describe, + IncrementalRestSalesforceStream, + RestSalesforceStream, + RestSalesforceSubStream, +) +_DEFAULT_CONCURRENCY = 10 +_MAX_CONCURRENCY = 10 logger = logging.getLogger("airbyte") @@ -32,16 +47,25 @@ class AirbyteStopSync(AirbyteTracedException): pass -class SourceSalesforce(AbstractSource): +class SourceSalesforce(ConcurrentSourceAdapter): DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" START_DATE_OFFSET_IN_YEARS = 2 MAX_WORKERS = 5 - + stop_sync_on_stream_failure = True message_repository = InMemoryMessageRepository(Level(AirbyteLogFormatter.level_mapping[logger.level])) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.catalog = None + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState], **kwargs): + if config: + concurrency_level = min(config.get("num_workers", _DEFAULT_CONCURRENCY), _MAX_CONCURRENCY) + else: + concurrency_level = _DEFAULT_CONCURRENCY + logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") + concurrent_source = ConcurrentSource.create( + concurrency_level, concurrency_level // 2, logger, self._slice_logger, self.message_repository + ) + super().__init__(concurrent_source) + self.catalog = catalog + self.state = state @staticmethod def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: @@ -49,7 +73,24 @@ def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: sf.login() return sf + @staticmethod + def _validate_stream_slice_step(stream_slice_step: str): + if stream_slice_step: + try: + duration = pendulum.parse(stream_slice_step) + if not isinstance(duration, pendulum.Duration): + message = "Stream slice step Interval should be provided in ISO 8601 format." + elif duration < pendulum.Duration(seconds=1): + message = "Stream slice step Interval is too small. It should be no less than 1 second. Please set higher value and try again." + else: + return + raise ParserError(message) + except ParserError as e: + internal_message = "Incorrect stream slice step" + raise AirbyteTracedException(failure_type=FailureType.config_error, internal_message=internal_message, message=e.args[0]) + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[str]]: + self._validate_stream_slice_step(config.get("stream_slice_step")) try: salesforce = self._get_sf_object(config) salesforce.describe() @@ -68,8 +109,10 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None @classmethod - def _get_api_type(cls, stream_name: str, properties: Mapping[str, Any], force_use_bulk_api: bool) -> str: + def _get_api_type(cls, stream_name: str, json_schema: Mapping[str, Any], force_use_bulk_api: bool) -> str: + """Get proper API type: rest or bulk""" # Salesforce BULK API currently does not support loading fields with data type base64 and compound data + properties = json_schema.get("properties", {}) properties_not_supported_by_bulk = { key: value for key, value in properties.items() if value.get("format") == "base64" or "object" in value["type"] } @@ -86,6 +129,49 @@ def _get_api_type(cls, stream_name: str, properties: Mapping[str, Any], force_us return "rest" return "bulk" + @classmethod + def _get_stream_type(cls, stream_name: str, api_type: str): + """Get proper stream class: full_refresh, incremental or substream + + SubStreams (like ContentDocumentLink) do not support incremental sync because of query restrictions, look here: + https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_contentdocumentlink.htm + """ + parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") + if api_type == "rest": + full_refresh = RestSalesforceSubStream if parent_name else RestSalesforceStream + incremental = IncrementalRestSalesforceStream + elif api_type == "bulk": + full_refresh = BulkSalesforceSubStream if parent_name else BulkSalesforceStream + incremental = BulkIncrementalSalesforceStream + else: + raise Exception(f"Stream {stream_name} cannot be processed by REST or BULK API.") + return full_refresh, incremental + + @classmethod + def prepare_stream(cls, stream_name: str, json_schema, sobject_options, sf_object, authenticator, config): + """Choose proper stream class: syncMode(full_refresh/incremental), API type(Rest/Bulk), SubStream""" + pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) + stream_kwargs = { + "stream_name": stream_name, + "schema": json_schema, + "pk": pk, + "sobject_options": sobject_options, + "sf_api": sf_object, + "authenticator": authenticator, + "start_date": config.get("start_date"), + } + + api_type = cls._get_api_type(stream_name, json_schema, config.get("force_use_bulk_api", False)) + full_refresh, incremental = cls._get_stream_type(stream_name, api_type) + if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: + stream_class = incremental + stream_kwargs["replication_key"] = replication_key + stream_kwargs["stream_slice_step"] = config.get("stream_slice_step", "P30D") + else: + stream_class = full_refresh + + return stream_class, stream_kwargs + @classmethod def generate_streams( cls, @@ -93,57 +179,76 @@ def generate_streams( stream_objects: Mapping[str, Any], sf_object: Salesforce, ) -> List[Stream]: - """ "Generates a list of stream by their names. It can be used for different tests too""" + """Generates a list of stream by their names. It can be used for different tests too""" authenticator = TokenAuthenticator(sf_object.access_token) - stream_properties = sf_object.generate_schemas(stream_objects) + schemas = sf_object.generate_schemas(stream_objects) + default_args = [sf_object, authenticator, config] streams = [] for stream_name, sobject_options in stream_objects.items(): - streams_kwargs = {"sobject_options": sobject_options} - selected_properties = stream_properties.get(stream_name, {}).get("properties", {}) - - api_type = cls._get_api_type(stream_name, selected_properties, config.get("force_use_bulk_api", False)) - if api_type == "rest": - full_refresh, incremental = RestSalesforceStream, IncrementalRestSalesforceStream - elif api_type == "bulk": - full_refresh, incremental = BulkSalesforceStream, BulkIncrementalSalesforceStream - else: - raise Exception(f"Stream {stream_name} cannot be processed by REST or BULK API.") - - json_schema = stream_properties.get(stream_name, {}) - pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) - streams_kwargs.update(dict(sf_api=sf_object, pk=pk, stream_name=stream_name, schema=json_schema, authenticator=authenticator)) - if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: - start_date = config.get( - "start_date", (datetime.now() - relativedelta(years=cls.START_DATE_OFFSET_IN_YEARS)).strftime(cls.DATETIME_FORMAT) - ) - stream = incremental(**streams_kwargs, replication_key=replication_key, start_date=start_date) - else: - stream = full_refresh(**streams_kwargs) + json_schema = schemas.get(stream_name, {}) + + stream_class, kwargs = cls.prepare_stream(stream_name, json_schema, sobject_options, *default_args) + + parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") + if parent_name: + # get minimal schema required for getting proper class name full_refresh/incremental, rest/bulk + parent_schema = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("schema_minimal") + parent_class, parent_kwargs = cls.prepare_stream(parent_name, parent_schema, sobject_options, *default_args) + kwargs["parent"] = parent_class(**parent_kwargs) + + stream = stream_class(**kwargs) + + api_type = cls._get_api_type(stream_name, json_schema, config.get("force_use_bulk_api", False)) if api_type == "rest" and not stream.primary_key and stream.too_many_properties: logger.warning( - f"Can not instantiate stream {stream_name}. " - f"It is not supported by the BULK API and can not be implemented via REST because the number of its properties " - f"exceeds the limit and it lacks a primary key." + f"Can not instantiate stream {stream_name}. It is not supported by the BULK API and can not be " + "implemented via REST because the number of its properties exceeds the limit and it lacks a primary key." ) continue streams.append(stream) return streams def streams(self, config: Mapping[str, Any]) -> List[Stream]: + if not config.get("start_date"): + config["start_date"] = (datetime.now() - relativedelta(years=self.START_DATE_OFFSET_IN_YEARS)).strftime(self.DATETIME_FORMAT) sf = self._get_sf_object(config) stream_objects = sf.get_validated_streams(config=config, catalog=self.catalog) streams = self.generate_streams(config, stream_objects, sf) streams.append(Describe(sf_api=sf, catalog=self.catalog)) - # TODO: incorporate state & ConcurrentCursor when we support incremental + state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in streams}, state=self.state) + configured_streams = [] + for stream in streams: sync_mode = self._get_sync_mode_from_catalog(stream) if sync_mode == SyncMode.full_refresh: - configured_streams.append(StreamFacade.create_from_stream(stream, self, logger, self.MAX_WORKERS, None, NoopCursor())) + cursor = NoopCursor() + state = None else: - configured_streams.append(stream) + cursor_field_key = stream.cursor_field or "" + if not isinstance(cursor_field_key, str): + raise AssertionError(f"A string cursor field key is required, but got {cursor_field_key}.") + cursor_field = CursorField(cursor_field_key) + legacy_state = state_manager.get_stream_state(stream.name, stream.namespace) + cursor = ConcurrentCursor( + stream.name, + stream.namespace, + legacy_state, + self.message_repository, + state_manager, + stream.state_converter, + cursor_field, + self._get_slice_boundary_fields(stream, state_manager), + config["start_date"], + ) + state = cursor.state + + configured_streams.append(StreamFacade.create_from_stream(stream, self, logger, state, cursor)) return configured_streams + def _get_slice_boundary_fields(self, stream: Stream, state_manager: ConnectorStateManager) -> Optional[Tuple[str, str]]: + return ("start_date", "end_date") + def _get_sync_mode_from_catalog(self, stream: Stream) -> Optional[SyncMode]: if self.catalog: for catalog_stream in self.catalog.streams: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml index 642e180c078b..1882e4e66c4a 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml @@ -60,9 +60,21 @@ connectionSpecification: description: Toggle to use Bulk API (this might cause empty fields for some streams) default: false order: 6 + stream_slice_step: + title: Stream Slice Step for Incremental sync + type: string + description: The size of the time window (ISO8601 duration) to slice requests. + default: P30D + order: 7 + examples: + - PT12H + - P7D + - P30D + - P1M + - P1Y streams_criteria: type: array - order: 7 + order: 8 items: type: object required: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index ba30fe856263..da5064f8b148 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -18,8 +18,9 @@ import requests # type: ignore[import] from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import IsoMillisConcurrentStreamStateConverter from airbyte_cdk.sources.streams.core import Stream, StreamData -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException from numpy import nan @@ -27,7 +28,7 @@ from requests import codes, exceptions from requests.models import PreparedRequest -from .api import UNSUPPORTED_FILTERING_STREAMS, Salesforce +from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .availability_strategy import SalesforceAvailabilityStrategy from .exceptions import SalesforceException, TmpFileIOError from .rate_limiting import default_backoff_handler @@ -37,15 +38,24 @@ csv.field_size_limit(CSV_FIELD_SIZE_LIMIT) DEFAULT_ENCODING = "utf-8" +LOOKBACK_SECONDS = 600 # based on https://trailhead.salesforce.com/trailblazer-community/feed/0D54V00007T48TASAZ class SalesforceStream(HttpStream, ABC): + state_converter = IsoMillisConcurrentStreamStateConverter() page_size = 2000 transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) encoding = DEFAULT_ENCODING def __init__( - self, sf_api: Salesforce, pk: str, stream_name: str, sobject_options: Mapping[str, Any] = None, schema: dict = None, **kwargs + self, + sf_api: Salesforce, + pk: str, + stream_name: str, + sobject_options: Mapping[str, Any] = None, + schema: dict = None, + start_date=None, + **kwargs, ): super().__init__(**kwargs) self.sf_api = sf_api @@ -53,6 +63,14 @@ def __init__( self.stream_name = stream_name self.schema: Mapping[str, Any] = schema # type: ignore[assignment] self.sobject_options = sobject_options + self.start_date = self.format_start_date(start_date) + + @staticmethod + def format_start_date(start_date: Optional[str]) -> Optional[str]: + """Transform the format `2021-07-25` into the format `2021-07-25T00:00:00Z`""" + if start_date: + return pendulum.parse(start_date).strftime("%Y-%m-%dT%H:%M:%SZ") # type: ignore[attr-defined,no-any-return] + return None @property def max_properties_length(self) -> int: @@ -93,6 +111,16 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]: return f"After {self.max_retries} retries the connector has failed with a network error. It looks like Salesforce API experienced temporary instability, please try again later." return super().get_error_display_message(exception) + def get_start_date_from_state(self, stream_state: Mapping[str, Any] = None) -> pendulum.DateTime: + if self.state_converter.is_state_message_compatible(stream_state): + # stream_state is in the concurrent format + if stream_state.get("slices", []): + return stream_state["slices"][0]["end"] + elif stream_state and not self.state_converter.is_state_message_compatible(stream_state): + # stream_state has not been converted to the concurrent format; this is not expected + return pendulum.parse(stream_state.get(self.cursor_field), tz="UTC") + return pendulum.parse(self.start_date, tz="UTC") + class PropertyChunk: """ @@ -112,6 +140,8 @@ def __init__(self, properties: Mapping[str, Any]): class RestSalesforceStream(SalesforceStream): + state_converter = IsoMillisConcurrentStreamStateConverter() + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) assert self.primary_key or not self.too_many_properties @@ -141,14 +171,18 @@ def request_params( Salesforce SOQL Query: https://developer.salesforce.com/docs/atlas.en-us.232.0.api_rest.meta/api_rest/dome_queryall.htm """ if next_page_token: - """ - If `next_page_token` is set, subsequent requests use `nextRecordsUrl`, and do not include any parameters. - """ + # If `next_page_token` is set, subsequent requests use `nextRecordsUrl`, and do not include any parameters. return {} property_chunk = property_chunk or {} query = f"SELECT {','.join(property_chunk.keys())} FROM {self.name} " + if self.name in PARENT_SALESFORCE_OBJECTS: + # add where clause: " WHERE ContentDocumentId IN ('06905000000NMXXXXX', ...)" + parent_field = PARENT_SALESFORCE_OBJECTS[self.name]["field"] + parent_ids = [f"'{parent_record[parent_field]}'" for parent_record in stream_slice["parents"]] + query += f" WHERE ContentDocumentId IN ({','.join(parent_ids)})" + if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: query += f"ORDER BY {self.primary_key} ASC" @@ -282,6 +316,31 @@ def _fetch_next_page_for_chunk( return request, response +class BatchedSubStream(HttpSubStream): + state_converter = IsoMillisConcurrentStreamStateConverter() + SLICE_BATCH_SIZE = 200 + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + """Instead of yielding one parent record at a time, make stream slice contain a batch of parent records. + + It allows to get records by one requests (instead of only one). + """ + batched_slice = [] + for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state): + if len(batched_slice) == self.SLICE_BATCH_SIZE: + yield {"parents": batched_slice} + batched_slice = [] + batched_slice.append(stream_slice["parent"]) + if batched_slice: + yield {"parents": batched_slice} + + +class RestSalesforceSubStream(BatchedSubStream, RestSalesforceStream): + pass + + class BulkSalesforceStream(SalesforceStream): DEFAULT_WAIT_TIMEOUT_SECONDS = 86400 # 24-hour bulk job running time MAX_CHECK_INTERVAL_SECONDS = 2.0 @@ -542,6 +601,12 @@ def request_params( if next_page_token: query += next_page_token["next_token"] + if self.name in PARENT_SALESFORCE_OBJECTS: + # add where clause: " WHERE ContentDocumentId IN ('06905000000NMXXXXX', '06905000000Mxp7XXX', ...)" + parent_field = PARENT_SALESFORCE_OBJECTS[self.name]["field"] + parent_ids = [f"'{parent_record[parent_field]}'" for parent_record in stream_slice["parents"]] + query += f" WHERE ContentDocumentId IN ({','.join(parent_ids)})" + return {"q": query} def read_records( @@ -605,6 +670,10 @@ def get_standard_instance(self) -> SalesforceStream: return new_cls(**stream_kwargs) +class BulkSalesforceSubStream(BatchedSubStream, BulkSalesforceStream): + pass + + @BulkSalesforceStream.transformer.registerCustomTransform def transform_empty_string_to_none(instance: Any, schema: Any): """ @@ -619,35 +688,34 @@ def transform_empty_string_to_none(instance: Any, schema: Any): class IncrementalRestSalesforceStream(RestSalesforceStream, ABC): state_checkpoint_interval = 500 - STREAM_SLICE_STEP = 30 _slice = None - def __init__(self, replication_key: str, start_date: Optional[str], **kwargs): + def __init__(self, replication_key: str, stream_slice_step: str = "P30D", **kwargs): super().__init__(**kwargs) self.replication_key = replication_key - self.start_date = self.format_start_date(start_date) - - @staticmethod - def format_start_date(start_date: Optional[str]) -> Optional[str]: - """Transform the format `2021-07-25` into the format `2021-07-25T00:00:00Z`""" - if start_date: - return pendulum.parse(start_date).strftime("%Y-%m-%dT%H:%M:%SZ") # type: ignore[attr-defined,no-any-return] - return None + self._stream_slice_step = stream_slice_step def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - start, end = (None, None) now = pendulum.now(tz="UTC") - initial_date = pendulum.parse((stream_state or {}).get(self.cursor_field, self.start_date), tz="UTC") + assert LOOKBACK_SECONDS is not None and LOOKBACK_SECONDS >= 0 + + initial_date = self.get_start_date_from_state(stream_state) - pendulum.Duration(seconds=LOOKBACK_SECONDS) + slice_start = initial_date + while slice_start < now: + slice_end = slice_start + self.stream_slice_step + self._slice = { + "start_date": slice_start.isoformat(timespec="milliseconds"), + "end_date": min(slice_end, now).isoformat(timespec="milliseconds"), + } + yield self._slice - slice_number = 1 - while not end == now: - start = initial_date.add(days=(slice_number - 1) * self.STREAM_SLICE_STEP) - end = min(now, initial_date.add(days=slice_number * self.STREAM_SLICE_STEP)) - self._slice = {"start_date": start.isoformat(timespec="milliseconds"), "end_date": end.isoformat(timespec="milliseconds")} - yield {"start_date": start.isoformat(timespec="milliseconds"), "end_date": end.isoformat(timespec="milliseconds")} - slice_number = slice_number + 1 + slice_start += self.stream_slice_step + + @property + def stream_slice_step(self) -> pendulum.Duration: + return pendulum.parse(self._stream_slice_step) def request_params( self, @@ -723,6 +791,7 @@ def request_params( class Describe(Stream): + state_converter = IsoMillisConcurrentStreamStateConverter() """ Stream of sObjects' (Salesforce Objects) describe: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_sobject_describe.htm diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 81c6b2faa3ec..6b31ad9d7d9d 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -2,14 +2,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import csv import io import logging import re -from datetime import datetime +from datetime import datetime, timedelta from typing import List -from unittest.mock import Mock +from unittest.mock import Mock, patch import freezegun import pendulum @@ -18,6 +17,7 @@ from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode, Type from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.utils import AirbyteTracedException from conftest import encoding_symbols_parameters, generate_stream from requests.exceptions import HTTPError @@ -28,12 +28,41 @@ CSV_FIELD_SIZE_LIMIT, BulkIncrementalSalesforceStream, BulkSalesforceStream, + BulkSalesforceSubStream, Describe, IncrementalRestSalesforceStream, RestSalesforceStream, SalesforceStream, ) +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) +_ANY_CONFIG = {} +_ANY_STATE = None + + +@pytest.mark.parametrize( + "stream_slice_step, expected_error_message", + [ + ("2023", "Stream slice step Interval should be provided in ISO 8601 format."), + ("PT0.1S", "Stream slice step Interval is too small. It should be no less than 1 second."), + ("PT1D", "Unable to parse string"), + ("P221S", "Unable to parse string"), + ], + ids=[ + "incorrect_ISO_8601_format", + "too_small_duration_provided", + "incorrect_date_format", + "incorrect_time_format", + ], +) +def test_stream_slice_step_validation(stream_slice_step: str, expected_error_message): + _ANY_CONFIG.update({"stream_slice_step": stream_slice_step}) + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) + logger = logging.getLogger("airbyte") + with pytest.raises(AirbyteTracedException) as e: + source.check_connection(logger, _ANY_CONFIG) + assert expected_error_message in e.value.message + @pytest.mark.parametrize( "login_status_code, login_json_resp, expected_error_msg, is_config_error", @@ -61,7 +90,7 @@ def test_login_authentication_error_handler( stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg, is_config_error ): - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) logger = logging.getLogger("airbyte") requests_mock.register_uri( "POST", "https://login.salesforce.com/services/oauth2/token", json=login_json_resp, status_code=login_status_code @@ -228,9 +257,7 @@ def test_bulk_sync_failed_retry(stream_config, stream_api): "start_date_provided,stream_name,expected_start_date", [ (True, "Account", "2010-01-18T21:18:20Z"), - (False, "Account", None), (True, "ActiveFeatureLicenseMetric", "2010-01-18T21:18:20Z"), - (False, "ActiveFeatureLicenseMetric", None), ], ) def test_stream_start_date( @@ -340,7 +367,7 @@ def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type_he def test_check_connection_rate_limit( stream_config, login_status_code, login_json_resp, discovery_status_code, discovery_resp_json, expected_error_msg ): - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) logger = logging.getLogger("airbyte") with requests_mock.Mocker() as m: @@ -377,10 +404,9 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): stream_1.page_size = 6 stream_1.state_checkpoint_interval = 5 - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.streams = Mock() source.streams.return_value = streams - logger = logging.getLogger("airbyte") json_response = [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "TotalRequests Limit exceeded."}] with requests_mock.Mocker() as m: @@ -404,17 +430,16 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): m.register_uri("DELETE", stream.path() + f"/{job_id}") m.register_uri("POST", stream.path(), creation_responses) - - result = [i for i in source.read(logger=logger, config=stream_config, catalog=bulk_catalog, state=state)] + result = read(source=source, config=stream_config, catalog=bulk_catalog, state=state) assert stream_1.request_params.called assert ( not stream_2.request_params.called ), "The second stream should not be executed, because the first stream finished with Rate Limit." - records = [item for item in result if item.type == Type.RECORD] + records = result.records assert len(records) == 6 # stream page size: 6 - state_record = [item for item in result if item.type == Type.STATE][0] + state_record = result.state_messages[0] assert state_record.state.data["Account"]["LastModifiedDate"] == "2021-10-05T00:00:00+00:00" # state checkpoint interval is 5. @@ -431,13 +456,12 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): stream_2: IncrementalRestSalesforceStream = generate_stream("AcceptedEventRelation", stream_config, stream_api) stream_1.state_checkpoint_interval = 3 + streams = [stream_1, stream_2] configure_request_params_mock(stream_1, stream_2) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.streams = Mock() - source.streams.return_value = [stream_1, stream_2] - - logger = logging.getLogger("airbyte") + source.streams.return_value = streams next_page_url = "/services/data/v57.0/query/012345" response_1 = { @@ -473,17 +497,17 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): m.register_uri("GET", stream_1.path(), json=response_1, status_code=200) m.register_uri("GET", next_page_url, json=response_2, status_code=403) - result = [i for i in source.read(logger=logger, config=stream_config, catalog=rest_catalog, state=state)] + result = read(source=source, config=stream_config, catalog=rest_catalog, state=state) assert stream_1.request_params.called assert ( not stream_2.request_params.called ), "The second stream should not be executed, because the first stream finished with Rate Limit." - records = [item for item in result if item.type == Type.RECORD] + records = result.records assert len(records) == 5 - state_record = [item for item in result if item.type == Type.STATE][0] + state_record = result.state_messages[0] assert state_record.state.data["KnowledgeArticle"]["LastModifiedDate"] == "2021-11-17T00:00:00+00:00" @@ -618,7 +642,7 @@ def test_forwarding_sobject_options(stream_config, stream_names, catalog_stream_ ], }, ) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.catalog = catalog streams = source.streams(config=stream_config) expected_names = catalog_stream_names if catalog else stream_names @@ -633,28 +657,6 @@ def test_forwarding_sobject_options(stream_config, stream_names, catalog_stream_ return -@pytest.mark.parametrize( - "stream_names,catalog_stream_names,", - ( - ( - ["stream_1", "stream_2", "Describe"], - None, - ), - ( - ["stream_1", "stream_2"], - ["stream_1", "stream_2", "Describe"], - ), - ( - ["stream_1", "stream_2", "stream_3", "Describe"], - ["stream_1", "Describe"], - ), - ), -) -def test_unspecified_and_incremental_streams_are_not_concurrent(stream_config, stream_names, catalog_stream_names) -> None: - for stream in _get_streams(stream_config, stream_names, catalog_stream_names, SyncMode.incremental): - assert isinstance(stream, (SalesforceStream, Describe)) - - @pytest.mark.parametrize( "stream_names,catalog_stream_names,", ( @@ -718,7 +720,7 @@ def _get_streams(stream_config, stream_names, catalog_stream_names, sync_type) - ], }, ) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.catalog = catalog return source.streams(config=stream_config) @@ -881,21 +883,37 @@ def test_bulk_stream_error_on_wait_for_job(requests_mock, stream_config, stream_ @freezegun.freeze_time("2023-01-01") -def test_bulk_stream_slices(stream_config_date_format, stream_api): +@pytest.mark.parametrize( + "lookback, stream_slice_step, expected_len_stream_slices, expect_error", + [(None, "P30D", 0, True), (0, "P30D", 158, False), (10, "P1D", 4732, False), (10, "PT12H", 9463, False), (-1, "P30D", 0, True)], + ids=["lookback-is-none", "lookback-is-0-step-30D", "lookback-is-valid-step-1D", "lookback-is-valid-step-12H", "lookback-is-negative"], +) +def test_bulk_stream_slices( + stream_config_date_format, stream_api, lookback, expect_error, stream_slice_step: str, expected_len_stream_slices: int +): + stream_config_date_format["stream_slice_step"] = stream_slice_step stream: BulkIncrementalSalesforceStream = generate_stream("FakeBulkStream", stream_config_date_format, stream_api) - stream_slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) - expected_slices = [] - today = pendulum.today(tz="UTC") - start_date = pendulum.parse(stream.start_date, tz="UTC") - while start_date < today: - expected_slices.append( - { - "start_date": start_date.isoformat(timespec="milliseconds"), - "end_date": min(today, start_date.add(days=stream.STREAM_SLICE_STEP)).isoformat(timespec="milliseconds"), - } - ) - start_date = start_date.add(days=stream.STREAM_SLICE_STEP) - assert expected_slices == stream_slices + with patch("source_salesforce.streams.LOOKBACK_SECONDS", lookback): + if expect_error: + with pytest.raises(AssertionError): + list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) + else: + stream_slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) + + expected_slices = [] + today = pendulum.today(tz="UTC") + start_date = pendulum.parse(stream.start_date, tz="UTC") - timedelta(seconds=lookback) + while start_date < today: + end_date = start_date + stream.stream_slice_step + expected_slices.append( + { + "start_date": start_date.isoformat(timespec="milliseconds"), + "end_date": min(today, end_date).isoformat(timespec="milliseconds"), + } + ) + start_date += stream.stream_slice_step + assert expected_slices == stream_slices + assert len(stream_slices) == expected_len_stream_slices @freezegun.freeze_time("2023-04-01") @@ -904,7 +922,7 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api stream_config_date_format.update({"start_date": "2023-01-01"}) stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) source.streams = Mock() source.streams.return_value = [stream] @@ -934,7 +952,8 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api logger = logging.getLogger("airbyte") state = {"Account": {"LastModifiedDate": "2023-01-01T10:10:10.000Z"}} bulk_catalog.streams.pop(1) - result = [i for i in source.read(logger=logger, config=stream_config_date_format, catalog=bulk_catalog, state=state)] + with patch("source_salesforce.streams.LOOKBACK_SECONDS", 0): + result = [i for i in source.read(logger=logger, config=stream_config_date_format, catalog=bulk_catalog, state=state)] actual_state_values = [item.state.data.get("Account").get(stream.cursor_field) for item in result if item.type == Type.STATE] # assert request params @@ -955,3 +974,53 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api # if connector meets record with cursor `2023-04-01` out of current slice range 2023-01-31 <> 2023-03-02, we ignore all other values and set state to slice end_date expected_state_values = ["2023-01-15T00:00:00+00:00", "2023-03-02T10:10:10+00:00", "2023-04-01T00:00:00+00:00"] assert actual_state_values == expected_state_values + + +def test_request_params_incremental(stream_config_date_format, stream_api): + stream = generate_stream("ContentDocument", stream_config_date_format, stream_api) + params = stream.request_params(stream_state={}, stream_slice={"start_date": "2020", "end_date": "2021"}) + + assert params == {"q": "SELECT LastModifiedDate, Id FROM ContentDocument WHERE LastModifiedDate >= 2020 AND LastModifiedDate < 2021"} + + +def test_request_params_substream(stream_config_date_format, stream_api): + stream = generate_stream("ContentDocumentLink", stream_config_date_format, stream_api) + params = stream.request_params(stream_state={}, stream_slice={"parents": [{"Id": 1}, {"Id": 2}]}) + + assert params == {"q": "SELECT LastModifiedDate, Id FROM ContentDocumentLink WHERE ContentDocumentId IN ('1','2')"} + + +@freezegun.freeze_time("2023-03-20") +def test_stream_slices_for_substream(stream_config, stream_api, requests_mock): + """Test BulkSalesforceSubStream for ContentDocumentLink (+ parent ContentDocument) + + ContentDocument return 1 record for each slice request. + Given start/end date leads to 3 date slice for ContentDocument, thus 3 total records + + ContentDocumentLink + It means that ContentDocumentLink should have 2 slices, with 2 and 1 records in each + """ + stream_config["start_date"] = "2023-01-01" + stream: BulkSalesforceSubStream = generate_stream("ContentDocumentLink", stream_config, stream_api) + stream.SLICE_BATCH_SIZE = 2 # each ContentDocumentLink should contain 2 records from parent ContentDocument stream + + job_id = "fake_job" + requests_mock.register_uri("POST", stream.path(), json={"id": job_id}) + requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) + requests_mock.register_uri( + "GET", + stream.path() + f"/{job_id}/results", + [{"text": "Field1,LastModifiedDate,ID\ntest,2021-11-16,123", "headers": {"Sforce-Locator": "null"}}], + ) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id}") + + stream_slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) + assert stream_slices == [ + { + "parents": [ + {"Field1": "test", "ID": "123", "LastModifiedDate": "2021-11-16"}, + {"Field1": "test", "ID": "123", "LastModifiedDate": "2021-11-16"}, + ] + }, + {"parents": [{"Field1": "test", "ID": "123", "LastModifiedDate": "2021-11-16"}]}, + ] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index eeacdd2235d2..48defd5e5ff2 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -3,10 +3,13 @@ # import json +from typing import List from unittest.mock import Mock import pytest from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStateMessage from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce @@ -32,9 +35,13 @@ def rest_catalog(): @pytest.fixture(scope="module") -def state(): - state = {"Account": {"LastModifiedDate": "2021-10-01T21:18:20.000Z"}, "Asset": {"SystemModstamp": "2021-10-02T05:08:29.000Z"}} - return state +def state() -> List[AirbyteStateMessage]: + return ( + StateBuilder() + .with_stream_state("Account", {"LastModifiedDate": "2021-10-01T21:18:20.000Z"}) + .with_stream_state("Asset", {"SystemModstamp": "2021-10-02T05:08:29.000Z"}) + .build() + ) @pytest.fixture(scope="module") diff --git a/airbyte-integrations/connectors/source-salesloft/main.py b/airbyte-integrations/connectors/source-salesloft/main.py index 8ee79923b913..6ae4308e9ab6 100644 --- a/airbyte-integrations/connectors/source-salesloft/main.py +++ b/airbyte-integrations/connectors/source-salesloft/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_salesloft import SourceSalesloft +from source_salesloft.run import run if __name__ == "__main__": - source = SourceSalesloft() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-salesloft/metadata.yaml b/airbyte-integrations/connectors/source-salesloft/metadata.yaml index 7fdf8beebb6b..a5c2cc049e9e 100644 --- a/airbyte-integrations/connectors/source-salesloft/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesloft/metadata.yaml @@ -15,6 +15,10 @@ data: icon: salesloft.svg license: MIT name: SalesLoft + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-salesloft registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-salesloft/setup.py b/airbyte-integrations/connectors/source-salesloft/setup.py index bd9fc0a48179..f272bb4432de 100644 --- a/airbyte-integrations/connectors/source-salesloft/setup.py +++ b/airbyte-integrations/connectors/source-salesloft/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-salesloft=source_salesloft.run:run", + ], + }, name="source_salesloft", description="Source implementation for Salesloft.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py b/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py new file mode 100644 index 000000000000..e68c3785893c --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_salesloft import SourceSalesloft + + +def run(): + source = SourceSalesloft() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/main.py b/airbyte-integrations/connectors/source-sap-fieldglass/main.py index c9273ef8ae47..8d8230917289 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/main.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sap_fieldglass import SourceSapFieldglass +from source_sap_fieldglass.run import run if __name__ == "__main__": - source = SourceSapFieldglass() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml index 2fdfa8eeeba4..6362a99fd348 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml +++ b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml @@ -8,6 +8,10 @@ data: icon: sapfieldglass.svg license: MIT name: SAP Fieldglass + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sap-fieldglass registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py index 0f5e02083b5b..7365a4beb5ce 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sap-fieldglass=source_sap_fieldglass.run:run", + ], + }, name="source_sap_fieldglass", description="Source implementation for Sap Fieldglass.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py b/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py new file mode 100644 index 000000000000..69768a3d9efc --- /dev/null +++ b/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sap_fieldglass import SourceSapFieldglass + + +def run(): + source = SourceSapFieldglass() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle index 516c96a06808..8ae37e30a930 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle @@ -1,10 +1,9 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.5.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } @@ -17,10 +16,5 @@ dependencies { //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.testcontainers.jdbc - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-scaffold-java-jdbc') - - testFixturesImplementation libs.testcontainers.jdbc + testFixturesApi 'org.testcontainers:jdbc:1.19.4' } diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml index 65fc8a95d574..83f3cf72db6a 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: database connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-scaffold-java-jdbc githubIssueLabel: source-scaffold-java-jdbc icon: scaffold-java-jdbc.svg diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/main.py b/airbyte-integrations/connectors/source-scaffold-source-http/main.py index f8030f72e927..f51b0283a681 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_scaffold_source_http import SourceScaffoldSourceHttp +from source_scaffold_source_http.run import run if __name__ == "__main__": - source = SourceScaffoldSourceHttp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml index 3e50afa2130e..c56a4810bde9 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-scaffold-source-http registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py index 4fbb5fc38b4d..a7a496b52161 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-scaffold-source-http=source_scaffold_source_http.run:run", + ], + }, name="source_scaffold_source_http", description="Source implementation for Scaffold Source Http.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py new file mode 100644 index 000000000000..94b4f015f312 --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_scaffold_source_http import SourceScaffoldSourceHttp + + +def run(): + source = SourceScaffoldSourceHttp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/main.py b/airbyte-integrations/connectors/source-scaffold-source-python/main.py index 0b8a20830163..038eaf6dba55 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_scaffold_source_python import SourceScaffoldSourcePython +from source_scaffold_source_python.run import run if __name__ == "__main__": - source = SourceScaffoldSourcePython() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml index 2a740dccdc56..24ab5f9c09c1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-scaffold-source-python registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index b302f081011f..ebeda07f6998 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-scaffold-source-python=source_scaffold_source_python.run:run", + ], + }, name="source_scaffold_source_python", description="Source implementation for Scaffold Source Python.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py new file mode 100644 index 000000000000..6bb55fc68e7d --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_scaffold_source_python import SourceScaffoldSourcePython + + +def run(): + source = SourceScaffoldSourcePython() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-search-metrics/main.py b/airbyte-integrations/connectors/source-search-metrics/main.py index c07a9cf2a41b..29e5e8a133a6 100644 --- a/airbyte-integrations/connectors/source-search-metrics/main.py +++ b/airbyte-integrations/connectors/source-search-metrics/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_search_metrics import SourceSearchMetrics +from source_search_metrics.run import run if __name__ == "__main__": - source = SourceSearchMetrics() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml index ee0f969317bd..58b77c1ed967 100644 --- a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml +++ b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml @@ -8,11 +8,15 @@ data: icon: searchmetrics.svg license: MIT name: SearchMetrics - registries: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-search-metrics + registries: # Removed from registries due to LEGACY STATE cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/search-metrics tags: diff --git a/airbyte-integrations/connectors/source-search-metrics/setup.py b/airbyte-integrations/connectors/source-search-metrics/setup.py index fda5333cc2ae..2e3f6e81f310 100644 --- a/airbyte-integrations/connectors/source-search-metrics/setup.py +++ b/airbyte-integrations/connectors/source-search-metrics/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-search-metrics=source_search_metrics.run:run", + ], + }, name="source_search_metrics", description="Source implementation for Search Metrics.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py b/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py new file mode 100644 index 000000000000..29a0ebc75add --- /dev/null +++ b/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_search_metrics import SourceSearchMetrics + + +def run(): + source = SourceSearchMetrics() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-secoda/main.py b/airbyte-integrations/connectors/source-secoda/main.py index 81d52e2f8a48..96bc9de24afa 100644 --- a/airbyte-integrations/connectors/source-secoda/main.py +++ b/airbyte-integrations/connectors/source-secoda/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_secoda import SourceSecoda +from source_secoda.run import run if __name__ == "__main__": - source = SourceSecoda() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-secoda/metadata.yaml b/airbyte-integrations/connectors/source-secoda/metadata.yaml index 03c428b40206..cc7a8383aefe 100644 --- a/airbyte-integrations/connectors/source-secoda/metadata.yaml +++ b/airbyte-integrations/connectors/source-secoda/metadata.yaml @@ -8,6 +8,10 @@ data: icon: secoda.svg license: MIT name: Secoda + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-secoda registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-secoda/setup.py b/airbyte-integrations/connectors/source-secoda/setup.py index 3b603c1d6e21..b80c65173e94 100644 --- a/airbyte-integrations/connectors/source-secoda/setup.py +++ b/airbyte-integrations/connectors/source-secoda/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-secoda=source_secoda.run:run", + ], + }, name="source_secoda", description="Source implementation for Secoda.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-secoda/source_secoda/run.py b/airbyte-integrations/connectors/source-secoda/source_secoda/run.py new file mode 100644 index 000000000000..ed456d531ce1 --- /dev/null +++ b/airbyte-integrations/connectors/source-secoda/source_secoda/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_secoda import SourceSecoda + + +def run(): + source = SourceSecoda() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sendgrid/README.md b/airbyte-integrations/connectors/source-sendgrid/README.md index 36f73fd3d575..599bdb7eeddf 100644 --- a/airbyte-integrations/connectors/source-sendgrid/README.md +++ b/airbyte-integrations/connectors/source-sendgrid/README.md @@ -1,119 +1,55 @@ -# Sendgrid Source +# Sendgrid source connector + This is the repository for the Sendgrid source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/sendgrid). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/sendgrid). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sendgrid) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sendgrid/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/sendgrid) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sendgrid/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source sendgrid test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-sendgrid spec +poetry run source-sendgrid check --config secrets/config.json +poetry run source-sendgrid discover --config secrets/config.json +poetry run source-sendgrid read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-sendgrid build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-sendgrid:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-sendgrid:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-sendgrid:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-sendgrid:dev . -# Running the spec command against your patched connector -docker run airbyte/source-sendgrid:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-sendgrid:dev spec @@ -122,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendgrid:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sendgrid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-sendgrid test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sendgrid test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sendgrid.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/sendgrid.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl index 188db682c2c6..acf7830c5b8e 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl @@ -184,12 +184,12 @@ {"stream": "suppression_group_members", "data": {"email": "test-forsuppressiongroup number8@example.com", "group_id": 14772, "group_name": "Test Suggestions Group 12", "created_at": 1612363238}, "emitted_at": 1631093393000} {"stream": "suppression_group_members", "data": {"email": "test-forsuppressiongroup number9@example.com", "group_id": 14772, "group_name": "Test Suggestions Group 12", "created_at": 1612363238}, "emitted_at": 1631093393000} {"stream": "suppression_group_members", "data": {"email": "avida.d3@gmail.com", "group_id": 14780, "group_name": "Test Suggestions Group 20", "created_at": 1631093329}, "emitted_at": 1631093393000} -{"stream": "bounces", "data": { "created": 1621442821, "email": "vadym.hevlich@zazmicinvalid", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} -{"stream": "bounces", "data": { "created": 1621441107, "email": "vadym.hevlich@zazmiccom2", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} -{"stream": "bounces", "data": { "created": 1621442883, "email": "vadym.hevlich@zazmic_com", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} -{"stream": "bounces", "data": { "created": 1621441104, "email": "vadym.hevlich@zazmiccom1", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} -{"stream": "bounces", "data": { "created": 1621442811, "email": "vadym.hevlich@zazmicio", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680685} -{"stream": "bounces", "data": { "created": 1621430037, "email": "vadym.hevlich@zazmiccom", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680685} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmic_com", "created": 1621439283}, "emitted_at": 1708535996116} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmicinvalid", "created": 1621439221}, "emitted_at": 1708535996116} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmicio", "created": 1621439211}, "emitted_at": 1708535996116} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmiccom2", "created": 1621437507}, "emitted_at": 1708535996117} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmiccom1", "created": 1621437504}, "emitted_at": 1708535996117} +{"stream": "bounces", "data": {"status": "", "reason": "Invalid Domain", "email": "vadym.hevlich@zazmiccom", "created": 1621426437}, "emitted_at": 1708535996117} {"stream": "campaigns", "data": {"created_at": "2021-09-08T09:07:48Z", "id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:11:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750589} {"stream": "campaigns", "data": {"created_at": "2021-09-08T09:04:36Z", "id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419", "name": "Copy of Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:09:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750589} {"stream": "campaigns", "data": {"created_at": "2021-09-08T08:53:59Z", "id": "4e5be6a3-1082-11ec-8512-9afd40c324e6", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T08:57:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750590} diff --git a/airbyte-integrations/connectors/source-sendgrid/main.py b/airbyte-integrations/connectors/source-sendgrid/main.py index f9720b12f1c8..e1c5a04db60d 100644 --- a/airbyte-integrations/connectors/source-sendgrid/main.py +++ b/airbyte-integrations/connectors/source-sendgrid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sendgrid import SourceSendgrid +from source_sendgrid.run import run if __name__ == "__main__": - source = SourceSendgrid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml index fae7388cf808..9955875b363d 100644 --- a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 - dockerImageTag: 0.4.1 + dockerImageTag: 0.4.3 dockerRepository: airbyte/source-sendgrid documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid githubIssueLabel: source-sendgrid icon: sendgrid.svg license: MIT name: Sendgrid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sendgrid registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sendgrid/poetry.lock b/airbyte-integrations/connectors/source-sendgrid/poetry.lock new file mode 100644 index 000000000000..be3562417885 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/poetry.lock @@ -0,0 +1,1206 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.39" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.39.tar.gz", hash = "sha256:93e50c6586a41592e3debdcaa051e0c02d8b45cb59a3b411f62d427aa29bbaeb"}, + {file = "airbyte_cdk-0.51.39-py3-none-any.whl", hash = "sha256:ce2436f9c07a2631c5b5c00f7324e728cfb3f81817b58ca0015892f7f94365ac"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.0" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.0" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.0-py3-none-any.whl", hash = "sha256:e6a31fcd237504198a678d02c0040a8798f281c39203da61a5abce67842c5360"}, + {file = "airbyte_protocol_models-0.4.0.tar.gz", hash = "sha256:518736015c29ac60b6b8964a1b0d9b52e40020bcbd89e2545cc781f0b37d0f2b"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.1.1" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, + {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, + {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, + {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, + {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, + {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, + {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, + {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, + {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, + {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "8a246317365f1708c1268c85f6a66e59fbc624bd1d3449ff8533756564b443b0" diff --git a/airbyte-integrations/connectors/source-sendgrid/pyproject.toml b/airbyte-integrations/connectors/source-sendgrid/pyproject.toml new file mode 100644 index 000000000000..35ad4315bcc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.4.3" +name = "source-sendgrid" +description = "Source implementation for Sendgrid." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/sendgrid" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_sendgrid" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.51.39" +requests = "==2.31.0" +pandas = "==2.1.1" +backoff = "==2.2.1" + +[tool.poetry.scripts] +source-sendgrid = "source_sendgrid.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-sendgrid/requirements.txt b/airbyte-integrations/connectors/source-sendgrid/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-sendgrid/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py deleted file mode 100644 index 64dc92cce236..000000000000 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "backoff", "requests", "pandas"] - -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "requests-mock", -] - -setup( - name="source_sendgrid", - description="Source implementation for Sendgrid.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py new file mode 100644 index 000000000000..d2c1b798edb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sendgrid import SourceSendgrid + + +def run(): + source = SourceSendgrid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index 9275048d660d..68f1932665cd 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -299,7 +299,13 @@ def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str]: tmp_file, "wb" ) as data_file: for chunk in response.iter_content(chunk_size=chunk_size): - data_file.write(decompressor.decompress(chunk)) + try: + # see if it's compressed. we are seeing some that are not all of a sudden. + # but let's also guard against the case where sendgrid changes it back. + data_file.write(decompressor.decompress(chunk)) + except zlib.error as e: + # it's not actually compressed! + data_file.write(chunk) # check the file exists if os.path.isfile(tmp_file): return tmp_file, self.encoding diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 14fab1cb6bee..750a1db4c8b3 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -153,7 +153,7 @@ def test_should_retry_on_permission_error(requests_mock, stream_class, status, e def test_compressed_contact_response(requests_mock): stream = Contacts() - with open(os.path.dirname(__file__) + "/compressed_response", "rb") as compressed_response: + with open(os.path.dirname(__file__) + "/compressed_response", "rb") as file_response: url = "https://api.sendgrid.com/v3/marketing/contacts/exports" requests_mock.register_uri("POST", url, [{"json": {"id": "random_id"}, "status_code": 202}]) url = "https://api.sendgrid.com/v3/marketing/contacts/exports/random_id" @@ -162,7 +162,28 @@ def test_compressed_contact_response(requests_mock): {"json": {"status": "ready", "urls": ["https://sample_url/sample_csv.csv.gzip"]}, "status_code": 202}, ] requests_mock.register_uri("GET", url, resp_bodies) - requests_mock.register_uri("GET", "https://sample_url/sample_csv.csv.gzip", [{"body": compressed_response, "status_code": 202}]) + requests_mock.register_uri("GET", "https://sample_url/sample_csv.csv.gzip", [{"body": file_response, "status_code": 202}]) + recs = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + decompressed_response = pd.read_csv(os.path.dirname(__file__) + "/decompressed_response.csv", dtype=str) + expected_records = [ + {k.lower(): v for k, v in x.items()} for x in decompressed_response.replace({nan: None}).to_dict(orient="records") + ] + + assert recs == expected_records + + +def test_uncompressed_contact_response(requests_mock): + stream = Contacts() + with open(os.path.dirname(__file__) + "/decompressed_response.csv", "rb") as file_response: + url = "https://api.sendgrid.com/v3/marketing/contacts/exports" + requests_mock.register_uri("POST", url, [{"json": {"id": "random_id"}, "status_code": 202}]) + url = "https://api.sendgrid.com/v3/marketing/contacts/exports/random_id" + resp_bodies = [ + {"json": {"status": "pending", "id": "random_id", "urls": []}, "status_code": 202}, + {"json": {"status": "ready", "urls": ["https://sample_url/sample_csv.csv.gzip"]}, "status_code": 202}, + ] + requests_mock.register_uri("GET", url, resp_bodies) + requests_mock.register_uri("GET", "https://sample_url/sample_csv.csv.gzip", [{"body": file_response, "status_code": 202}]) recs = list(stream.read_records(sync_mode=SyncMode.full_refresh)) decompressed_response = pd.read_csv(os.path.dirname(__file__) + "/decompressed_response.csv", dtype=str) expected_records = [ diff --git a/airbyte-integrations/connectors/source-sendinblue/Dockerfile b/airbyte-integrations/connectors/source-sendinblue/Dockerfile index 80211cef5582..6e2828317ee3 100644 --- a/airbyte-integrations/connectors/source-sendinblue/Dockerfile +++ b/airbyte-integrations/connectors/source-sendinblue/Dockerfile @@ -34,5 +34,5 @@ COPY source_sendinblue ./source_sendinblue ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-sendinblue diff --git a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml index 144dbb788a95..106bcca9d06f 100644 --- a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml @@ -26,12 +26,11 @@ acceptance_tests: # exact_order: no # extra_records: yes incremental: - bypass_reason: "This connector does not implement incremental sync" - # TODO uncomment this block this block if your connector implements incremental sync: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json index 52b0f2c2118f..663a73cd59d6 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json @@ -1,5 +1,13 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "modifiedAt": "2050-10-31T02:00:22.240+01:00" + }, + "stream_descriptor": { + "name": "contacts" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json index 8b74ccf7cd93..ec8321689992 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json @@ -22,10 +22,10 @@ "stream": { "name": "contacts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-sendinblue/main.py b/airbyte-integrations/connectors/source-sendinblue/main.py index 8f9399e259f2..7227005ddf6f 100644 --- a/airbyte-integrations/connectors/source-sendinblue/main.py +++ b/airbyte-integrations/connectors/source-sendinblue/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sendinblue import SourceSendinblue +from source_sendinblue.run import run if __name__ == "__main__": - source = SourceSendinblue() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml index 6d5b26d340ba..0318419f4d85 100644 --- a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml @@ -5,13 +5,17 @@ data: connectorSubtype: api connectorType: source definitionId: 2e88fa20-a2f6-43cc-bba6-98a0a3f244fb - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-sendinblue documentationUrl: https://docs.airbyte.com/integrations/sources/sendinblue githubIssueLabel: source-sendinblue icon: sendinblue.svg license: MIT name: Sendinblue + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sendinblue registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sendinblue/setup.py b/airbyte-integrations/connectors/source-sendinblue/setup.py index 03d9985367a3..e452b66f447a 100644 --- a/airbyte-integrations/connectors/source-sendinblue/setup.py +++ b/airbyte-integrations/connectors/source-sendinblue/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sendinblue=source_sendinblue.run:run", + ], + }, name="source_sendinblue", description="Source implementation for Sendinblue.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml index 94feb419dd32..83667afcd717 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml @@ -23,6 +23,19 @@ definitions: page_size_option: inject_into: "request_parameter" field_name: "limit" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: modifiedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: "2000-01-01T00:00:00Z" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + inject_into: request_parameter + field_name: modifiedSince retriever: record_selector: $ref: "#/definitions/selector" @@ -53,6 +66,8 @@ definitions: path: "/smtp/templates" contacts_stream: $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/incremental_sync" $parameters: name: contacts primary_key: id diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py new file mode 100644 index 000000000000..fbf77ce5a97a --- /dev/null +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sendinblue import SourceSendinblue + + +def run(): + source = SourceSendinblue() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json index 02ab48afdaf0..b72b892c4360 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -24,7 +25,9 @@ "type": ["null", "string"] }, "sender": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "replyTo": { "type": ["null", "string"] @@ -62,6 +65,9 @@ "modifiedAt": { "type": ["null", "string"] }, + "previewText": { + "type": ["null", "string"] + }, "shareLink": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json index e23ac3c423a2..d20dfde96bab 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -24,7 +25,9 @@ "type": ["null", "string"] }, "attributes": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} } } } diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json index 8c4fd441f208..e57f22730e2e 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -18,7 +19,12 @@ "type": ["null", "boolean"] }, "sender": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + "subject": { + "type": ["null", "string"] }, "replyTo": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-senseforce/main.py b/airbyte-integrations/connectors/source-senseforce/main.py index ba269b9ba0ef..3a6a5e7fd4c7 100644 --- a/airbyte-integrations/connectors/source-senseforce/main.py +++ b/airbyte-integrations/connectors/source-senseforce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_senseforce import SourceSenseforce +from source_senseforce.run import run if __name__ == "__main__": - source = SourceSenseforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-senseforce/metadata.yaml b/airbyte-integrations/connectors/source-senseforce/metadata.yaml index 3c9417880e60..c5d5d8dd7695 100644 --- a/airbyte-integrations/connectors/source-senseforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-senseforce/metadata.yaml @@ -12,6 +12,10 @@ data: icon: senseforce.svg license: MIT name: Senseforce + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-senseforce registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-senseforce/setup.py b/airbyte-integrations/connectors/source-senseforce/setup.py index 976590623a83..4c653a13c824 100644 --- a/airbyte-integrations/connectors/source-senseforce/setup.py +++ b/airbyte-integrations/connectors/source-senseforce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-senseforce=source_senseforce.run:run", + ], + }, name="source_senseforce", description="Source implementation for Senseforce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py b/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py new file mode 100644 index 000000000000..944e3468fed7 --- /dev/null +++ b/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_senseforce import SourceSenseforce + + +def run(): + source = SourceSenseforce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile deleted file mode 100644 index 4eaed1f20ed1..000000000000 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_sentry ./source_sentry - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/README.md b/airbyte-integrations/connectors/source-sentry/README.md index 96b46caade7c..5646c9f4be76 100644 --- a/airbyte-integrations/connectors/source-sentry/README.md +++ b/airbyte-integrations/connectors/source-sentry/README.md @@ -1,69 +1,55 @@ -# Sentry Source +# Sentry source connector + This is the repository for the Sentry source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/sentry). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/sentry). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sentry) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sentry/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/sentry) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sentry/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source sentry test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-sentry spec +poetry run source-sentry check --config secrets/config.json +poetry run source-sentry discover --config secrets/config.json +poetry run source-sentry read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-sentry build ``` -An image will be built with the tag `airbyte/source-sentry:dev`. +An image will be available on your host with the tag `airbyte/source-sentry:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-sentry:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-sentry:dev spec @@ -72,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sentry:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sentry:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-sentry test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sentry test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sentry.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/sentry.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index b989c169e2e8..c1bbe22949d7 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -2,6 +2,11 @@ acceptance_tests: basic_read: tests: - config_path: secrets/config.json + empty_streams: + - name: issues + bypass_reason: "Project sssues are not being returned by the Sentry API." + - name: events + bypass_reason: "No event records exist for the test project." timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" @@ -43,9 +48,15 @@ acceptance_tests: incremental: tests: - config_path: secrets/config.json + skip_comprehensive_incremental_tests: true configured_catalog_path: integration_tests/configured_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json + missing_streams: + - name: issues + bypass_reason: "Project issues are not being returned by the Sentry API." + - name: events + bypass_reason: "No event records exist for the test project." spec: tests: - spec_path: source_sentry/spec.json diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl index 1dd8a4f0937f..6720558b606b 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl @@ -1,6 +1,5 @@ -{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["event:admin", "member:read", "project:admin", "project:read", "alerts:read", "alerts:write", "team:read", "project:write", "org:integrations", "team:admin", "event:write", "team:write", "org:read", "event:read", "project:releases"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"version": "checkout-app@3.2"}, "options": {"sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "recapServerUrl": null, "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2023-01-11", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["mobile-cpu-memory-in-transactions", "performance-issues-all-events-tab", "performance-transaction-name-only-search-indexed", "anr-rate", "performance-new-widget-designs", "integrations-auto-repo-linking", "india-promotion", "performance-large-http-payload-detector", "session-replay-recording-scrubbing", "sourcemaps-bundle-flat-file-indexing", "performance-view", "performance-consecutive-db-issue", "discover-events-rate-limit", "dashboards-rh-widget", "projconfig-exclude-measurements", "crons-new-onboarding", "performance-n-plus-one-api-calls-detector", "performance-issues-render-blocking-assets-detector", "session-replay-event-linking", "dashboards-mep", "ondemand-budgets", "open-membership", "alert-crash-free-metrics", "integrations-stacktrace-link", "performance-issues-http-overhead-detector", "monitors", "profiling-ui-frames", "profiling-billing", "performance-mep-bannerless-ui", "getting-started-doc-with-product-selection", "am2-billing", "session-replay-optimized-search", "profile-frame-drop-experimental-ingest", "project-performance-settings-admin", "performance-file-io-main-thread-detector", "auto-enable-codecov", "invite-members-rate-limits", "session-replay-ui", "open-ai-suggestion", "performance-db-main-thread-detector", "ds-sliding-window-org", "issue-details-most-helpful-event", "customer-domains", "symbol-sources", "performance-http-overhead-post-process-group", "performance-span-histogram-view", "promotion-mobperf-gift50kerr", "issue-platform", "onboarding", "escalating-metrics-backend", "issue-alert-fallback-targeting", "performance-tracing-without-performance", "profile-file-io-main-thread-ingest", "performance-http-overhead-ingest", "derive-code-mappings", "performance-issues-search", "promotion-be-adoption-enabled", "slack-overage-notifications", "paid-to-free-promotion", "minute-resolution-sessions", "device-classification", "profiling-cpu-chart", "ds-org-recalibration", "performance-landing-page-stats-period", "session-replay", "shared-issues", "profile-image-decode-main-thread-post-process-group", "profile-image-decode-main-thread-visible", "business-to-team-promotion", "noisy-alert-warning", "profile-file-io-main-thread-post-process-group", "advanced-search", "integrations-deployment", "event-attachments", "profile-json-decode-main-thread-post-process-group", "track-button-click-events", "transaction-name-normalize", "profile-file-io-main-thread-visible", "profiling-global-suspect-functions", "dynamic-sampling", "escalating-issues", "promotion-mobperf-discount20", "onboarding-sdk-selection", "profiling-view", "metrics-extraction", "profile-json-decode-main-thread-ingest", "performance-issues-m-n-plus-one-db-detector", "transaction-name-mark-scrubbed-as-sanitized", "sdk-crash-detection", "org-subdomains", "performance-metrics-backed-transaction-summary", "enterprise-spike-protection-window", "profile-image-decode-main-thread-ingest", "performance-consecutive-http-detector", "session-replay-trial-ended-banner", "session-replay-slack-new-issue", "session-replay-issue-emails", "transaction-metrics-extraction", "performance-onboarding-checklist", "profile-json-decode-main-thread-visible", "mep-rollout-flag", "metric-alert-chartcuterie", "streamline-targeting-context", "performance-issues-compressed-assets-detector", "project-stats", "issue-details-most-helpful-event-ui", "release-health-drop-sessions", "profile-frame-drop-experimental-post-process-group", "profiling", "profiling-memory-chart", "issue-details-stacktrace-improvements", "device-class-synthesis", "performance-http-overhead-visible", "team-project-creation-all", "performance-slow-db-issue"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us1.sentry.io"}, "hasAuthProvider": false}, "plugins": [], "platforms": [], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}, {"id": "recalibrationRule", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1695121555366} -{"stream": "projects", "data": {"id": "6712547", "slug": "demo-integration", "name": "demo-integration", "platform": "javascript-react", "dateCreated": "2022-09-02T15:01:28.946777Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view"], "firstEvent": "2022-09-02T15:36:50.870000Z", "firstTransactionEvent": false, "access": ["member:read", "alerts:read", "team:admin", "event:write", "project:admin", "org:read", "team:write", "event:read", "event:admin", "project:write", "org:integrations", "team:read", "alerts:write", "project:read", "project:releases"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#bf833f", "status": "active", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["projconfig-exclude-measurements", "anr-rate", "promotion-be-adoption-enabled", "performance-issues-all-events-tab", "issue-details-stacktrace-improvements", "integrations-stacktrace-link", "device-class-synthesis", "monitors", "transaction-name-mark-scrubbed-as-sanitized", "slack-overage-notifications", "symbol-sources", "auto-enable-codecov", "onboarding-sdk-selection", "session-replay-slack-new-issue", "performance-n-plus-one-api-calls-detector", "performance-large-http-payload-detector", "performance-view", "performance-issues-search", "release-health-drop-sessions", "device-classification", "profile-json-decode-main-thread-post-process-group", "advanced-search", "india-promotion", "promotion-mobperf-gift50kerr", "profiling-ui-frames", "ds-org-recalibration", "dashboards-mep", "team-project-creation-all", "profiling-global-suspect-functions", "crons-new-onboarding", "metric-alert-chartcuterie", "performance-span-histogram-view", "performance-issues-render-blocking-assets-detector", "session-replay-optimized-search", "profile-image-decode-main-thread-ingest", "session-replay", "performance-landing-page-stats-period", "profile-file-io-main-thread-ingest", "profile-image-decode-main-thread-visible", "issue-details-most-helpful-event-ui", "minute-resolution-sessions", "profiling-billing", "sdk-crash-detection", "profile-image-decode-main-thread-post-process-group", "escalating-metrics-backend", "derive-code-mappings", "transaction-metrics-extraction", "promotion-mobperf-discount20", "profile-frame-drop-experimental-post-process-group", "performance-metrics-backed-transaction-summary", "github-disable-on-broken", "profiling-view", "integrations-auto-repo-linking", "org-subdomains", "open-ai-suggestion", "profiling-cpu-chart", "event-attachments", "enterprise-spike-protection-window", "profile-json-decode-main-thread-ingest", "dynamic-sampling", "performance-mep-bannerless-ui", "invite-members-rate-limits", "integrations-deployment", "profile-json-decode-main-thread-visible", "performance-file-io-main-thread-detector", "streamline-targeting-context", "performance-db-main-thread-detector", "performance-consecutive-db-issue", "issue-details-most-helpful-event", "profiling", "performance-new-widget-designs", "session-replay-issue-emails", "performance-transaction-name-only-search-indexed", "performance-http-overhead-post-process-group", "open-membership", "transaction-name-normalize", "performance-http-overhead-ingest", "project-performance-settings-admin", "paid-to-free-promotion", "profile-file-io-main-thread-post-process-group", "session-replay-ui", "ondemand-budgets", "getting-started-doc-with-product-selection", "performance-issues-m-n-plus-one-db-detector", "mobile-cpu-memory-in-transactions", "profiling-memory-chart", "issue-platform", "shared-issues", "performance-issues-http-overhead-detector", "performance-http-overhead-visible", "performance-tracing-without-performance", "escalating-issues", "track-button-click-events", "onboarding", "metrics-extraction", "customer-domains", "dashboards-rh-widget", "project-stats", "session-replay-trial-ended-banner", "business-to-team-promotion", "performance-issues-compressed-assets-detector", "session-replay-recording-scrubbing", "performance-onboarding-checklist", "discover-events-rate-limit", "sourcemaps-bundle-flat-file-indexing", "alert-crash-free-metrics", "performance-consecutive-http-detector", "ds-sliding-window-org", "mep-rollout-flag", "profile-file-io-main-thread-visible", "am2-billing", "performance-slow-db-issue", "profile-frame-drop-experimental-ingest", "issue-alert-fallback-targeting"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us1.sentry.io"}, "hasAuthProvider": false}}, "emitted_at": 1695036641442} -{"stream": "projects", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["member:read", "alerts:read", "team:admin", "event:write", "project:admin", "org:read", "team:write", "event:read", "event:admin", "project:write", "org:integrations", "team:read", "alerts:write", "project:read", "project:releases"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["projconfig-exclude-measurements", "anr-rate", "promotion-be-adoption-enabled", "performance-issues-all-events-tab", "issue-details-stacktrace-improvements", "integrations-stacktrace-link", "device-class-synthesis", "monitors", "transaction-name-mark-scrubbed-as-sanitized", "slack-overage-notifications", "symbol-sources", "auto-enable-codecov", "onboarding-sdk-selection", "session-replay-slack-new-issue", "performance-n-plus-one-api-calls-detector", "performance-large-http-payload-detector", "performance-view", "performance-issues-search", "release-health-drop-sessions", "device-classification", "profile-json-decode-main-thread-post-process-group", "advanced-search", "india-promotion", "promotion-mobperf-gift50kerr", "profiling-ui-frames", "ds-org-recalibration", "dashboards-mep", "team-project-creation-all", "profiling-global-suspect-functions", "crons-new-onboarding", "metric-alert-chartcuterie", "performance-span-histogram-view", "performance-issues-render-blocking-assets-detector", "session-replay-optimized-search", "profile-image-decode-main-thread-ingest", "session-replay", "performance-landing-page-stats-period", "profile-file-io-main-thread-ingest", "profile-image-decode-main-thread-visible", "issue-details-most-helpful-event-ui", "minute-resolution-sessions", "profiling-billing", "sdk-crash-detection", "profile-image-decode-main-thread-post-process-group", "escalating-metrics-backend", "derive-code-mappings", "transaction-metrics-extraction", "promotion-mobperf-discount20", "profile-frame-drop-experimental-post-process-group", "performance-metrics-backed-transaction-summary", "github-disable-on-broken", "profiling-view", "integrations-auto-repo-linking", "org-subdomains", "open-ai-suggestion", "profiling-cpu-chart", "event-attachments", "enterprise-spike-protection-window", "profile-json-decode-main-thread-ingest", "dynamic-sampling", "performance-mep-bannerless-ui", "invite-members-rate-limits", "integrations-deployment", "profile-json-decode-main-thread-visible", "performance-file-io-main-thread-detector", "streamline-targeting-context", "performance-db-main-thread-detector", "performance-consecutive-db-issue", "issue-details-most-helpful-event", "profiling", "performance-new-widget-designs", "session-replay-issue-emails", "performance-transaction-name-only-search-indexed", "performance-http-overhead-post-process-group", "open-membership", "transaction-name-normalize", "performance-http-overhead-ingest", "project-performance-settings-admin", "paid-to-free-promotion", "profile-file-io-main-thread-post-process-group", "session-replay-ui", "ondemand-budgets", "getting-started-doc-with-product-selection", "performance-issues-m-n-plus-one-db-detector", "mobile-cpu-memory-in-transactions", "profiling-memory-chart", "issue-platform", "shared-issues", "performance-issues-http-overhead-detector", "performance-http-overhead-visible", "performance-tracing-without-performance", "escalating-issues", "track-button-click-events", "onboarding", "metrics-extraction", "customer-domains", "dashboards-rh-widget", "project-stats", "session-replay-trial-ended-banner", "business-to-team-promotion", "performance-issues-compressed-assets-detector", "session-replay-recording-scrubbing", "performance-onboarding-checklist", "discover-events-rate-limit", "sourcemaps-bundle-flat-file-indexing", "alert-crash-free-metrics", "performance-consecutive-http-detector", "ds-sliding-window-org", "mep-rollout-flag", "profile-file-io-main-thread-visible", "am2-billing", "performance-slow-db-issue", "profile-frame-drop-experimental-ingest", "issue-alert-fallback-targeting"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us1.sentry.io"}, "hasAuthProvider": false}}, "emitted_at": 1695036641444} +{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "span-metrics-extraction-resource", "span-metrics-extraction", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["event:admin", "event:write", "project:read", "org:read", "team:read", "org:integrations", "project:write", "alerts:write", "team:write", "event:read", "project:admin", "project:releases", "alerts:read", "member:read", "team:admin"], "hasAccess": true, "hasCustomMetrics": false, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasFeedbacks": false, "hasNewFeedbacks": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"version": "checkout-app@3.2"}, "options": {"sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:chunk-load-error": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true, "quotas:spike-protection-disabled": false}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "recapServerUrl": null, "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2023-01-11", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null, "avatarUrl": null}, "features": ["new-page-filter", "derive-code-mappings", "open-membership", "issue-platform-api-crons-sd", "sourcemaps-upload-release-as-artifact-bundle", "onboarding-sdk-selection", "minute-resolution-sessions", "performance-issues-search", "profile-json-decode-main-thread-ingest", "issue-platform-crons-sd", "profiling-statistical-detectors-breakpoint", "issue-stream-performance", "performance-http-overhead-ingest", "issue-platform", "ondemand-budgets", "session-replay-count-query-optimize", "profile-file-io-main-thread-post-process-group", "session-replay-event-linking", "performance-db-main-thread-detector", "slack-overage-notifications", "suspect-commits-all-frames", "feedback-visible", "feedback-post-process-group", "source-maps-debugger-blue-thunder-edition", "frontend-domainsplit", "device-classification", "crons-disable-new-projects", "transaction-name-mark-scrubbed-as-sanitized", "performance-span-histogram-view", "user-feedback-ingest", "ds-org-recalibration", "device-class-synthesis", "performance-mep-bannerless-ui", "promotion-mobperf-gift50kerr", "performance-transaction-name-only-search-indexed", "profile-image-decode-main-thread-ingest", "monitors", "session-replay-issue-emails", "promotion-be-adoption-enabled", "feedback-ingest", "performance-issues-compressed-assets-detector", "starfish-aggregate-span-waterfall", "performance-large-http-payload-detector", "profiling-global-suspect-functions", "transaction-metrics-extraction", "performance-http-overhead-post-process-group", "am2-billing", "project-stats", "starfish-browser-resource-module-ui", "session-replay", "profiling-ui-frames", "session-replay-recording-scrubbing", "sdk-crash-detection", "performance-consecutive-http-detector", "profiling-billing", "performance-tracing-without-performance", "performance-issues-http-overhead-detector", "session-replay-trial-ended-banner", "profile-image-decode-main-thread-post-process-group", "alerts-migration-enabled", "profiling-battery-usage-chart", "user-feedback-ui", "ds-sliding-window-org", "shared-issues", "issue-stream-performance-cache", "performance-view", "performance-calculate-score-relay", "event-attachments", "session-replay-show-hydration-errors", "performance-landing-page-stats-period", "symbol-sources", "profiling-summary-redesign", "metric-alert-chartcuterie", "discover-events-rate-limit", "starfish-browser-webvitals", "paid-to-free-promotion", "invite-members-rate-limits", "integrations-deployment", "release-health-drop-sessions", "session-replay-ui", "performance-duration-regression-ingest", "mep-rollout-flag", "profiling-view", "metrics-extraction", "profiling-memory-chart", "on-demand-metrics-extraction", "dashboard-widget-indicators", "performance-http-overhead-visible", "integrations-stacktrace-link", "auto-enable-codecov", "noisy-alert-warning", "advanced-search", "profile-json-decode-main-thread-visible", "performance-issues-render-blocking-assets-detector", "session-replay-accessibility-issues", "performance-statistical-detectors-breakpoint", "performance-n-plus-one-api-calls-detector", "customer-domains", "session-replay-a11y-tab", "escalating-issues", "streamline-targeting-context", "dynamic-sampling", "dashboards-rh-widget", "performance-database-view-query-source", "business-to-team-promotion", "profile-frame-drop-experimental-ingest", "mobile-cpu-memory-in-transactions", "starfish-browser-webvitals-pageoverview-v2", "performance-slow-db-issue", "performance-statistical-detectors-ema", "promotion-reserved-txn-discount", "session-replay-slack-new-issue", "profiling-statistical-detectors-ema", "profile-file-io-main-thread-visible", "profile-json-decode-main-thread-post-process-group", "performance-metrics-backed-transaction-summary", "onboarding", "profile-function-regression-exp-ingest", "org-subdomains", "performance-new-widget-designs", "performance-consecutive-db-issue", "performance-file-io-main-thread-detector", "trace-view-load-more", "profile-frame-drop-experimental-post-process-group", "profiling", "escalating-metrics-backend", "alert-crash-free-metrics", "promotion-mobperf-discount20", "dashboards-mep", "issue-alert-fallback-targeting", "transaction-name-normalize", "performance-issues-m-n-plus-one-db-detector", "performance-database-view", "profiling-cpu-chart", "integrations-gh-invite", "profiling-differential-flamegraph", "performance-onboarding-checklist", "profile-file-io-main-thread-ingest", "performance-screens-view", "profile-image-decode-main-thread-visible", "india-promotion", "session-replay-onboarding-cta-button", "performance-issues-all-events-tab"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}, "plugins": [], "platforms": [], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}, {"id": "recalibrationRule", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1704483339623} {"stream": "releases", "data": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": [], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "emitted_at": 1689246658349} -{"stream": "issues", "data": {"id": "4365423845", "shareId": null, "shortId": "AIRBYTE-09-4", "title": "This is an example Python exception", "culprit": "raven.scripts.runner in main", "permalink": "https://airbyte-09.sentry.io/issues/4365423845/", "logger": null, "level": "error", "status": "unresolved", "statusDetails": {}, "substatus": "ongoing", "isPublic": false, "platform": "python", "project": {"id": "5942472", "name": "airbyte-09", "slug": "airbyte-09", "platform": "python"}, "type": "default", "metadata": {"title": "This is an example Python exception", "in_app_frame_mix": "system-only"}, "numComments": 0, "assignedTo": null, "isBookmarked": false, "isSubscribed": false, "subscriptionDetails": null, "hasSeen": true, "annotations": [], "issueType": "error", "issueCategory": "error", "isUnhandled": false, "count": "11", "userCount": 1, "firstSeen": "2023-08-02T23:22:34.982000Z", "lastSeen": "2023-09-08T13:39:15.138000Z"}, "emitted_at": 1695036639898} -{"stream": "events", "data": {"id": "ea46cf69cbc346c2b7a891dbdb2df289", "groupID": "4365423845", "eventID": "ea46cf69cbc346c2b7a891dbdb2df289", "projectID": "5942472", "size": 8281, "entries": [{"data": {"formatted": "This is an example Python exception"}, "type": "message"}, {"data": {"frames": [{"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "build_msg", "rawFunction": null, "symbol": null, "context": [[298, " frames = stack"], [299, ""], [300, " data.update({"], [301, " 'sentry.interfaces.Stacktrace': {"], [302, " 'frames': get_stack_info(frames,"], [303, " transformer=self.transform)"], [304, " },"], [305, " })"], [306, ""], [307, " if 'sentry.interfaces.Stacktrace' in data:"], [308, " if self.include_paths:"]], "lineNo": 303, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "sourceLink": null, "vars": {"'culprit'": null, "'data'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'date'": "datetime.datetime(2013, 8, 13, 3, 8, 24, 880386)", "'event_id'": "'54a322436e1b47b88e239b78998ae742'", "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'frames'": "", "'handler'": "", "'k'": "'sentry.interfaces.Message'", "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'public_key'": null, "'result'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null, "'v'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "capture", "rawFunction": null, "symbol": null, "context": [[454, " if not self.is_enabled():"], [455, " return"], [456, ""], [457, " data = self.build_msg("], [458, " event_type, data, date, time_spent, extra, stack, tags=tags,"], [459, " **kwargs)"], [460, ""], [461, " self.send(**data)"], [462, ""], [463, " return (data.get('event_id'),)"], [464, ""]], "lineNo": 459, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "sourceLink": null, "vars": {"'data'": null, "'date'": null, "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "captureMessage", "rawFunction": null, "symbol": null, "context": [[572, " \"\"\""], [573, " Creates an event from ``message``."], [574, ""], [575, " >>> client.captureMessage('My event just happened!')"], [576, " \"\"\""], [577, " return self.capture('raven.events.Message', message=message, **kwargs)"], [578, ""], [579, " def captureException(self, exc_info=None, **kwargs):"], [580, " \"\"\""], [581, " Creates an event from an exception."], [582, ""]], "lineNo": 577, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "sourceLink": null, "vars": {"'kwargs'": {"'data'": null, "'extra'": {"'go_deeper'": ["[{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}]"], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'level'": 20, "'stack'": true, "'tags'": null}, "'message'": "'This is a test message generated using ``raven test``'", "'self'": ""}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "send_test_message", "rawFunction": null, "symbol": null, "context": [[72, " level=logging.INFO,"], [73, " stack=True,"], [74, " tags=options.get('tags', {}),"], [75, " extra={"], [76, " 'user': get_uid(),"], [77, " 'loadavg': get_loadavg(),"], [78, " },"], [79, " ))"], [80, ""], [81, " if client.state.did_fail():"], [82, " print('error!')"]], "lineNo": 77, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "sourceLink": null, "vars": {"'client'": "", "'data'": null, "'k'": "'secret_key'", "'options'": {"'data'": null, "'tags'": null}}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "main", "rawFunction": null, "symbol": null, "context": [[107, " print(\"Using DSN configuration:\")"], [108, " print(\" \", dsn)"], [109, " print()"], [110, ""], [111, " client = Client(dsn, include_paths=['raven'])"], [112, " send_test_message(client, opts.__dict__)"]], "lineNo": 112, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "sourceLink": null, "vars": {"'args'": ["'test'", "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'"], "'client'": "", "'dsn'": "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'", "'opts'": "", "'parser'": "", "'root'": ""}}], "framesOmitted": null, "registers": null, "hasSystemFrames": false}, "type": "stacktrace"}, {"data": {"apiTarget": null, "method": "GET", "url": "http://example.com/foo", "query": [["foo", "bar"]], "fragment": null, "data": {"hello": "world"}, "headers": [["Content-Type", "application/json"], ["Referer", "http://example.com"], ["User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"]], "cookies": [["foo", "bar"], ["biz", "baz"]], "env": {"ENV": "prod"}, "inferredContentType": "application/json"}, "type": "request"}], "dist": null, "message": "This is an example Python exception", "title": "This is an example Python exception", "location": null, "user": {"id": "1", "email": "sentry@example.com", "username": "sentry", "ip_address": "127.0.0.1", "name": "Sentry", "data": null}, "contexts": {"browser": {"name": "Chrome", "version": "28.0.1500", "type": "browser"}, "client_os": {"name": "Windows", "version": "8", "type": "os"}}, "sdk": null, "context": {"emptyList": [], "emptyMap": {}, "length": 10837790, "results": [1, 2, 3, 4, 5], "session": {"foo": "bar"}, "unauthorized": false, "url": "http://example.org/foo/bar/"}, "packages": {"my.package": "1.0.0"}, "type": "default", "metadata": {"in_app_frame_mix": "system-only", "title": "This is an example Python exception"}, "tags": [{"key": "browser", "value": "Chrome 28.0.1500"}, {"key": "browser.name", "value": "Chrome"}, {"key": "client_os", "value": "Windows 8"}, {"key": "client_os.name", "value": "Windows"}, {"key": "environment", "value": "prod"}, {"key": "level", "value": "error"}, {"key": "sample_event", "value": "yes"}, {"key": "server_name", "value": "web01.example.org"}, {"key": "url", "value": "http://example.com/foo"}, {"key": "user", "value": "id:1", "query": "user.id:\"1\""}], "platform": "python", "dateReceived": "2023-09-08T13:40:15.139108Z", "errors": [], "occurrence": null, "_meta": {"entries": {}, "message": null, "user": null, "contexts": null, "sdk": null, "context": null, "packages": null, "tags": {}}, "crashFile": null, "culprit": "raven.scripts.runner in main", "dateCreated": "2023-09-08T13:39:15Z", "fingerprints": ["3a2b45089d0211943e5a6645fb4cea3f"], "groupingConfig": {"id": "newstyle:2023-01-11", "enhancements": "eJybzDRxc15qeXFJZU6qlZGBkbGugaGuoeEEAHJMCAM"}}, "emitted_at": 1695036638369} +{"stream":"projects","data":{"id":"4505884239200256","slug":"flutter_test","name":"flutter_test","platform":"flutter","dateCreated":"2023-09-15T11:26:50.595810Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction"],"firstEvent":null,"firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf603f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641904} +{"stream":"projects","data":{"id":"4505884219408384","slug":"android_test_project","name":"android_test_project","platform":"android","dateCreated":"2023-09-15T11:21:48.131009Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction","releases"],"firstEvent":null,"firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#3f95bf","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641905} +{"stream":"projects","data":{"id":"6712547","slug":"demo-integration","name":"demo-integration","platform":"javascript-react","dateCreated":"2022-09-02T15:01:28.946777Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction"],"firstEvent":"2022-09-02T15:36:50.870000Z","firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf833f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641905} diff --git a/airbyte-integrations/connectors/source-sentry/main.py b/airbyte-integrations/connectors/source-sentry/main.py index 0844bc450390..1c7adc746e97 100644 --- a/airbyte-integrations/connectors/source-sentry/main.py +++ b/airbyte-integrations/connectors/source-sentry/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sentry import SourceSentry +from source_sentry.run import run if __name__ == "__main__": - source = SourceSentry() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index 00288df32e3a..c0fc7fe02f15 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -1,29 +1,34 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 - dockerImageTag: 0.3.0 - maxSecondsBetweenMessages: 64800 + dockerImageTag: 0.4.1 dockerRepository: airbyte/source-sentry + documentationUrl: https://docs.airbyte.com/integrations/sources/sentry githubIssueLabel: source-sentry icon: sentry.svg license: MIT + maxSecondsBetweenMessages: 64800 name: Sentry + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sentry registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/sentry + supportLevel: certified tags: - - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sentry/poetry.lock b/airbyte-integrations/connectors/source-sentry/poetry.lock new file mode 100644 index 000000000000..bbca06bb4b3a --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.2.tar.gz", hash = "sha256:5fe6b8b6f97059c812b50cdb37f193a3075aea15cf6c43dda2dab2b2fd059dd3"}, + {file = "airbyte_cdk-0.58.2-py3-none-any.whl", hash = "sha256:56cccc4275bbae65e3f26d90f10b5834600e911942a3624d91b341a7c57c2ec9"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "467cb54cd0e2b4a40734c43b04a3b90e2996e5b07aebe20307f83edea3e06973" diff --git a/airbyte-integrations/connectors/source-sentry/pyproject.toml b/airbyte-integrations/connectors/source-sentry/pyproject.toml new file mode 100644 index 000000000000..ba550dd78fae --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.4.1" +name = "source-sentry" +description = "Source implementation for Sentry." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/sentry" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_sentry" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.2" + +[tool.poetry.scripts] +source-sentry = "source_sentry.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-sentry/requirements.txt b/airbyte-integrations/connectors/source-sentry/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-sentry/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py deleted file mode 100644 index 18d172dcd942..000000000000 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", - "requests_mock~=1.9", -] - -setup( - name="source_sentry", - description="Source implementation for Sentry.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py new file mode 100644 index 000000000000..acf82b00a03d --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sentry import SourceSentry + + +def run(): + source = SourceSentry() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json index 219fe41cace1..70fff3323463 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json @@ -4,24 +4,24 @@ "properties": { "type": { "type": ["null", "string"] }, "eventID": { - "type": "string" + "type": ["string", "null"] }, "tags": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "value": { - "type": "string" + "type": ["string", "null"] }, "key": { - "type": "string" + "type": ["string", "null"] } } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "user": { @@ -43,32 +43,32 @@ "type": ["null", "object"], "properties": { "isStaff": { - "type": "boolean" + "type": ["boolean", "null"] } } }, "id": { - "type": "string" + "type": ["string", "null"] } } }, "message": { - "type": "string" + "type": ["string", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "platform": { - "type": "string" + "type": ["string", "null"] }, "event.type": { - "type": "string" + "type": ["string", "null"] }, "groupID": { - "type": "string" + "type": ["string", "null"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "_meta": { "type": ["null", "object"], @@ -95,7 +95,7 @@ "length": { "type": ["null", "integer"] }, "results": { "type": ["null", "array"], - "items": { "type": "integer" } + "items": { "type": ["integer", "null"] } }, "session": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json index c5660d0dc62c..b56907d69728 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json @@ -3,43 +3,43 @@ "type": "object", "properties": { "annotations": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "assignedTo": { "type": ["null", "object"] }, "count": { - "type": "string" + "type": ["string", "null"] }, "culprit": { - "type": "string" + "type": ["string", "null"] }, "firstSeen": { - "type": "string" + "type": ["string", "null"] }, "hasSeen": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "isSubscribed": { - "type": "boolean" + "type": ["boolean", "null"] }, "lastSeen": { - "type": "string" + "type": ["string", "null"] }, "level": { - "type": "string" + "type": ["string", "null"] }, "logger": { "type": ["null", "string"] @@ -47,46 +47,46 @@ "metadata": { "anyOf": [ { - "type": "object", + "type": ["object", "null"], "properties": { "title": { - "type": "string" + "type": ["string", "null"] } } }, { - "type": "object", + "type": ["object", "null"], "properties": { "filename": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "value": { - "type": "string" + "type": ["string", "null"] } } } ] }, "numComments": { - "type": "integer" + "type": ["integer", "null"] }, "permalink": { - "type": "string" + "type": ["string", "null"] }, "project": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } }, @@ -94,40 +94,40 @@ "type": ["null", "string"] }, "shortId": { - "type": "string" + "type": ["string", "null"] }, "stats": { - "type": "object", + "type": ["object", "null"], "properties": { "24h": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "number" + "type": ["number", "null"] } } } } }, "status": { - "type": "string", + "type": ["string", "null"], "enum": ["resolved", "unresolved", "ignored"] }, "statusDetails": { - "type": "object" + "type": ["object", "null"] }, "subscriptionDetails": { "type": ["null", "object"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "userCount": { - "type": "integer" + "type": ["integer", "null"] }, "isUnhandled": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json index a78ca92f0f5f..16e132f4bfff 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json @@ -3,101 +3,104 @@ "type": "object", "properties": { "allowedDomains": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "color": { - "type": "string" + "type": ["string", "null"] }, "dataScrubber": { - "type": "boolean" + "type": ["boolean", "null"] }, "dataScrubberDefaults": { - "type": "boolean" + "type": ["boolean", "null"] }, "dateCreated": { - "type": "string" + "type": ["string", "null"] }, "defaultEnvironment": { "type": ["null", "string"] }, "digestsMaxDelay": { - "type": "integer" + "type": ["integer", "null"] }, "digestsMinDelay": { - "type": "integer" + "type": ["integer", "null"] }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "firstEvent": { "type": ["null", "string"] }, "hasAccess": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isInternal": { - "type": "boolean" + "type": ["boolean", "null"] }, "isMember": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "latestRelease": { "type": ["null", "object"], "properties": { "authors": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "email": { - "type": "string" + "type": ["string", "null"] } } } }, "commitCount": { - "type": "integer" + "type": ["integer", "null"] }, "data": { - "type": "object" + "type": ["object", "null"] }, "dateCreated": { - "type": "string" + "type": ["string", "null"] }, "dateReleased": { "type": ["null", "string"] }, "deployCount": { - "type": "integer" + "type": ["integer", "null"] }, "firstEvent": { "type": ["null", "string"] @@ -112,21 +115,21 @@ "type": ["null", "string"] }, "newGroups": { - "type": "integer" + "type": ["integer", "null"] }, "owner": { "type": ["null", "string"] }, "projects": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } } @@ -135,86 +138,89 @@ "type": ["null", "string"] }, "shortVersion": { - "type": "string" + "type": ["string", "null"] }, "url": { "type": ["null", "string"] }, "version": { - "type": "string" + "type": ["string", "null"] } } }, "name": { - "type": "string" + "type": ["string", "null"] }, "options": { - "type": "object", + "type": ["object", "null"], "properties": { "feedback:branding": { - "type": "boolean" + "type": ["boolean", "null"] }, "filters:blacklisted_ips": { - "type": "string" + "type": ["string", "null"] }, "filters:error_messages": { - "type": "string" + "type": ["string", "null"] }, "filters:releases": { - "type": "string" + "type": ["string", "null"] }, "sentry:csp_ignored_sources": { - "type": "string" + "type": ["string", "null"] }, "sentry:csp_ignored_sources_defaults": { - "type": "boolean" + "type": ["boolean", "null"] }, "sentry:reprocessing_active": { - "type": "boolean" + "type": ["boolean", "null"] } } }, "organization": { - "type": "object", + "type": ["object", "null"], "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "id": { - "type": "string" + "type": ["string", "null"] }, "isEarlyAdopter": { - "type": "boolean" + "type": ["boolean", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "require2FA": { - "type": "boolean" + "type": ["boolean", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] } } } @@ -224,91 +230,116 @@ "type": ["null", "string"] }, "platforms": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "plugins": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { + "altIsSentryApp": {}, "assets": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "author": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "url": { - "type": "string" + "type": ["string", "null"] } } }, "canDisable": { - "type": "boolean" + "type": ["boolean", "null"] }, "contexts": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "description": { - "type": "string" + "type": ["string", "null"] }, + "deprecationDate": {}, "doc": { - "type": "string" + "type": ["string", "null"] }, "enabled": { - "type": "boolean" + "type": ["boolean", "null"] + }, + "features": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "featureDescriptions": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "additionalProperties": true + } }, + "firstPartyAlternative": {}, "hasConfiguration": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] + }, + "isDeprecated": { + "type": ["boolean", "null"] + }, + "isHidden": { + "type": ["boolean", "null"] }, "isTestable": { - "type": "boolean" + "type": ["boolean", "null"] }, "metadata": { - "type": "object" + "type": ["object", "null"], + "additionalProperties": true }, "name": { - "type": "string" + "type": ["string", "null"] }, "resourceLinks": { "type": ["null", "array"], "items": { - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "title": { - "type": "string" + "type": ["string", "null"] }, "url": { - "type": "string" + "type": ["string", "null"] } } } }, "shortName": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "version": { "type": ["null", "string"] @@ -317,86 +348,86 @@ } }, "processingIssues": { - "type": "integer" + "type": ["integer", "null"] }, "relayPiiConfig": { "type": ["null", "string"] }, "resolveAge": { - "type": "integer" + "type": ["integer", "null"] }, "safeFields": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "scrapeJavaScript": { - "type": "boolean" + "type": ["boolean", "null"] }, "scrubIPAddresses": { - "type": "boolean" + "type": ["boolean", "null"] }, "securityToken": { - "type": "string" + "type": ["string", "null"] }, "securityTokenHeader": { "type": ["null", "string"] }, "sensitiveFields": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string" + "type": ["string", "null"] }, "storeCrashReports": { "type": ["null", "boolean"] }, "subjectPrefix": { - "type": "string" + "type": ["string", "null"] }, "subjectTemplate": { - "type": "string" + "type": ["string", "null"] }, "team": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } }, "teams": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } } }, "verifySSL": { - "type": "boolean" + "type": ["boolean", "null"] }, "access": { "type": ["null", "array"], @@ -439,9 +470,15 @@ "hasProfiles": { "type": ["null", "boolean"] }, "hasReplays": { "type": ["null", "boolean"] }, "hasSessions": { "type": ["null", "boolean"] }, + "hasFeedbacks": { "type": ["null", "boolean"] }, + "hasNewFeedbacks": { "type": ["null", "boolean"] }, + "hasCustomMetrics": { "type": ["null", "boolean"] }, "recapServerUrl": { "type": ["null", "string"] }, "secondaryGroupingConfig": { "type": ["null", "string"] }, "secondaryGroupingExpiry": { "type": ["null", "integer"] }, - "symbolSources": { "type": ["null", "string"] } + "symbolSources": { "type": ["null", "string"] }, + "stats": {}, + "transactionStats": {}, + "sessionStats": {} } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json index 2883aad1db39..6abc90ab49e2 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json @@ -3,104 +3,124 @@ "type": "object", "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "color": { - "type": "string" + "type": ["string", "null"] }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "firstEvent": { "type": ["null", "string"] }, "hasAccess": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isInternal": { - "type": "boolean" + "type": ["boolean", "null"] }, "isMember": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "organization": { - "type": "object", + "type": ["object", "null"], "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["string", "null"] } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "id": { - "type": "string" + "type": ["string", "null"] }, "isEarlyAdopter": { - "type": "boolean" + "type": ["boolean", "null"] + }, + "hasAuthProvider": { + "type": ["boolean", "null"] + }, + "links": { + "type": ["object", "null"], + "properties": { + "organizationUrl": { + "type": ["string", "null"] + }, + "regionUrl": { + "type": ["string", "null"] + } + } }, "name": { - "type": "string" + "type": ["string", "null"] }, "require2FA": { - "type": "boolean" + "type": ["boolean", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] } } }, "requireEmailVerification": { - "type": "boolean" + "type": ["boolean", "null"] }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } } } @@ -109,10 +129,10 @@ "type": ["null", "string"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string", + "type": ["string", "null"], "enum": ["active", "disabled", "pending_deletion", "deletion_in_progress"] }, "access": { @@ -126,6 +146,9 @@ "hasProfiles": { "type": ["null", "boolean"] }, "hasReplays": { "type": ["null", "boolean"] }, "hasSessions": { "type": ["null", "boolean"] }, + "hasFeedbacks": { "type": ["null", "boolean"] }, + "hasNewFeedbacks": { "type": ["null", "boolean"] }, + "hasCustomMetrics": { "type": ["null", "boolean"] }, "hasMinifiedStackTrace": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json index d6ea85853ac0..5ecc0f17f0ef 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 2cd4f4d516cf..1482228b362a 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -118,7 +118,7 @@ def state(self, value: Mapping[str, Any]): class Events(SentryIncremental): """ - Docs: https://docs.sentry.io/api/events/list-a-projects-events/ + Docs: https://docs.sentry.io/api/events/list-a-projects-error-events/ """ primary_key = "id" diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 1ac42a5217e2..87376d158902 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -2,11 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock, patch import pendulum as pdm import pytest -from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination +import requests +from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryIncremental, SentryStreamPagination INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} @@ -110,6 +111,39 @@ def test_project_detail_request_params(): expected = {} assert stream.request_params(stream_state=None, next_page_token=None) == expected +def test_issues_parse_response(mocker): + with patch('source_sentry.streams.Issues._get_cursor_value') as mock_get_cursor_value: + stream = Issues(**INIT_ARGS) + mock_get_cursor_value.return_value = "time" + state = {} + response = requests.Response() + mocker.patch.object(response, "json", return_value=[{"id": "1"}]) + result = list(stream.parse_response(response, state)) + assert result[0] == {"id": "1"} + +def test_project_detail_parse_response(mocker): + stream = ProjectDetail(organization="test_org", project="test_proj", hostname="sentry.io") + response = requests.Response() + response.json = Mock(return_value={"id": "1"}) + result = list(stream.parse_response(response)) + assert result[0] == {"id": "1"} + +class MockSentryIncremental(SentryIncremental): + def path(): + return '/test/path' + +def test_sentry_incremental_parse_response(mocker): + with patch('source_sentry.streams.SentryIncremental.filter_by_state') as mock_filter_by_state: + stream = MockSentryIncremental(hostname="sentry.io") + mock_filter_by_state.return_value = True + state = None + response = requests.Response() + mocker.patch.object(response, "json", return_value=[{"id": "1"}]) + mock_filter_by_state.return_value = iter(response.json()) + result = list(stream.parse_response(response, state)) + print(result) + assert result[0] == {"id": "1"} + @pytest.mark.parametrize( "state, expected", diff --git a/airbyte-integrations/connectors/source-serpstat/main.py b/airbyte-integrations/connectors/source-serpstat/main.py index 92fb7edc0474..75c87dbf86f9 100644 --- a/airbyte-integrations/connectors/source-serpstat/main.py +++ b/airbyte-integrations/connectors/source-serpstat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_serpstat import SourceSerpstat +from source_serpstat.run import run if __name__ == "__main__": - source = SourceSerpstat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-serpstat/metadata.yaml b/airbyte-integrations/connectors/source-serpstat/metadata.yaml index e764c9c80059..849ad93a11da 100644 --- a/airbyte-integrations/connectors/source-serpstat/metadata.yaml +++ b/airbyte-integrations/connectors/source-serpstat/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.serpstat.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-serpstat registries: oss: enabled: true @@ -18,5 +22,5 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/serpstat tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-serpstat/setup.py b/airbyte-integrations/connectors/source-serpstat/setup.py index 42ab32a171b0..9c7f53024268 100644 --- a/airbyte-integrations/connectors/source-serpstat/setup.py +++ b/airbyte-integrations/connectors/source-serpstat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-serpstat=source_serpstat.run:run", + ], + }, name="source_serpstat", description="Source implementation for Serpstat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py b/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py new file mode 100644 index 000000000000..3dc6051ee8f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_serpstat import SourceSerpstat + + +def run(): + source = SourceSerpstat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py index c2b1e7afb5b3..9e6488da04cc 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/main.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sftp_bulk import SourceFtp +from source_sftp_bulk.run import run if __name__ == "__main__": - source = SourceFtp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml index 9adbfaedfa33..6bddeaa476e5 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml @@ -12,6 +12,10 @@ data: icon: sftp.svg license: MIT name: SFTP Bulk + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sftp-bulk registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sftp-bulk/setup.py b/airbyte-integrations/connectors/source-sftp-bulk/setup.py index 6d9d1990f634..282a57ca1fac 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/setup.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/setup.py @@ -16,13 +16,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "docker==5.0.3"] setup( + entry_points={ + "console_scripts": [ + "source-sftp-bulk=source_sftp_bulk.run:run", + ], + }, name="source_sftp_bulk", description="Source implementation for SFTP Bulk.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py new file mode 100644 index 000000000000..c3e00b8100cd --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sftp_bulk import SourceFtp + + +def run(): + source = SourceFtp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sftp/build.gradle b/airbyte-integrations/connectors/source-sftp/build.gradle index da35e916d0e2..8daa60fa57d1 100644 --- a/airbyte-integrations/connectors/source-sftp/build.gradle +++ b/airbyte-integrations/connectors/source-sftp/build.gradle @@ -1,23 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.sftp.SftpSource' } @@ -26,5 +16,5 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.2' implementation 'com.jcraft:jsch:0.1.55' - testImplementation libs.testcontainers + testImplementation 'org.testcontainers:testcontainers:1.19.4' } diff --git a/airbyte-integrations/connectors/source-sftp/metadata.yaml b/airbyte-integrations/connectors/source-sftp/metadata.yaml index dd9961561c93..d3688c9bb8d2 100644 --- a/airbyte-integrations/connectors/source-sftp/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: a827c52e-791c-4135-a245-e233c5255199 - dockerImageTag: 0.1.2 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-sftp documentationUrl: https://docs.airbyte.com/integrations/sources/sftp githubIssueLabel: source-sftp diff --git a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpClient.java b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpClient.java index 788e8bb89eb8..dfe006e48023 100644 --- a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpClient.java +++ b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpClient.java @@ -90,6 +90,7 @@ public boolean isConnected() { return channelSftp != null && channelSftp.isConnected(); } + @SuppressWarnings("rawtypes") public Vector lsFile(SupportedFileExtension fileExtension) { try { return channelSftp.ls("*." + fileExtension.typeName); diff --git a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpCommand.java b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpCommand.java index 69d67913bfcc..8b4df1a25105 100644 --- a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpCommand.java +++ b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpCommand.java @@ -79,6 +79,7 @@ public Map getFilesSchemas() { return fileSchemas; } + @SuppressWarnings("unchecked") private Set getFileNames() { checkIfConnected(); Vector entries = new Vector<>(); diff --git a/airbyte-integrations/connectors/source-shopify/README.md b/airbyte-integrations/connectors/source-shopify/README.md index 1f99a2f8acef..86eeb80672b5 100644 --- a/airbyte-integrations/connectors/source-shopify/README.md +++ b/airbyte-integrations/connectors/source-shopify/README.md @@ -1,125 +1,55 @@ -# Shopify Source +# Shopify source connector -This is the repository for the Shopify CDK source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/shopify). -## Local development +This is the repository for the Shopify source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/shopify). -#### Minimum Python version required `= 3.9.0` +## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python3 -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Once you finished with installing python requirements: -``` -deactivate +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shopify) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shopify/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/shopify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shopify/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source shopify test creds` -and place them into `secrets/config.json`. ### Locally running the connector -Use your .venv inside your connector in order to proceed: ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json --state integration_tests/state.json +poetry run source-shopify spec +poetry run source-shopify check --config secrets/config.json +poetry run source-shopify discover --config secrets/config.json +poetry run source-shopify read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-shopify build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-shopify:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-shopify:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-shopify:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-shopify:dev . -# Running the spec command against your patched connector -docker run airbyte/source-shopify:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-shopify:dev spec @@ -128,30 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shopify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-shopify test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-shopify test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/shopify.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/shopify.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index 4b1a575ff8ae..bb0b185cfeda 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -27,7 +27,7 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" - timeout_seconds: 3600 + timeout_seconds: 4800 expect_records: path: "integration_tests/expected_records.jsonl" empty_streams: @@ -68,12 +68,12 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 3600 + timeout_seconds: 14400 full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 4800 ignored_fields: products: - name: variants/*/updated_at diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl index 937e24d35c87..22205205125d 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl @@ -25,7 +25,7 @@ {"stream": "inventory_levels", "data": {"inventory_item_id": 42185194668221, "location_id": 63590301885, "available": 12, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194668221", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194668221"}, "emitted_at": 1697194698578} {"stream": "inventory_levels", "data": {"inventory_item_id": 42185194700989, "location_id": 63590301885, "available": 3, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194700989", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194700989"}, "emitted_at": 1697194698579} {"stream": "inventory_levels", "data": {"inventory_item_id": 42185194733757, "location_id": 63590301885, "available": 38, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194733757", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194733757"}, "emitted_at": 1697194698579} -{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-02-25T16:20:00-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194701440} +{"stream":"locations","data":{"id":63590301885,"name":"Heroiv UPA 72","address1":"Heroiv UPA 72","address2":"","city":"Lviv","zip":"30100","province":null,"country":"UA","phone":"","created_at":"2021-06-22T18:00:29-07:00","updated_at":"2023-11-28T07:08:27-08:00","country_code":"UA","country_name":"Ukraine","province_code":null,"legacy":false,"active":true,"admin_graphql_api_id":"gid://shopify/Location/63590301885","localized_country_name":"Ukraine","localized_province_name":null,"shop_url":"airbyte-integration-test"},"emitted_at":1704314548257} {"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194703693} {"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194703694} {"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194704159} @@ -36,8 +36,8 @@ {"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "key": "test_definition_list_1", "value": "Teste\n", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194711312} {"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "key": "test_definition", "value": "Taster", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194711313} {"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 929019691197, "created_at": "2023-04-24T07:18:06-07:00", "updated_at": "2023-04-24T07:18:06-07:00", "owner_resource": "draft_order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194714876} -{"stream": "metafield_locations", "data": {"id": 21524407255229, "namespace": "inventory", "key": "warehouse_2", "value": "234", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:27-07:00", "updated_at": "2022-10-12T02:21:27-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407255229", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194717772} -{"stream": "metafield_locations", "data": {"id": 21524407681213, "namespace": "inventory", "key": "warehouse_233", "value": "564", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:35-07:00", "updated_at": "2022-10-12T02:21:35-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407681213", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194717773} +{"stream":"metafield_locations","data":{"id":21524407255229,"namespace":"inventory","key":"warehouse_2","value":"234","description":null,"owner_id":63590301885,"created_at":"2022-10-12T02:21:27-07:00","updated_at":"2022-10-12T02:21:27-07:00","owner_resource":"location","type":"number_integer","admin_graphql_api_id":"gid://shopify/Metafield/21524407255229","shop_url":"airbyte-integration-test"},"emitted_at":1704314554082} +{"stream":"metafield_locations","data":{"id":21524407681213,"namespace":"inventory","key":"warehouse_233","value":"564","description":null,"owner_id":63590301885,"created_at":"2022-10-12T02:21:35-07:00","updated_at":"2022-10-12T02:21:35-07:00","owner_resource":"location","type":"number_integer","admin_graphql_api_id":"gid://shopify/Metafield/21524407681213","shop_url":"airbyte-integration-test"},"emitted_at":1704314554084} {"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "key": "purchase_order", "value": "trtrtr", "description": null, "owner_id": 4147980107965, "created_at": "2023-04-13T05:09:08-07:00", "updated_at": "2023-04-13T05:09:08-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194720313} {"stream": "metafield_orders", "data": {"id": 22365749805245, "namespace": "my_fields", "key": "purchase_order", "value": "Test Draft Order Metafield", "description": null, "owner_id": 3935377129661, "created_at": "2023-04-14T03:52:40-07:00", "updated_at": "2023-04-14T03:52:40-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365749805245", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194720780} {"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1697194723743} diff --git a/airbyte-integrations/connectors/source-shopify/main.py b/airbyte-integrations/connectors/source-shopify/main.py index a45dc5aaf611..aca13eebbb25 100644 --- a/airbyte-integrations/connectors/source-shopify/main.py +++ b/airbyte-integrations/connectors/source-shopify/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_shopify import SourceShopify +from source_shopify.run import run if __name__ == "__main__": - source = SourceShopify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml index 2adaad09d627..0eb582dd457d 100644 --- a/airbyte-integrations/connectors/source-shopify/metadata.yaml +++ b/airbyte-integrations/connectors/source-shopify/metadata.yaml @@ -7,17 +7,21 @@ data: - ${shop}.myshopify.com - shopify.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 - dockerImageTag: 1.1.4 + dockerImageTag: 1.1.8 dockerRepository: airbyte/source-shopify documentationUrl: https://docs.airbyte.com/integrations/sources/shopify githubIssueLabel: source-shopify icon: shopify.svg license: ELv2 name: Shopify + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-shopify registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-shopify/poetry.lock b/airbyte-integrations/connectors/source-shopify/poetry.lock new file mode 100644 index 000000000000..faca58b052f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/poetry.lock @@ -0,0 +1,1040 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.59.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.59.0.tar.gz", hash = "sha256:2f7bc07556cc7f42f0daf41d09be08fd22102864d087a27c8999f6f13fe67aad"}, + {file = "airbyte_cdk-0.59.0-py3-none-any.whl", hash = "sha256:94c561c053b8be3a66bfefe420812ced9237403441249408e2af5445214a6f7b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "8.0.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, + {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "sgqlc" +version = "16.3" +description = "Simple GraphQL Client" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "sgqlc-16.3-py3-none-any.whl", hash = "sha256:89d468386a4ba4b5ade991623228b6fb0a25bea1f25643ccac130fb3ef565b72"}, + {file = "sgqlc-16.3.tar.gz", hash = "sha256:be08857775aa3e65ef7b2c1f0cdcc65dd5794907b162b393c189187fee664558"}, +] + +[package.dependencies] +graphql-core = ">=3.1.7,<4.0.0" + +[package.extras] +requests = ["requests"] +websocket = ["websocket-client"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "71f6b166e4220683dea93a273110eca25f9680be5842ee802d78a9583f31ee2f" diff --git a/airbyte-integrations/connectors/source-shopify/pyproject.toml b/airbyte-integrations/connectors/source-shopify/pyproject.toml new file mode 100644 index 000000000000..65ae0c202895 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.1.8" +name = "source-shopify" +description = "Source CDK implementation for Shopify." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/shopify" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_shopify" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.59.0" +sgqlc = "==16.3" + +[tool.poetry.scripts] +source-shopify = "source_shopify.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.12.0" +pytest = "^8.0.0" diff --git a/airbyte-integrations/connectors/source-shopify/requirements.txt b/airbyte-integrations/connectors/source-shopify/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-shopify/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-shopify/setup.py b/airbyte-integrations/connectors/source-shopify/setup.py deleted file mode 100644 index d1ec88ea5e7b..000000000000 --- a/airbyte-integrations/connectors/source-shopify/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "sgqlc~=16.0"] - -TEST_REQUIREMENTS = [ - "pytest", - "pytest-mock", - "requests-mock", -] - -setup( - name="source_shopify", - description="Source CDK implementation for Shopify.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py b/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py index e729c74be0f7..e8d38c64cdff 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py @@ -26,10 +26,12 @@ def _camel_to_snake(camel_case: str): def get_query_products(first: int, filter_field: str, filter_value: str, next_page_token: Optional[str]): op = sgqlc.operation.Operation(_schema_root.query_type) snake_case_filter_field = _camel_to_snake(filter_field) - if next_page_token: - products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'", after=next_page_token) - else: - products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'") + products_args = { + "first": first, + "query": f"{snake_case_filter_field}:>'{filter_value}'" if filter_value else None, + "after": next_page_token, + } + products = op.products(**products_args) products.nodes.id() products.nodes.title() products.nodes.updated_at() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/run.py b/airbyte-integrations/connectors/source-shopify/source_shopify/run.py new file mode 100644 index 000000000000..9c13e936ca71 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceShopify + + +def run(): + source = SourceShopify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 8c37b5f18106..8225aa5f08c6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -78,6 +78,8 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> else: params["order"] = f"{self.order_field} asc" params[self.filter_field] = self.default_filter_field_value + if self.config.get("end_date") and self.filter_field == "updated_at_min": + params["updated_at_max"] = self.config.get("end_date") return params @limiter.balance_rate_limit() diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py new file mode 100644 index 000000000000..d1a9f02de29b --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from source_shopify.graphql import get_query_products + + +@pytest.mark.parametrize( + "page_size, filter_value, next_page_token, expected_query", + [ + (100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + (200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + (250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + ], +) +def test_get_query_products(page_size, filter_value, next_page_token, expected_query): + assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query diff --git a/airbyte-integrations/connectors/source-shortio/main.py b/airbyte-integrations/connectors/source-shortio/main.py index 0c6be2ce4469..eaef1e24f90d 100644 --- a/airbyte-integrations/connectors/source-shortio/main.py +++ b/airbyte-integrations/connectors/source-shortio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_shortio import SourceShortio +from source_shortio.run import run if __name__ == "__main__": - source = SourceShortio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-shortio/metadata.yaml b/airbyte-integrations/connectors/source-shortio/metadata.yaml index ad64fec0533c..b14f05840fa2 100644 --- a/airbyte-integrations/connectors/source-shortio/metadata.yaml +++ b/airbyte-integrations/connectors/source-shortio/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - https://api.short.io - https://api-v2.short.cm + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-shortio registries: oss: enabled: true @@ -23,7 +27,7 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/shortio tags: - language:python - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-shortio/setup.py b/airbyte-integrations/connectors/source-shortio/setup.py index 6c8cbba1b880..d0b2d6cc581c 100644 --- a/airbyte-integrations/connectors/source-shortio/setup.py +++ b/airbyte-integrations/connectors/source-shortio/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-shortio=source_shortio.run:run", + ], + }, name="source_shortio", description="Source implementation for Shortio.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/run.py b/airbyte-integrations/connectors/source-shortio/source_shortio/run.py new file mode 100644 index 000000000000..dce618a444e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_shortio import SourceShortio + + +def run(): + source = SourceShortio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-slack/README.md b/airbyte-integrations/connectors/source-slack/README.md index 1b407f1aed59..9f21fa72ec44 100644 --- a/airbyte-integrations/connectors/source-slack/README.md +++ b/airbyte-integrations/connectors/source-slack/README.md @@ -1,67 +1,91 @@ -# Orbit Source +# Slack source connector -This is the repository for the Orbit configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/orbit). + +This is the repository for the Slack source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/slack). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/orbit) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orbit/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source orbit test creds` -and place them into `secrets/config.json`. -### Locally running the connector docker image +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name source-orbit build +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/slack) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_slack/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-slack spec +poetry run source-slack check --config secrets/config.json +poetry run source-slack discover --config secrets/config.json +poetry run source-slack read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -An image will be built with the tag `airbyte/source-orbit:dev`. +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -**Via `docker build`:** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-orbit:dev . +airbyte-ci connectors --name=source-slack build ``` -#### Run +An image will be available on your host with the tag `airbyte/source-slack:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-orbit:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orbit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-slack:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-slack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-slack test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` -### Publishing a new version of the connector +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-slack test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/slack.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/slack.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 0d248e4409e1..b94b7cf70bc0 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -26,6 +26,10 @@ acceptance_tests: expect_records: path: "integration_tests/expected_records.jsonl" timeout_seconds: 4800 + ignored_fields: + channels: + - name: updated + bypass_reason: Value can change while interacting with data full_refresh: tests: - config_path: "secrets/config.json" @@ -38,3 +42,6 @@ acceptance_tests: future_state: future_state_path: "integration_tests/abnormal_state.json" timeout_seconds: 4800 + # When running multiple syncs in a row, we may get the same record set because of a lookback window. + # This may fail the test but this is expected behavior of the connector. + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl index 9ddbf42168e5..e5b6c7911376 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl @@ -1,29 +1,15 @@ -{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123063, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123075, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123086, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171463} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171464} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171464} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171629} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} -{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695111199104} -{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695111707952} -{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695110320605} -{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1695110320606} -{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1695110320606} -{"stream": "threads", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695112005658} -{"stream": "threads", "data": {"client_msg_id": "e1e2d142-a0dd-4587-86e3-2dcb439ead82", "type": "message", "text": "<@U04LY6NARHU> Test test", "user": "U04L65GPMKN", "ts": "1683104515.919709", "blocks": [{"type": "rich_text", "block_id": "xVnQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " Test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104515.919709}, "emitted_at": 1695112005659} -{"stream": "threads", "data": {"client_msg_id": "ffccbb24-8dd6-476d-87bf-65e5fa033cb9", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "Lvl", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104528.084359}, "emitted_at": 1695112005659} -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354906} -{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} -{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} -{"stream": "users", "data": {"id": "U04L2LC770E", "team_id": "T04KX3KDDU6", "name": "deactivateduser521176", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090821, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} -{"stream": "users", "data": {"id": "U04L69BPZFX", "team_id": "T04KX3KDDU6", "name": "deactivateduser839125", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png", "image_32": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png", "image_48": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png", "image_72": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png", "image_192": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png", "image_512": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811889, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} -{"stream": "users", "data": {"id": "U04L94Y2JPM", "team_id": "T04KX3KDDU6", "name": "deactivateduser962255", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png", "image_32": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png", "image_48": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png", "image_72": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png", "image_192": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png", "image_512": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090815, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04LMS8F7JM", "team_id": "T04KX3KDDU6", "name": "deactivateduser421996", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811683, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354912} +{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826841, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735549} +{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826857, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735550} +{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826875, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735550} +{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736171} +{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} +{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} +{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707568738170} +{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1707569060525} +{"stream": "channel_messages", "data": {"type": "message", "subtype": "reminder_add", "text": " set up a reminder \u201ctest reminder\u201d in this channel at 9AM tomorrow, Eastern European Summer Time.", "user": "U04L65GPMKN", "ts": "1695814864.744249", "channel_id": "C04LTCM2Y56", "float_ts": 1695814864.744249}, "emitted_at": 1707569208689} +{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707569354932} +{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1707569354933} +{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1707569354933} +{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1707569357949} +{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} +{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} diff --git a/airbyte-integrations/connectors/source-slack/main.py b/airbyte-integrations/connectors/source-slack/main.py index 735ad5e72296..b2ff9c851163 100644 --- a/airbyte-integrations/connectors/source-slack/main.py +++ b/airbyte-integrations/connectors/source-slack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_slack import SourceSlack +from source_slack.run import run if __name__ == "__main__": - source = SourceSlack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-slack/metadata.yaml b/airbyte-integrations/connectors/source-slack/metadata.yaml index d4ab7220630a..06e16de8b167 100644 --- a/airbyte-integrations/connectors/source-slack/metadata.yaml +++ b/airbyte-integrations/connectors/source-slack/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.9 dockerRepository: airbyte/source-slack documentationUrl: https://docs.airbyte.com/integrations/sources/slack githubIssueLabel: source-slack icon: slack.svg license: MIT name: Slack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-slack registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-slack/poetry.lock b/airbyte-integrations/connectors/source-slack/poetry.lock new file mode 100644 index 000000000000..608c63e12b1b --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.7" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.7.tar.gz", hash = "sha256:00e379e2379b38683992027114a2190f49befec8cbac67d0a2c907786111e77b"}, + {file = "airbyte_cdk-0.58.7-py3-none-any.whl", hash = "sha256:09b31d32899cc6dc91e39716e8d1601503a7884d837752e683d1e3ef7dfe73be"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "6d6c74c129dad767e4712df44cb1a80c195a3a27c8b39861a86d80acc72a19ed" diff --git a/airbyte-integrations/connectors/source-slack/pyproject.toml b/airbyte-integrations/connectors/source-slack/pyproject.toml new file mode 100644 index 000000000000..b7b6103bf52d --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.9" +name = "source-slack" +description = "Source implementation for Slack." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/slack" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_slack" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +pendulum = "==2.1.2" +airbyte-cdk = "==0.58.7" + +[tool.poetry.scripts] +source-slack = "source_slack.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-slack/requirements.txt b/airbyte-integrations/connectors/source-slack/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-slack/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py deleted file mode 100644 index 66fc79eeca33..000000000000 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", -] - -setup( - name="source_slack", - description="Source implementation for Slack.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=["airbyte-cdk", "pendulum>=2,<3"], - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/run.py b/airbyte-integrations/connectors/source-slack/source_slack/run.py new file mode 100644 index 000000000000..14caa9ab08e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_slack import SourceSlack + + +def run(): + source = SourceSlack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 38e48b0dd61c..f85b210d0d40 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -10,6 +10,36 @@ "properties": { "type": { "type": ["null", "string"] + }, + "block_id": { + "type": ["null", "string"] + }, + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "text": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "type": { + "type": ["null", "string"] + } + } + } } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index dee131fed53f..2571351507c5 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -49,6 +49,39 @@ "properties": { "type": { "type": ["null", "string"] + }, + "block_id": { + "type": ["null", "string"] + }, + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "text": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + } + } + } + }, + "type": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json index d13dce63ee13..47a1d6e9da1a 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json @@ -10,6 +10,9 @@ "type": "object", "additionalProperties": true, "properties": { + "always_active": { + "type": ["null", "boolean"] + }, "avatar_hash": { "type": "string" }, @@ -34,6 +37,13 @@ "email": { "type": "string" }, + "fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "huddle_state": { + "type": "string" + }, "image_24": { "type": "string" }, @@ -75,6 +85,18 @@ }, "skype": { "type": "string" + }, + "status_emoji_display_info": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "status_expiration": { + "type": ["null", "integer"] + }, + "status_text_canonical": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index c677b4bdbca7..e785114f865f 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import copy + from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -14,7 +14,9 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from pendulum import DateTime, Period +from pendulum import DateTime + +from .utils import chunk_date_range class SlackStream(HttpStream, ABC): @@ -80,13 +82,67 @@ def should_retry(self, response: requests.Response) -> bool: return response.status_code == requests.codes.REQUEST_TIMEOUT or super().should_retry(response) +class JoinChannelsStream(HttpStream): + """ + This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. + Its responses should only be logged for debugging reasons, not read as records. + """ + + url_base = "https://slack.com/api/" + http_method = "POST" + primary_key = "id" + + def __init__(self, channel_filter: List[str] = None, **kwargs): + self.channel_filter = channel_filter or [] + super().__init__(**kwargs) + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable: + """ + Override to simply indicate that the specific channel was joined successfully. + This method should not return any data, but should return an empty iterable. + """ + self.logger.info(f"Successfully joined channel: {stream_slice['channel_name']}") + return [] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + The pagination is not applicable to this Service Stream. + """ + return None + + def path(self, **kwargs) -> str: + return "conversations.join" + + def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: + return {"channel": stream_slice["channel"]} + + class ChanneledStream(SlackStream, ABC): """Slack stream with channel filter""" - def __init__(self, channel_filter: List[str] = [], **kwargs): + def __init__(self, channel_filter: List[str] = [], join_channels: bool = False, **kwargs): self.channel_filter = channel_filter + self.join_channels = join_channels + self.kwargs = kwargs super().__init__(**kwargs) + @property + def join_channels_stream(self) -> JoinChannelsStream: + return JoinChannelsStream(authenticator=self.kwargs.get("authenticator"), channel_filter=self.channel_filter) + + def should_join_to_channel(self, channel: Mapping[str, Any]) -> bool: + """ + The `is_member` property indicates whether or not the API Bot is already assigned / joined to the channel. + https://api.slack.com/types/conversation#booleans + """ + return self.join_channels and not channel.get("is_member") + + def make_join_channel_slice(self, channel: Mapping[str, Any]) -> Mapping[str, Any]: + channel_id: str = channel.get("id") + channel_name: str = channel.get("name") + self.logger.info(f"Joining Slack Channel: `{channel_name}`") + return {"channel": channel_id, "channel_name": channel_name} + class Channels(ChanneledStream): data_field = "channels" @@ -103,19 +159,30 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: params["types"] = "public_channel" return params - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[MutableMapping]: + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[MutableMapping]: json_response = response.json() channels = json_response.get(self.data_field, []) if self.channel_filter: channels = [channel for channel in channels if channel["name"] in self.channel_filter] yield from channels + def read_records(self, sync_mode: SyncMode, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + Override the default `read_records` method to provide the `JoinChannelsStream` functionality, + and be able to read all the channels, not just the ones that already has the API Bot joined. + """ + for channel in super().read_records(sync_mode=sync_mode): + # check the channel should be joined before reading + if self.should_join_to_channel(channel): + # join the channel before reading it + yield from self.join_channels_stream.read_records( + sync_mode=sync_mode, + stream_slice=self.make_join_channel_slice(channel), + ) + # reading the channel data + self.logger.info(f"Reading the channel: `{channel.get('name')}`") + yield channel + class ChannelMembers(ChanneledStream): data_field = "members" @@ -134,7 +201,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, # Slack just returns raw IDs as a string, so we want to put them in a "join table" format yield {"member_id": member_id, "channel_id": stream_slice["channel_id"]} - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) for channel_record in channels_stream.read_records(sync_mode=SyncMode.full_refresh): yield {"channel_id": channel_record["id"]} @@ -148,21 +215,6 @@ def path(self, **kwargs) -> str: # Incremental Streams -def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1), end_date: Optional[DateTime] = None) -> Iterable[Period]: - """ - Yields a list of the beginning and ending timestamps of each day between the start date and now. - The return value is a pendulum.period - """ - - end_date = end_date or pendulum.now() - # Each stream_slice contains the beginning and ending timestamp for a 24 hour period - chunk_start_date = start_date - while chunk_start_date < end_date: - chunk_end_date = min(chunk_start_date + interval, end_date) - yield pendulum.period(chunk_start_date, chunk_end_date) - chunk_start_date = chunk_end_date - - class IncrementalMessageStream(ChanneledStream, ABC): data_field = "messages" cursor_field = "float_ts" @@ -179,8 +231,7 @@ def set_sub_primary_key(self): for index, value in enumerate(self.primary_key): setattr(self, f"sub_primary_key_{index + 1}", value) else: - logger = AirbyteLogger() - logger.error("Failed during setting sub primary keys. Primary key should be list.") + self.logger.error("Failed during setting sub primary keys. Primary key should be list.") def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) @@ -223,7 +274,7 @@ def path(self, **kwargs) -> str: def use_cache(self) -> bool: return True - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: stream_state = stream_state or {} start_date = pendulum.from_timestamp(stream_state.get(self.cursor_field, self._start_ts)) end_date = self._end_ts and pendulum.from_timestamp(self._end_ts) @@ -246,14 +297,14 @@ def __init__(self, lookback_window: Mapping[str, int], **kwargs): def path(self, **kwargs) -> str: return "conversations.replies" - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: """ The logic for incrementally syncing threads is not very obvious, so buckle up. To get all messages in a thread, one must specify the channel and timestamp of the parent (first) message of that thread, basically its ID. - One complication is that threads can be updated at any time in the future. Therefore, if we wanted to comprehensively sync data + One complication is that threads can be updated at Any time in the future. Therefore, if we wanted to comprehensively sync data i.e: get every single response in a thread, we'd have to read every message in the slack instance every time we ran a sync, because otherwise there is no way to guarantee that a thread deep in the past didn't receive a new message. @@ -296,50 +347,6 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite # yield an empty slice to checkpoint state later yield {} - def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - """ - Filtering already read records for incremental sync. Copied state value to X after the last sync - to really 100% make sure no one can edit the state during the run. - """ - - initial_state = copy.deepcopy(stream_state) or {} - - for record in super().read_records(stream_state=stream_state, **kwargs): - if record.get(self.cursor_field, 0) >= initial_state.get(self.cursor_field, 0): - yield record - - -class JoinChannelsStream(HttpStream): - """ - This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. - Its responses should only be logged for debugging reasons, not read as records. - """ - - url_base = "https://slack.com/api/" - http_method = "POST" - primary_key = "id" - - def __init__(self, channel_filter: List[str] = None, **kwargs): - self.channel_filter = channel_filter or [] - super().__init__(**kwargs) - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - return [{"message": f"Successfully joined channel: {stream_slice['channel_name']}"}] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None # No pagination - - def path(self, **kwargs) -> str: - return "conversations.join" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) - for channel in channels_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"channel": channel["id"], "channel_name": channel["name"]} - - def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: - return {"channel": stream_slice["channel"]} - class SourceSlack(AbstractSource): def _get_authenticator(self, config: Mapping[str, Any]): @@ -377,8 +384,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: end_date = end_date and pendulum.parse(end_date) threads_lookback_window = pendulum.Duration(days=config["lookback_window"]) channel_filter = config.get("channel_filter", []) + should_join_to_channels = config.get("join_channels") - channels = Channels(authenticator=authenticator, channel_filter=channel_filter) + channels = Channels(authenticator=authenticator, join_channels=should_join_to_channels, channel_filter=channel_filter) streams = [ channels, ChannelMembers(authenticator=authenticator, channel_filter=channel_filter), @@ -399,13 +407,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Users(authenticator=authenticator), ] - # To sync data from channels, the bot backed by this token needs to join all those channels. This operation is idempotent. - if config["join_channels"]: - logger = AirbyteLogger() - logger.info("joining Slack channels") - join_channels_stream = JoinChannelsStream(authenticator=authenticator, channel_filter=channel_filter) - for stream_slice in join_channels_stream.stream_slices(): - for message in join_channels_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): - logger.info(message["message"]) - return streams diff --git a/airbyte-integrations/connectors/source-slack/source_slack/utils.py b/airbyte-integrations/connectors/source-slack/source_slack/utils.py new file mode 100644 index 000000000000..7507dbab3565 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/utils.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Iterable, Optional + +import pendulum +from pendulum import DateTime, Period + + +def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1), end_date: Optional[DateTime] = None) -> Iterable[Period]: + """ + Yields a list of the beginning and ending timestamps of each day between the start date and now. + The return value is a pendulum.period + """ + + end_date = end_date or pendulum.now() + # Each stream_slice contains the beginning and ending timestamp for a 24 hour period + chunk_start_date = start_date + while chunk_start_date < end_date: + chunk_end_date = min(chunk_start_date + interval, end_date) + yield pendulum.period(chunk_start_date, chunk_end_date) + chunk_start_date = chunk_end_date diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 2fa9d3d332fe..6d9254730d5f 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -18,10 +18,10 @@ def conversations_list(requests_mock): "https://slack.com/api/conversations.list?limit=1000&types=public_channel", json={ "channels": [ - {"name": "advice-data-architecture", "id": 1}, - {"name": "advice-data-orchestration", "id": 2}, - {"name": "airbyte-for-beginners", "id": 3}, - {"name": "good-reads", "id": 4}, + {"name": "advice-data-architecture", "id": 1, "is_member": False}, + {"name": "advice-data-orchestration", "id": 2, "is_member": True}, + {"name": "airbyte-for-beginners", "id": 3, "is_member": False}, + {"name": "good-reads", "id": 4, "is_member": True}, ] }, ) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index 01d1bf1af9be..bef3bf26651f 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -11,12 +11,11 @@ @parametrized_configs -def test_streams(conversations_list, join_channels, config, is_valid): +def test_streams(conversations_list, config, is_valid): source = SourceSlack() if is_valid: streams = source.streams(config) assert len(streams) == 5 - assert join_channels.call_count == 2 else: with pytest.raises(Exception) as exc_info: _ = source.streams(config) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index b783be1817c9..d0327093318f 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -7,7 +7,7 @@ import pendulum import pytest from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_slack.source import Threads, Users +from source_slack.source import Channels, Threads, Users @pytest.fixture @@ -93,3 +93,18 @@ def test_get_updated_state(authenticator, legacy_token_config, current_state, la def test_backoff(authenticator, headers, expected_result): stream = Users(authenticator=authenticator) assert stream.backoff_time(Mock(headers=headers)) == expected_result + + +def test_channels_stream_with_autojoin(authenticator) -> None: + """ + The test uses the `conversations_list` fixture(autouse=true) as API mocker. + """ + expected = [ + {'name': 'advice-data-architecture', 'id': 1, 'is_member': False}, + {'name': 'advice-data-orchestration', 'id': 2, 'is_member': True}, + {'name': 'airbyte-for-beginners', 'id': 3, 'is_member': False}, + {'name': 'good-reads', 'id': 4, 'is_member': True}, + ] + stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) + assert list(stream.read_records(None)) == expected + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/main.py b/airbyte-integrations/connectors/source-smaily/main.py index 647f5a347f70..fbb55aa40a54 100644 --- a/airbyte-integrations/connectors/source-smaily/main.py +++ b/airbyte-integrations/connectors/source-smaily/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smaily import SourceSmaily +from source_smaily.run import run if __name__ == "__main__": - source = SourceSmaily() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smaily/metadata.yaml b/airbyte-integrations/connectors/source-smaily/metadata.yaml index f2105bbe100b..408ce79e8f32 100644 --- a/airbyte-integrations/connectors/source-smaily/metadata.yaml +++ b/airbyte-integrations/connectors/source-smaily/metadata.yaml @@ -8,6 +8,10 @@ data: icon: smaily.svg license: MIT name: Smaily + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-smaily registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-smaily/setup.py b/airbyte-integrations/connectors/source-smaily/setup.py index aa835b3698f6..f3b23a618c6a 100644 --- a/airbyte-integrations/connectors/source-smaily/setup.py +++ b/airbyte-integrations/connectors/source-smaily/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-smaily=source_smaily.run:run", + ], + }, name="source_smaily", description="Source implementation for Smaily.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/run.py b/airbyte-integrations/connectors/source-smaily/source_smaily/run.py new file mode 100644 index 000000000000..24297e8addb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smaily import SourceSmaily + + +def run(): + source = SourceSmaily() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smartengage/main.py b/airbyte-integrations/connectors/source-smartengage/main.py index f9aec77e67ec..d445802b0ed4 100644 --- a/airbyte-integrations/connectors/source-smartengage/main.py +++ b/airbyte-integrations/connectors/source-smartengage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smartengage import SourceSmartengage +from source_smartengage.run import run if __name__ == "__main__": - source = SourceSmartengage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smartengage/metadata.yaml b/airbyte-integrations/connectors/source-smartengage/metadata.yaml index 25485ef405e6..4fd15c960d1a 100644 --- a/airbyte-integrations/connectors/source-smartengage/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartengage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: smartengage.svg license: MIT name: SmartEngage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-smartengage registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-smartengage/setup.py b/airbyte-integrations/connectors/source-smartengage/setup.py index 958a3b00df73..e4dd81fe7a9b 100644 --- a/airbyte-integrations/connectors/source-smartengage/setup.py +++ b/airbyte-integrations/connectors/source-smartengage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-smartengage=source_smartengage.run:run", + ], + }, name="source_smartengage", description="Source implementation for Smartengage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py b/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py new file mode 100644 index 000000000000..593e4eaf40f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smartengage import SourceSmartengage + + +def run(): + source = SourceSmartengage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smartsheets/Dockerfile b/airbyte-integrations/connectors/source-smartsheets/Dockerfile index 023118cbedd7..00a47d39b74f 100644 --- a/airbyte-integrations/connectors/source-smartsheets/Dockerfile +++ b/airbyte-integrations/connectors/source-smartsheets/Dockerfile @@ -14,5 +14,5 @@ COPY $CODE_PATH ./$CODE_PATH ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.1 +LABEL io.airbyte.version=1.1.2 LABEL io.airbyte.name=airbyte/source-smartsheets diff --git a/airbyte-integrations/connectors/source-smartsheets/main.py b/airbyte-integrations/connectors/source-smartsheets/main.py index 3603f2be666a..62f5650b92ea 100644 --- a/airbyte-integrations/connectors/source-smartsheets/main.py +++ b/airbyte-integrations/connectors/source-smartsheets/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smartsheets import SourceSmartsheets +from source_smartsheets.run import run if __name__ == "__main__": - source = SourceSmartsheets() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml index 46e5bc5e5586..00660e496826 100644 --- a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml @@ -9,13 +9,17 @@ data: connectorSubtype: api connectorType: source definitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc - dockerImageTag: 1.1.1 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-smartsheets documentationUrl: https://docs.airbyte.com/integrations/sources/smartsheets githubIssueLabel: source-smartsheets icon: smartsheet.svg license: MIT name: Smartsheets + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-smartsheets registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-smartsheets/setup.py b/airbyte-integrations/connectors/source-smartsheets/setup.py index f30812c4b62f..006b0fab9b90 100644 --- a/airbyte-integrations/connectors/source-smartsheets/setup.py +++ b/airbyte-integrations/connectors/source-smartsheets/setup.py @@ -9,6 +9,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-smartsheets=source_smartsheets.run:run", + ], + }, name="source_smartsheets", description="Source implementation for Smartsheets.", author="Nate Nowack", @@ -18,5 +23,17 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, ) diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py new file mode 100644 index 000000000000..6195e74166ce --- /dev/null +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smartsheets import SourceSmartsheets + + +def run(): + source = SourceSmartsheets() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/.dockerignore b/airbyte-integrations/connectors/source-snapchat-marketing/.dockerignore deleted file mode 100644 index c99f3d738dd8..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!main.py -!source_snapchat_marketing -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile deleted file mode 100644 index 64dc6bd5a121..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base -FROM base as builder - - -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install --prefix=/install . - - -FROM base -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -WORKDIR /airbyte/integration_code -COPY main.py ./ -COPY source_snapchat_marketing ./source_snapchat_marketing - - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/README.md b/airbyte-integrations/connectors/source-snapchat-marketing/README.md index e9459deb5411..ac8a4af9ff18 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/README.md +++ b/airbyte-integrations/connectors/source-snapchat-marketing/README.md @@ -1,68 +1,55 @@ -# Snapchat Marketing Source +# Snapchat-Marketing source connector -This is the repository for the Snapchat Marketing source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/snapchat-marketing). + +This is the repository for the Snapchat-Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/snapchat-marketing). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/snapchat-marketing) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_snapchat_marketing/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/snapchat-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_snapchat_marketing/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source snapchat-marketing test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-snapchat-marketing spec +poetry run source-snapchat-marketing check --config secrets/config.json +poetry run source-snapchat-marketing discover --config secrets/config.json +poetry run source-snapchat-marketing read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-snapchat-marketing build ``` -An image will be built with the tag `airbyte/source-snapchat-marketing:dev`. +An image will be available on your host with the tag `airbyte/source-snapchat-marketing:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-snapchat-marketing:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-snapchat-marketing:dev spec @@ -71,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-snapchat-marketing:dev docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-snapchat-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-snapchat-marketing test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-snapchat-marketing test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/snapchat-marketing.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/snapchat-marketing.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/main.py b/airbyte-integrations/connectors/source-snapchat-marketing/main.py index 3b7dadc93a3e..d48c013d7dcf 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/main.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_snapchat_marketing import SourceSnapchatMarketing +from source_snapchat_marketing.run import run if __name__ == "__main__": - source = SourceSnapchatMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml index 060d8a709d1e..7c4cb1d20351 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml @@ -3,15 +3,21 @@ data: hosts: - accounts.snapchat.com - adsapi.snapchat.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-snapchat-marketing githubIssueLabel: source-snapchat-marketing icon: snapchat.svg license: MIT name: Snapchat Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-snapchat-marketing registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock b/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock new file mode 100644 index 000000000000..3d45c572b951 --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock @@ -0,0 +1,1033 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.62.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "f0ea85ffbecdae30ea01bfe99f00ec3b7c36bdc9d04c5066a06aa19cb1ec9353" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml b/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml new file mode 100644 index 000000000000..4b3af63198c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.3.2" +name = "source-snapchat-marketing" +description = "Source implementation for Snapchat Marketing." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/snapchat-marketing" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_snapchat_marketing" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.62.1" + +[tool.poetry.scripts] +source-snapchat-marketing = "source_snapchat_marketing.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.6.1" +pytest = "^6.1" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt b/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py deleted file mode 100644 index cc60f43cbe70..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] - -setup( - name="source_snapchat_marketing", - description="Source implementation for Snapchat Marketing.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py new file mode 100644 index 000000000000..61c972da8a2d --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_snapchat_marketing import SourceSnapchatMarketing + + +def run(): + source = SourceSnapchatMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index 6268f7193dba..ce05a63e5413 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -10,9 +10,9 @@ acceptance_tests: tests: - config_path: "secrets/config.json" status: "succeed" - discovery: - tests: - - config_path: "secrets/config.json" + # discovery: + # tests: + # - config_path: "secrets/config.json" basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index f54a3e44021c..a49ec3ff0ac8 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -1,41 +1,21 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.snowflake.SnowflakeSourceRunner' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -configurations { - all { - resolutionStrategy { - force 'org.jooq:jooq:3.13.4' - } - } -} - dependencies { implementation group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.14.1' - implementation 'com.zaxxer:HikariCP:5.0.1' + testImplementation 'org.testcontainers:jdbc:1.19.4' testImplementation 'org.hamcrest:hamcrest-all:1.3' - - integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' } diff --git a/airbyte-integrations/connectors/source-snowflake/metadata.yaml b/airbyte-integrations/connectors/source-snowflake/metadata.yaml index 2b9167b43348..3015891a4599 100644 --- a/airbyte-integrations/connectors/source-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/source-snowflake/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: database connectorType: source definitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 - dockerImageTag: 0.2.2 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-snowflake documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake githubIssueLabel: source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java index 8a43b80d99bb..012c71bb208b 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java @@ -13,12 +13,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import io.airbyte.cdk.db.factory.DataSourceFactory; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -28,24 +25,17 @@ import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.nio.file.Path; -import java.sql.JDBCType; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class SnowflakeJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class SnowflakeJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static JsonNode snConfig; @@ -78,67 +68,73 @@ static void init() { INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(true)"; } - @BeforeEach - public void setup() throws Exception { - super.setup(); - } - - @AfterEach - public void clean() throws Exception { - super.tearDown(); - DataSourceFactory.close(dataSource); - } - @Override public boolean supportsSchemas() { return true; } @Override - public JsonNode getConfig() { + protected JsonNode config() { return Jsons.clone(snConfig); } @Override - public String getDriverClass() { - return SnowflakeSource.DRIVER_CLASS; + protected SnowflakeTestDatabase createTestDatabase() { + final SnowflakeTestDatabase snowflakeTestDatabase = new SnowflakeTestDatabase(source().toDatabaseConfig(Jsons.clone(snConfig))); + for (final String schemaName : TEST_SCHEMAS) { + snowflakeTestDatabase.onClose(DROP_SCHEMA_QUERY, schemaName); + } + return snowflakeTestDatabase.initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { + protected SnowflakeSource source() { return new SnowflakeSource(AIRBYTE_OSS); } @Test - void testCheckFailure() throws Exception { + @Override + protected void testCheckFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + } } @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + } } @Test public void testCheckEmptyUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.USERNAME_KEY, ""); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 200011;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000; Error code: 200011;")); + } } @Test public void testCheckIncorrectHostFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("Could not connect with provided configuration")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("Could not connect with provided configuration")); + } } @Override @@ -171,30 +167,6 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } - @Override - protected List getTestMessages() { - return List.of( - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); - } - @Override protected void incrementalDateCheck() throws Exception { super.incrementalCursorCheck(COL_UPDATED_AT, @@ -204,73 +176,33 @@ protected void incrementalDateCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; - } - /* Test that schema config key is making discover pull tables of this schema only */ @Test void testDiscoverSchemaConfig() throws Exception { - // add table and data to a separate schema. - database.execute(connection -> { - connection.createStatement().execute( - String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('1','picard')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('2', 'crusher')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('3', 'vash')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement().execute( - String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME, Strings.addRandomSuffix(TABLE_NAME, "_", 4)))); - }); + // add table to a separate schema. + testdb.with(String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME, Strings.addRandomSuffix(TABLE_NAME, "_", 4)))); + final JsonNode config = config(); JsonNode confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME); - AirbyteCatalog actual = source.discover(confWithSchema); - - assertFalse(actual.getStreams().isEmpty()); + try (SnowflakeSource source = source()) { + AirbyteCatalog actual = source.discover(confWithSchema); - var streams = actual.getStreams().stream() - .filter(s -> !s.getNamespace().equals(SCHEMA_NAME)) - .collect(Collectors.toList()); + assertFalse(actual.getStreams().isEmpty()); - assertTrue(streams.isEmpty()); + var streams = actual.getStreams().stream().filter(s -> !s.getNamespace().equals(SCHEMA_NAME)).collect(Collectors.toList()); - confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME2); - actual = source.discover(confWithSchema); - assertFalse(actual.getStreams().isEmpty()); + assertTrue(streams.isEmpty()); - streams = actual.getStreams().stream() - .filter(s -> !s.getNamespace().equals(SCHEMA_NAME2)) - .collect(Collectors.toList()); + confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME2); + actual = source.discover(confWithSchema); + assertFalse(actual.getStreams().isEmpty()); - assertTrue(streams.isEmpty()); + streams = actual.getStreams().stream().filter(s -> !s.getNamespace().equals(SCHEMA_NAME2)).collect(Collectors.toList()); + assertTrue(streams.isEmpty()); + } } } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java index d1394f1a569c..dfbbbf21cfb6 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java @@ -29,6 +29,7 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.nio.file.Path; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import javax.sql.DataSource; @@ -41,6 +42,7 @@ public class SnowflakeSourceAcceptanceTest extends SourceAcceptanceTest { + RandomStringUtils.randomAlphanumeric(4).toUpperCase(); private static final String STREAM_NAME1 = "ID_AND_NAME1"; private static final String STREAM_NAME2 = "ID_AND_NAME2"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); // config which refers to the schema that the test is being run in. protected JsonNode config; @@ -142,7 +144,8 @@ protected DataSource createDataSource() { String.format(DatabaseDriver.SNOWFLAKE.getUrlFormatString(), config.get(JdbcUtils.HOST_KEY).asText()), Map.of("role", config.get("role").asText(), "warehouse", config.get("warehouse").asText(), - JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText())); + JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText()), + CONNECTION_TIMEOUT); } @Test diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java index 8b3408cf22a9..7201511b4554 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java @@ -15,7 +15,9 @@ import java.nio.file.Path; import java.util.Properties; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; +@Disabled("The provided refresh token is invalid.") public class SnowflakeSourceAuthAcceptanceTest extends SnowflakeSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index 3385b7c75ee2..523e43e322ef 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.source.snowflake.SnowflakeSource; import io.airbyte.protocol.models.JsonSchemaType; import java.nio.file.Path; +import java.time.Duration; import java.util.Map; import org.apache.commons.lang3.RandomStringUtils; import org.jooq.DSLContext; @@ -28,6 +29,7 @@ public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest private static final String SCHEMA_NAME = "SOURCE_DATA_TYPE_TEST_" + RandomStringUtils.randomAlphanumeric(4).toUpperCase(); private static final String INSERT_SEMI_STRUCTURED_SQL = "INSERT INTO %1$s (ID, TEST_COLUMN) SELECT %2$s, %3$s"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); private JsonNode config; private Database database; @@ -57,7 +59,8 @@ protected Database setupDatabase() throws Exception { Map.of( "role", config.get("role").asText(), "warehouse", config.get("warehouse").asText(), - JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText())); + JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText()), + CONNECTION_TIMEOUT); database = getDatabase(); @@ -77,13 +80,9 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - try { - final String dropSchemaQuery = String - .format("DROP SCHEMA IF EXISTS %s", SCHEMA_NAME); - database.query(ctx -> ctx.fetch(dropSchemaQuery)); - } finally { - dslContext.close(); - } + final String dropSchemaQuery = String + .format("DROP SCHEMA IF EXISTS %s", SCHEMA_NAME); + database.query(ctx -> ctx.fetch(dropSchemaQuery)); } @Override diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java new file mode 100644 index 000000000000..a4e194c8b099 --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.SNOWFLAKE; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class SnowflakeTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + + protected SnowflakeTestDatabase(final JsonNode snowflakeConfig) { + super(new NonContainer(snowflakeConfig.get(JdbcUtils.USERNAME_KEY).asText(), + snowflakeConfig.has(JdbcUtils.PASSWORD_KEY) ? snowflakeConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + snowflakeConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), SNOWFLAKE.getDriverClassName(), "")); + this.username = snowflakeConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = snowflakeConfig.has(JdbcUtils.PASSWORD_KEY) ? snowflakeConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = snowflakeConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return SNOWFLAKE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + static public class SnowflakeConfigBuilder extends TestDatabase.ConfigBuilder { + + protected SnowflakeConfigBuilder(SnowflakeTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/main.py b/airbyte-integrations/connectors/source-sonar-cloud/main.py index 81355fadfed1..5ba429ba3211 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/main.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sonar_cloud import SourceSonarCloud +from source_sonar_cloud.run import run if __name__ == "__main__": - source = SourceSonarCloud() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml index 1d1cf8e9b488..cc224211c7bd 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml @@ -11,6 +11,10 @@ data: icon: sonarcloud.svg license: MIT name: Sonar Cloud + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sonar-cloud registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sonar-cloud/setup.py b/airbyte-integrations/connectors/source-sonar-cloud/setup.py index e99684d61a65..2f8f82301482 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/setup.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sonar-cloud=source_sonar_cloud.run:run", + ], + }, name="source_sonar_cloud", description="Source implementation for Sonar Cloud.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py new file mode 100644 index 000000000000..e29261106741 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sonar_cloud import SourceSonarCloud + + +def run(): + source = SourceSonarCloud() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-spacex-api/main.py b/airbyte-integrations/connectors/source-spacex-api/main.py index 7ceab1979a8b..52f8d09f4508 100644 --- a/airbyte-integrations/connectors/source-spacex-api/main.py +++ b/airbyte-integrations/connectors/source-spacex-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_spacex_api import SourceSpacexApi +from source_spacex_api.run import run if __name__ == "__main__": - source = SourceSpacexApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml index 1da6b6776c15..6f1677c520b9 100644 --- a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: spacex.svg license: MIT name: SpaceX API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-spacex-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index 2d202b30ec9a..00b9cc0110f4 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-spacex-api=source_spacex_api.run:run", + ], + }, name="source_spacex_api", description="Source implementation for Spacex Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py new file mode 100644 index 000000000000..7baf4913b453 --- /dev/null +++ b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_spacex_api import SourceSpacexApi + + +def run(): + source = SourceSpacexApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-square/main.py b/airbyte-integrations/connectors/source-square/main.py index 064f99648db8..7102d93900cf 100644 --- a/airbyte-integrations/connectors/source-square/main.py +++ b/airbyte-integrations/connectors/source-square/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_square import SourceSquare +from source_square.run import run if __name__ == "__main__": - source = SourceSquare() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-square/metadata.yaml b/airbyte-integrations/connectors/source-square/metadata.yaml index 71b100e1f7c1..7e85289fa520 100644 --- a/airbyte-integrations/connectors/source-square/metadata.yaml +++ b/airbyte-integrations/connectors/source-square/metadata.yaml @@ -16,6 +16,10 @@ data: icon: square.svg license: MIT name: Square + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-square registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-square/setup.py b/airbyte-integrations/connectors/source-square/setup.py index fe3012decd5d..3b7c2204ad12 100644 --- a/airbyte-integrations/connectors/source-square/setup.py +++ b/airbyte-integrations/connectors/source-square/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-square=source_square.run:run", + ], + }, name="source_square", description="Source implementation for Square.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-square/source_square/run.py b/airbyte-integrations/connectors/source-square/source_square/run.py new file mode 100644 index 000000000000..df5f3fd74012 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_square import SourceSquare + + +def run(): + source = SourceSquare() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-statuspage/main.py b/airbyte-integrations/connectors/source-statuspage/main.py index 8046f076e8ab..2701aa6de07d 100644 --- a/airbyte-integrations/connectors/source-statuspage/main.py +++ b/airbyte-integrations/connectors/source-statuspage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_statuspage import SourceStatuspage +from source_statuspage.run import run if __name__ == "__main__": - source = SourceStatuspage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-statuspage/metadata.yaml b/airbyte-integrations/connectors/source-statuspage/metadata.yaml index 85a2cf0283ab..45dadd21b5cb 100644 --- a/airbyte-integrations/connectors/source-statuspage/metadata.yaml +++ b/airbyte-integrations/connectors/source-statuspage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: statuspage.svg license: MIT name: Statuspage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-statuspage registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-statuspage/setup.py b/airbyte-integrations/connectors/source-statuspage/setup.py index 772fedc5e034..f7adbd694ed6 100644 --- a/airbyte-integrations/connectors/source-statuspage/setup.py +++ b/airbyte-integrations/connectors/source-statuspage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-statuspage=source_statuspage.run:run", + ], + }, name="source_statuspage", description="Source implementation for Statuspage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py b/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py new file mode 100644 index 000000000000..d29716d7c02f --- /dev/null +++ b/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_statuspage import SourceStatuspage + + +def run(): + source = SourceStatuspage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-strava/main.py b/airbyte-integrations/connectors/source-strava/main.py index 054e4911089c..9fc5a4bb2918 100644 --- a/airbyte-integrations/connectors/source-strava/main.py +++ b/airbyte-integrations/connectors/source-strava/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_strava import SourceStrava +from source_strava.run import run if __name__ == "__main__": - source = SourceStrava() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-strava/metadata.yaml b/airbyte-integrations/connectors/source-strava/metadata.yaml index 0b8ffd4826b4..fd85c85ce0b1 100644 --- a/airbyte-integrations/connectors/source-strava/metadata.yaml +++ b/airbyte-integrations/connectors/source-strava/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - strava.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-strava registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-strava/setup.py b/airbyte-integrations/connectors/source-strava/setup.py index 8dda89709cd4..3bdc92646070 100644 --- a/airbyte-integrations/connectors/source-strava/setup.py +++ b/airbyte-integrations/connectors/source-strava/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-strava=source_strava.run:run", + ], + }, name="source_strava", description="Source implementation for Strava.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-strava/source_strava/run.py b/airbyte-integrations/connectors/source-strava/source_strava/run.py new file mode 100644 index 000000000000..c8b1590993cd --- /dev/null +++ b/airbyte-integrations/connectors/source-strava/source_strava/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_strava import SourceStrava + + +def run(): + source = SourceStrava() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-stripe/README.md b/airbyte-integrations/connectors/source-stripe/README.md index a941634a17cc..8b8e5526ae26 100644 --- a/airbyte-integrations/connectors/source-stripe/README.md +++ b/airbyte-integrations/connectors/source-stripe/README.md @@ -1,77 +1,91 @@ -# Stripe Source +# Stripe source connector + This is the repository for the Stripe source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/stripe). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/stripe). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Minimum Python version required `= 3.7.0` -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/stripe) +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/stripe) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source stripe test creds` -and place them into `secrets/config.json`. - ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-stripe spec +poetry run source-stripe check --config secrets/config.json +poetry run source-stripe discover --config secrets/config.json +poetry run source-stripe read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Unit Tests +### Running unit tests To run unit tests locally, from the connector directory run: ``` -python -m pytest unit_tests +poetry run pytest unit_tests ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-stripe build +``` + +An image will be available on your host with the tag `airbyte/source-stripe:dev`. + + +### Running as a docker container +Then run any of the connector commands as follows: ``` -docker build . --no-cache -t airbyte/source-stripe:dev \ -&& python -m pytest integration_tests -p integration_tests.acceptance +docker run --rm airbyte/source-stripe:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-stripe:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-stripe:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-stripe:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +### Running our CI test suite +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-stripe test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -### Publishing a new version of the connector +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-stripe test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/stripe.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/stripe.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 84469b4d8ce0..af2bc7f25dfe 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -18,18 +18,19 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" + fail_on_extra_columns: false # CATs are failing since https://github.com/airbytehq/airbyte/commit/dccb2fa7165f031fa1233d695897b07f9aacb39c, API Source team to fix this timeout_seconds: 3600 empty_streams: - name: "application_fees" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "application_fees_refunds" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "authorizations" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "bank_accounts" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "cards" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "early_fraud_warnings" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "external_account_bank_accounts" @@ -37,11 +38,11 @@ acceptance_tests: - name: "external_account_cards" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "payment_methods" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "persons" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "reviews" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "transactions" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "events" @@ -84,6 +85,16 @@ acceptance_tests: invoice_line_items: - name: margins bypass_reason: "API randomly returns this field" + subscriptions: + - name: current_period_start + bypass_reason: "Frequently changing data" + - name: current_period_end + bypass_reason: "Frequently changing data" + - name: latest_invoice + bypass_reason: "Frequently changing data" + customers: + - name: next_invoice_sequence + bypass_reason: "Frequently changing data" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index affd99f2e322..c6602a1a5df7 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -4,9 +4,10 @@ {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "object": "setup_attempt", "application": null, "created": 1649752937, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627241471} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627242509} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627243547} -{"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} -{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} -{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} +{"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "invoices": {"default_account_tax_ids": null}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} +{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"annual_revenue": null,"estimated_worker_count": null,"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "invoices": {"default_account_tax_ids": null}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} +{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"annual_revenue": null,"estimated_worker_count": null,"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "invoices": {"default_account_tax_ids": null}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} +{"stream": "accounts", "data": {"id": "acct_1HRPLyCpK2Z3jTFF", "object": "account", "capabilities": {"acss_debit_payments": "inactive", "afterpay_clearpay_payments": "inactive", "bancontact_payments": "inactive", "card_payments": "inactive", "eps_payments": "inactive", "giropay_payments": "inactive", "ideal_payments": "inactive", "p24_payments": "inactive", "sepa_debit_payments": "inactive", "sofort_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "default_currency": "usd", "details_submitted": false, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "America/Los_Angeles"}, "invoices": {"default_account_tax_ids": null}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1707141120832} {"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1697627269309} {"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1697627270253} {"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1697627270254} @@ -17,11 +18,11 @@ {"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1697627273833} {"stream": "file_links", "data": {"id": "link_1KnfIbEcXtiJtvvhyBLUqkSt", "object": "file_link", "created": 1649752929, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaXh1blBqMmY0MzI3SHZWbUZIeFVGU3Nl0022JjupYq"}, "emitted_at": 1697627273834} {"stream": "file_links", "data": {"id": "link_1KnfIUEcXtiJtvvh0ktKHfWz", "object": "file_link", "created": 1649752922, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfNzhlbE9MUGNYbkJzMkRLSWdEcnhvY3FH00DK5jBVaH"}, "emitted_at": 1697627273835} -{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"after_submit": null,"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} +{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "liability": null, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"after_submit": null,"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "issuer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} {"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDg4MTY4MDc20200Sa50llWu/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null, "updated": 1686158100}, "emitted_at": 1697627276386} {"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1646998902}, "emitted_at": 1697627278433} {"stream": "customers", "data": {"id": "cus_Kou8knsO3qQOwU", "object": "customer", "address": null, "balance": 0, "created": 1640123795, "currency": "usd", "default_source": "src_1MSID8EcXtiJtvvhxIT9lXRy", "delinquent": false, "description": null, "discount": null, "email": "edward.gao+stripe-test-customer-1@airbyte.io", "invoice_prefix": "CA35DF83", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "edgao-test-customer-1", "next_invoice_sequence": 2, "phone": null, "preferred_locales": [], "shipping": null, "tax_exempt": "none", "test_clock": null, "updated": 1640123795}, "emitted_at": 1697627278435} -{"stream": "customers", "data": { "id": "cus_NGoTFiJFVbSsvZ", "object": "customer", "address": { "city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA" }, "balance": 0, "created": 1675160053, "currency": "usd", "default_source": "src_1MWGs8EcXtiJtvvh4nYdQvEr", "delinquent": false, "description": "Test Customer 2 description", "discount": null, "email": "user1.sample@zohomail.eu", "invoice_prefix": "C09C1837", "invoice_settings": { "custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null }, "livemode": false, "metadata": {}, "name": "Test Customer 2", "next_invoice_sequence": 14, "phone": null, "preferred_locales": [ "en-US" ], "shipping": { "address": { "city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA" }, "name": "Test Customer 2", "phone": "" }, "tax_exempt": "none", "test_clock": null, "updated": 1675160053 }, "emitted_at": 1697627278439} +{"stream": "customers", "data": {"id": "cus_NGoTFiJFVbSsvZ", "object": "customer", "address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "balance": 0, "created": 1675160053, "currency": "usd", "default_source": "src_1MWGs8EcXtiJtvvh4nYdQvEr", "delinquent": false, "description": "Test Customer 2 description", "discount": null, "email": "user1.sample@zohomail.eu", "invoice_prefix": "C09C1837", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test Customer 2", "next_invoice_sequence": 15, "phone": null, "preferred_locales": ["en-US"], "shipping": {"address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "name": "Test Customer 2", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1675160053}, "emitted_at": 1697627278439} {"stream": "cardholders", "data": {"id": "ich_1KUKBeEcXtiJtvvhCEFgko6h", "object": "issuing.cardholder", "billing": {"address": {"city": "San Francisco", "country": "US", "line1": "1234 Main Street", "line2": null, "postal_code": "94111", "state": "CA"}}, "company": null, "created": 1645143542, "email": "jenny.rosen@example.com", "individual": null, "livemode": false, "metadata": {}, "name": "Jenny Rosen", "phone_number": "+18888675309", "preferred_locales": [], "requirements": {"disabled_reason": null, "past_due": []}, "spending_controls": {"allowed_categories": [], "blocked_categories": [], "spending_limits": [], "spending_limits_currency": null}, "status": "active", "type": "individual", "updated": 1645143542}, "emitted_at": 1697627292209} {"stream": "charges", "data": {"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"amount_authorized": 5300, "brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "extended_authorization": {"status": "disabled"}, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "incremental_authorization": {"status": "unavailable"}, "installments": null, "last4": "4242", "mandate": null, "multicapture": {"status": "unavailable"}, "network": "visa", "network_token": {"used": false}, "overcapture": {"maximum_amount_capturable": 5300, "status": "unavailable"}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1509-9197", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ35vqkGMgYYlboX7Hs6LBbBoR6yFToo5WeMCCwbkvCz7nl3E1KToovFFZKMJYnrpAHBlWJrVMJK6BWm", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640120473}, "emitted_at": 1697627293840} {"stream": "charges", "data": {"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"amount_authorized": 4200, "brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "extended_authorization": {"status": "disabled"}, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "incremental_authorization": {"status": "unavailable"}, "installments": null, "last4": "4242", "mandate": null, "multicapture": {"status": "unavailable"}, "network": "visa", "network_token": {"used": false}, "overcapture": {"maximum_amount_capturable": 4200, "status": "unavailable"}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1549-5630", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ35vqkGMgbg2Y1Ao1M6LBYViHyCHYtYZtCIzc8I1Pm_oXAcXtgPDTNCfzyB3XOfFO4N-RK2w9sLuPjq", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640119035}, "emitted_at": 1697627293843} @@ -30,9 +31,9 @@ {"stream": "coupons", "data": {"id": "4SUEGKZg", "object": "coupon", "amount_off": 200, "created": 1674209030, "currency": "usd", "duration": "repeating", "duration_in_months": 3, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0406\u0435\u043a\u0448\u0437\u0443", "percent_off": null, "redeem_by": null, "times_redeemed": 0, "valid": true, "updated": 1674209030}, "emitted_at": 1697627296927} {"stream": "coupons", "data": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true, "updated": 1674208993}, "emitted_at": 1697627296929} {"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "payment_method_details": {"card": {"brand": "visa", "network_reason_code": "83"}, "type": "card"}, "reason": "fraudulent", "status": "lost", "updated": 1674211590}, "emitted_at": 1697627298351} -{"stream": "invoices", "data": {"id": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 0, "amount_paid": 0, "amount_remaining": 0, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 0, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "manual", "charge": null, "collection_method": "send_invoice", "created": 1640123796, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "discounts": [], "due_date": 1688750070.0, "effective_at": 1686158070, "ending_balance": 0, "footer": null, "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDg4MTY4MTAw02007rgJxf1v?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDg4MTY4MTAw02007rgJxf1v/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "CA35DF83-0001", "on_behalf_of": null, "paid": true, "paid_out_of_band": false, "payment_intent": null, "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1640123795.0, "period_start": 1640123795.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 8400, "quote": null, "receipt_number": null, "rendering": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "paid", "status_transitions": {"finalized_at": 1686158070, "marked_uncollectible_at": null, "paid_at": 1686158100, "voided_at": null}, "subscription": null, "subscription_details": {"metadata": null}, "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax": null, "test_clock": null, "total": 8400, "total_discount_amounts": [], "total_excluding_tax": 8400, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1640123796.0, "updated": 1640123796}, "emitted_at": 1697627300463} -{"stream": "invoices", "data": {"id": "in_1MX37hEcXtiJtvvhRSl1KbQm", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 5500, "amount_paid": 5500, "amount_remaining": 0, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 1, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": true, "status": "complete"}, "billing_reason": "manual", "charge": "ch_3MX38QEcXtiJtvvh1y8YAJpg", "collection_method": "send_invoice", "created": 1675345605, "currency": "usd", "custom_fields": null, "customer": "cus_NGoTFiJFVbSsvZ", "customer_address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "customer_email": "user1.sample@zohomail.eu", "customer_name": "Test Customer 2", "customer_phone": null, "customer_shipping": {"address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "name": "Test Customer 2", "phone": ""}, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": "Thanks for your business!", "discount": null, "discounts": [], "due_date": 1677937605.0, "effective_at": 1675345650, "ending_balance": 0, "footer": "Test Invoice", "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDg4MTY4MTAw02009wP8HGgl?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDg4MTY4MTAw02009wP8HGgl/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1MX384EcXtiJtvvh3j2K123f", "object": "line_item", "amount": 6000, "amount_excluding_tax": 6000, "currency": "usd", "description": "Test Product 1", "discount_amounts": [{"amount": 500, "discount": "di_1MX384EcXtiJtvvhkOrY57Ep"}], "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice_item": "ii_1MX384EcXtiJtvvhguyn3iYb", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 3, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": false, "tax_rate": "txr_1MX384EcXtiJtvvhAhVE20Ii", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "2000"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1MX37hEcXtiJtvvhRSl1KbQm/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "C09C1837-0002", "on_behalf_of": null, "paid": true, "paid_out_of_band": false, "payment_intent": "pi_3MX38QEcXtiJtvvh10zsQJTC", "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1675345605.0, "period_start": 1675345605.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 0, "quote": null, "receipt_number": null, "rendering": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "paid", "status_transitions": {"finalized_at": 1675345650, "marked_uncollectible_at": null, "paid_at": 1675345673, "voided_at": null}, "subscription": null, "subscription_details": {"metadata": null}, "subtotal": 5500, "subtotal_excluding_tax": 5500, "tax": 0, "test_clock": null, "total": 5500, "total_discount_amounts": [{"amount": 500, "discount": "di_1MX384EcXtiJtvvhkOrY57Ep"}], "total_excluding_tax": 5500, "total_tax_amounts": [{"amount": 0, "inclusive": false, "tax_rate": "txr_1MX384EcXtiJtvvhAhVE20Ii", "taxability_reason": "not_collecting", "taxable_amount": 0}], "transfer_data": null, "webhooks_delivered_at": 1675345605.0, "updated": 1675345605}, "emitted_at": 1697627300891} -{"stream": "invoices", "data": {"id": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 72680, "amount_paid": 72680, "amount_remaining": 0, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 1, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": true, "status": "complete"}, "billing_reason": "manual", "charge": "ch_3MX2zPEcXtiJtvvh1BUGw8EC", "collection_method": "send_invoice", "created": 1675345019, "currency": "usd", "custom_fields": null, "customer": "cus_NGoTFiJFVbSsvZ", "customer_address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "customer_email": "user1.sample@zohomail.eu", "customer_name": "Test Customer 2", "customer_phone": null, "customer_shipping": {"address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "name": "Test Customer 2", "phone": ""}, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": "Thanks for your business!", "discount": null, "discounts": [], "due_date": 1677937018.0, "effective_at": 1675345090, "ending_balance": 0, "footer": "Test Invoice", "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDg4MTY4MTAw0200pHPBUPO0?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDg4MTY4MTAw0200pHPBUPO0/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1MX2yfEcXtiJtvvhiunY2j1x", "object": "line_item", "amount": 25200, "amount_excluding_tax": 25200, "currency": "usd", "description": "edgao-test-product", "discount_amounts": [{"amount": 2520, "discount": "di_1MX2ysEcXtiJtvvh8ORqRVKm"}], "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice_item": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 2, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": true, "tax_rate": "txr_1MX2yfEcXtiJtvvhVcMEMTRj", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "12600"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1MX2yFEcXtiJtvvhMXhUCgKx/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "C09C1837-0001", "on_behalf_of": null, "paid": true, "paid_out_of_band": false, "payment_intent": "pi_3MX2zPEcXtiJtvvh12VBcp6m", "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1675345018.0, "period_start": 1675345018.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 0, "quote": null, "receipt_number": null, "rendering": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 50000, "statement_descriptor": null, "status": "paid", "status_transitions": {"finalized_at": 1675345090, "marked_uncollectible_at": null, "paid_at": 1675345122, "voided_at": null}, "subscription": null, "subscription_details": {"metadata": null}, "subtotal": 22680, "subtotal_excluding_tax": 22680, "tax": 0, "test_clock": null, "total": 22680, "total_discount_amounts": [{"amount": 2520, "discount": "di_1MX2ysEcXtiJtvvh8ORqRVKm"}], "total_excluding_tax": 22680, "total_tax_amounts": [{"amount": 0, "inclusive": true, "tax_rate": "txr_1MX2yfEcXtiJtvvhVcMEMTRj", "taxability_reason": "not_collecting", "taxable_amount": 0}], "transfer_data": null, "webhooks_delivered_at": 1675345019.0, "updated": 1675345019}, "emitted_at": 1697627300895} +{"stream":"invoices","data":{"id":"in_1K9GK0EcXtiJtvvhSo2LvGqT","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":0,"amount_paid":0,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":0,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":false,"liability":null,"status":null},"billing_reason":"manual","charge":null,"collection_method":"send_invoice","created":1640123796,"currency":"usd","custom_fields":null,"customer":"cus_Kou8knsO3qQOwU","customer_address":null,"customer_email":"edward.gao+stripe-test-customer-1@airbyte.io","customer_name":"edgao-test-customer-1","customer_phone":null,"customer_shipping":null,"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":null,"discount":null,"discounts":[],"due_date":1688750070,"effective_at":1686158070,"ending_balance":0,"footer":null,"from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDk2MTc3MTMw0200ygIKWRVO?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDk2MTc3MTMw0200ygIKWRVO/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1K9GKLEcXtiJtvvhhHaYMebN","object":"line_item","amount":8400,"amount_excluding_tax":8400,"currency":"usd","description":"a box of parsnips","discount_amounts":[],"discountable":true,"discounts":[],"invoice_item":"ii_1K9GKLEcXtiJtvvhmr2AYOAx","livemode":false,"metadata":{},"period":{"end":1640123817,"start":1640123817},"plan":null,"price":{"id":"price_1K9GKLEcXtiJtvvhXbrg33lq","object":"price","active":false,"billing_scheme":"per_unit","created":1640123817,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_Kou8cQxtIpF1p7","recurring":null,"tax_behavior":"unspecified","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":8400,"unit_amount_decimal":"8400"},"proration":false,"proration_details":{"credited_items":null},"quantity":1,"subscription":null,"tax_amounts":[],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"8400"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"CA35DF83-0001","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":null,"payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1640123795,"period_start":1640123795,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":8400,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":0,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1686158070,"marked_uncollectible_at":null,"paid_at":1686158100,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":8400,"subtotal_excluding_tax":8400,"tax":null,"test_clock":null,"total":8400,"total_discount_amounts":[],"total_excluding_tax":8400,"total_tax_amounts":[],"transfer_data":null,"webhooks_delivered_at":1640123796,"updated":1640123796},"emitted_at":1705636368179} +{"stream":"invoices","data":{"id":"in_1MX37hEcXtiJtvvhRSl1KbQm","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":5500,"amount_paid":5500,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":1,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":true,"liability":{"type":"self"},"status":"complete"},"billing_reason":"manual","charge":"ch_3MX38QEcXtiJtvvh1y8YAJpg","collection_method":"send_invoice","created":1675345605,"currency":"usd","custom_fields":null,"customer":"cus_NGoTFiJFVbSsvZ","customer_address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"customer_email":"user1.sample@zohomail.eu","customer_name":"Test Customer 2","customer_phone":null,"customer_shipping":{"address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"name":"Test Customer 2","phone":""},"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":"Thanks for your business!","discount":null,"discounts":[],"due_date":1677937605,"effective_at":1675345650,"ending_balance":0,"footer":"Test Invoice","from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDk2MTc3MTY20200GeEOAZdh?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDk2MTc3MTY20200GeEOAZdh/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1MX384EcXtiJtvvh3j2K123f","object":"line_item","amount":6000,"amount_excluding_tax":6000,"currency":"usd","description":"Test Product 1","discount_amounts":[{"amount":500,"discount":"di_1MX384EcXtiJtvvhkOrY57Ep"}],"discountable":true,"discounts":["di_1MX384EcXtiJtvvhkOrY57Ep"],"invoice_item":"ii_1MX384EcXtiJtvvhguyn3iYb","livemode":false,"metadata":{},"period":{"end":1675345628,"start":1675345628},"plan":null,"price":{"id":"price_1MX364EcXtiJtvvhE3WgTl4O","object":"price","active":true,"billing_scheme":"per_unit","created":1675345504,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_NHcKselSHfKdfc","recurring":null,"tax_behavior":"exclusive","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":2000,"unit_amount_decimal":"2000"},"proration":false,"proration_details":{"credited_items":null},"quantity":3,"subscription":null,"tax_amounts":[{"amount":0,"inclusive":false,"tax_rate":"txr_1MX384EcXtiJtvvhAhVE20Ii","taxability_reason":"not_collecting","taxable_amount":0}],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"2000"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1MX37hEcXtiJtvvhRSl1KbQm/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"C09C1837-0002","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":"pi_3MX38QEcXtiJtvvh10zsQJTC","payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1675345605,"period_start":1675345605,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":0,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":0,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1675345650,"marked_uncollectible_at":null,"paid_at":1675345673,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":5500,"subtotal_excluding_tax":5500,"tax":0,"test_clock":null,"total":5500,"total_discount_amounts":[{"amount":500,"discount":"di_1MX384EcXtiJtvvhkOrY57Ep"}],"total_excluding_tax":5500,"total_tax_amounts":[{"amount":0,"inclusive":false,"tax_rate":"txr_1MX384EcXtiJtvvhAhVE20Ii","taxability_reason":"not_collecting","taxable_amount":0}],"transfer_data":null,"webhooks_delivered_at":1675345605,"updated":1675345605},"emitted_at":1705636368184} +{"stream":"invoices","data":{"id":"in_1MX2yFEcXtiJtvvhMXhUCgKx","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":72680,"amount_paid":72680,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":1,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":true,"liability":{"type":"self"},"status":"complete"},"billing_reason":"manual","charge":"ch_3MX2zPEcXtiJtvvh1BUGw8EC","collection_method":"send_invoice","created":1675345019,"currency":"usd","custom_fields":null,"customer":"cus_NGoTFiJFVbSsvZ","customer_address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"customer_email":"user1.sample@zohomail.eu","customer_name":"Test Customer 2","customer_phone":null,"customer_shipping":{"address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"name":"Test Customer 2","phone":""},"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":"Thanks for your business!","discount":null,"discounts":[],"due_date":1677937018,"effective_at":1675345090,"ending_balance":0,"footer":"Test Invoice","from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDk2MTc3MTY20200j28bcP5i?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDk2MTc3MTY20200j28bcP5i/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1MX2yfEcXtiJtvvhiunY2j1x","object":"line_item","amount":25200,"amount_excluding_tax":25200,"currency":"usd","description":"edgao-test-product","discount_amounts":[{"amount":2520,"discount":"di_1MX2ysEcXtiJtvvh8ORqRVKm"}],"discountable":true,"discounts":["di_1MX2ysEcXtiJtvvh8ORqRVKm"],"invoice_item":"ii_1MX2yfEcXtiJtvvhfhyOG7SP","livemode":false,"metadata":{},"period":{"end":1675345045,"start":1675345045},"plan":null,"price":{"id":"price_1K9GbqEcXtiJtvvhJ3lZe4i5","object":"price","active":true,"billing_scheme":"per_unit","created":1640124902,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_KouQ5ez86yREmB","recurring":null,"tax_behavior":"inclusive","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":12600,"unit_amount_decimal":"12600"},"proration":false,"proration_details":{"credited_items":null},"quantity":2,"subscription":null,"tax_amounts":[{"amount":0,"inclusive":true,"tax_rate":"txr_1MX2yfEcXtiJtvvhVcMEMTRj","taxability_reason":"not_collecting","taxable_amount":0}],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"12600"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1MX2yFEcXtiJtvvhMXhUCgKx/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"C09C1837-0001","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":"pi_3MX2zPEcXtiJtvvh12VBcp6m","payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1675345018,"period_start":1675345018,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":0,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":50000,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1675345090,"marked_uncollectible_at":null,"paid_at":1675345122,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":22680,"subtotal_excluding_tax":22680,"tax":0,"test_clock":null,"total":22680,"total_discount_amounts":[{"amount":2520,"discount":"di_1MX2ysEcXtiJtvvh8ORqRVKm"}],"total_excluding_tax":22680,"total_tax_amounts":[{"amount":0,"inclusive":true,"tax_rate":"txr_1MX2yfEcXtiJtvvhVcMEMTRj","taxability_reason":"not_collecting","taxable_amount":0}],"transfer_data":null,"webhooks_delivered_at":1675345019,"updated":1675345019},"emitted_at":1705636368184} {"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400", "updated": 1640123817}, "emitted_at": 1697627302138} {"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000", "updated": 1675345628}, "emitted_at": 1697627302391} {"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600", "updated": 1675345045}, "emitted_at": 1697627302392} @@ -46,8 +47,8 @@ {"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839715, "url": null}, "emitted_at": 1697627307635} {"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "features": [], "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1696839789, "url": null}, "emitted_at": 1697627307877} {"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839225, "url": null}, "emitted_at": 1697627307879} -{"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1697550676.0, "billing_thresholds": null, "cancel_at": 1705499476.0, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1702821076.0, "current_period_start": 1700229076, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": null, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1ODSSHEcXtiJtvvhW5LllxDH", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "active", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1700232971060} -{"stream": "subscription_schedule", "data": {"id": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "object": "subscription_schedule", "application": null, "canceled_at": null, "completed_at": null, "created": 1697550676, "current_phase": {"end_date": 1705499476, "start_date": 1697550676}, "customer": "cus_NGoTFiJFVbSsvZ", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": "Test Test", "invoice_settings": "{'days_until_due': None}", "on_behalf_of": null, "transfer_data": null}, "end_behavior": "cancel", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": "charge_automatically", "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": "Test Test", "end_date": 1705499476, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1697550676, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "active", "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "test_clock": null, "updated": 1697550676}, "emitted_at": 1697627312079} +{"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true, "liability": {"type": "self"}}, "billing_cycle_anchor": 1697550676.0, "billing_cycle_anchor_config": null, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1705499476.0, "current_period_start": 1702821076, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": 1705329724.0, "invoice_settings": {"account_tax_ids": null, "issuer": {"type": "self"}}, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1OOKkUEcXtiJtvvheUUavyuB", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "canceled", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1707158969393} +{"stream":"subscription_schedule","data":{"id":"sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP","object":"subscription_schedule","application":null,"canceled_at":"1705329724","completed_at":null,"created":1697550676,"current_phase":null,"customer":"cus_NGoTFiJFVbSsvZ","default_settings":{"application_fee_percent":null,"automatic_tax":{"enabled":false, "liability": null},"billing_cycle_anchor":"automatic","billing_thresholds":null,"collection_method":"charge_automatically","default_payment_method":null,"default_source":null,"description":"Test Test","invoice_settings":"{'account_tax_ids': None, 'days_until_due': None, 'issuer': {'type': 'self'}}","on_behalf_of":null,"transfer_data":null},"end_behavior":"cancel","livemode":false,"metadata":{},"phases":[{"add_invoice_items":[],"application_fee_percent":null,"automatic_tax":{"enabled":true, "liability": {"type": "self"}},"billing_cycle_anchor":null,"billing_thresholds":null,"collection_method":"charge_automatically","coupon":null,"currency":"usd","default_payment_method":null,"default_tax_rates":[],"description":"Test Test","end_date":1705499476,"invoice_settings":"{'account_tax_ids': None, 'days_until_due': None, 'issuer': None}","items":[{"billing_thresholds":null,"metadata":{},"plan":"price_1MSHZoEcXtiJtvvh6O8TYD8T","price":"price_1MSHZoEcXtiJtvvh6O8TYD8T","quantity":1,"tax_rates":[]}],"metadata":{},"on_behalf_of":null,"proration_behavior":"create_prorations","start_date":1697550676,"transfer_data":null,"trial_end":null}],"released_at":null,"released_subscription":null,"renewal_interval":null,"status":"canceled","subscription":"sub_1O2Dg0EcXtiJtvvhz7Q4zS0n","test_clock":null,"updated":1697550676},"emitted_at":1705636378620} {"stream": "transfers", "data": {"id": "tr_1NH18zEcXtiJtvvhnd827cNO", "object": "transfer", "amount": 10000, "amount_reversed": 0, "balance_transaction": "txn_1NH190EcXtiJtvvhBO3PeR7p", "created": 1686301085, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NH18zEYmRTj5on1GkCCsqLK", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/transfers/tr_1NH18zEcXtiJtvvhnd827cNO/reversals"}, "reversed": false, "source_transaction": null, "source_type": "card", "transfer_group": null, "updated": 1686301085}, "emitted_at": 1697627313262} {"stream": "transfers", "data": {"id": "tr_1NGoaCEcXtiJtvvhjmHtOGOm", "object": "transfer", "amount": 100, "amount_reversed": 100, "balance_transaction": "txn_1NGoaDEcXtiJtvvhsZrNMsdJ", "created": 1686252800, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NGoaCEYmRTj5on1LAlAIG3a", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [{"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}], "has_more": false, "total_count": 1.0, "url": "/v1/transfers/tr_1NGoaCEcXtiJtvvhjmHtOGOm/reversals"}, "reversed": true, "source_transaction": null, "source_type": "card", "transfer_group": "ORDER10", "updated": 1686252800}, "emitted_at": 1697627313264} {"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "destination_details": {"card": {"reference": "5871771120000631", "reference_status": "available", "reference_type": "acquirer_reference_number", "type": "refund"}, "type": "card"}, "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1701882752716} @@ -69,4 +70,4 @@ {"stream": "invoice_line_items", "data": {"id": "il_1MX2yfEcXtiJtvvhiunY2j1x", "object": "line_item", "amount": 25200, "amount_excluding_tax": 25200, "currency": "usd", "description": "edgao-test-product", "discount_amounts": [{"amount": 2520, "discount": "di_1MX2ysEcXtiJtvvh8ORqRVKm"}], "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice_item": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 2, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": true, "tax_rate": "txr_1MX2yfEcXtiJtvvhVcMEMTRj", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "12600", "invoice_id": "in_1MX2yFEcXtiJtvvhMXhUCgKx"}, "emitted_at": 1697627336449} {"stream": "subscription_items", "data": {"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}, "emitted_at": 1697627337431} {"stream": "transfer_reversals", "data": {"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}, "emitted_at": 1697627338960} -{"stream": "usage_records", "data": {"id": "sis_1ODTdwEcXtiJtvvhZChEVsbN", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": 1700229076}, "subscription_item": "si_OptSP2o3XZUBpx", "total_usage": 1}, "emitted_at": 1700233660884} \ No newline at end of file +{"stream": "usage_records", "data": {"id": "sis_1OUqWiEcXtiJtvvh3WGqc4Vk", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": 1702821076}, "subscription_item": "si_OptSP2o3XZUBpx", "total_usage": 1}, "emitted_at": 1700233660884} diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index a8ed671a8292..971f33a69dd1 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -3,43 +3,7 @@ # -import sys -import traceback -from datetime import datetime -from typing import List - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type -from source_stripe import SourceStripe - - -def _get_source(args: List[str]): - catalog_path = AirbyteEntrypoint.extract_catalog(args) - config_path = AirbyteEntrypoint.extract_config(args) - try: - return SourceStripe( - SourceStripe.read_catalog(catalog_path) if catalog_path else None, - SourceStripe.read_config(config_path) if config_path else None, - ) - except Exception as error: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - return None - +from source_stripe.run import run if __name__ == "__main__": - _args = sys.argv[1:] - source = _get_source(_args) - if source: - launch(source, _args) + run() diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index 257fd8162b86..75c4e5b6eccb 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 5.1.0 + dockerImageTag: 5.2.4 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe icon: stripe.svg license: ELv2 name: Stripe + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-stripe registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-stripe/poetry.lock b/airbyte-integrations/connectors/source-stripe/poetry.lock new file mode 100644 index 000000000000..250155d9a76c --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/poetry.lock @@ -0,0 +1,1062 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.60.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.60.1.tar.gz", hash = "sha256:fc5212b2962c1dc6aca9cc6f1c2000d7636b7509915846c126420c2b0c814317"}, + {file = "airbyte_cdk-0.60.1-py3-none-any.whl", hash = "sha256:94b33c0f6851d1e2546eac3cec54c67489239595d9e0a496ef57c3fc808e89e3"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.6" +files = [ + {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, + {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "stripe" +version = "2.56.0" +description = "Python bindings for the Stripe API" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "stripe-2.56.0-py2.py3-none-any.whl", hash = "sha256:6c685eeadf9e3608315b6d84b4f5f2da2909179b65633ce20f296be22ed21a98"}, + {file = "stripe-2.56.0.tar.gz", hash = "sha256:2ff904fb8dee0d25f135059468a876852d24dc8cbe0b45d7aff56a028045777c"}, +] + +[package.dependencies] +requests = {version = ">=2.20", markers = "python_version >= \"3.0\""} + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "e1b6a4bb5a2d863623daeb1a4194106b45024cdba1d06cfbfe85a91949cad482" diff --git a/airbyte-integrations/connectors/source-stripe/pyproject.toml b/airbyte-integrations/connectors/source-stripe/pyproject.toml new file mode 100644 index 000000000000..fc915f21ffaa --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "5.2.4" +name = "source-stripe" +description = "Source implementation for Stripe." +authors = [ "Airbyte ",] +license = "Elv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/stripe" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_stripe" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +stripe = "==2.56.0" +pendulum = "==2.1.2" +airbyte-cdk = "==0.60.1" + +[tool.poetry.scripts] +source-stripe = "source_stripe.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest = "^6.1" +freezegun = "==1.2.2" +pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-stripe/requirements.txt b/airbyte-integrations/connectors/source-stripe/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-stripe/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py deleted file mode 100644 index aab9a737d197..000000000000 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk==0.55.2", "stripe==2.56.0", "pendulum==2.1.2"] - -TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", "requests_mock~=1.8", "freezegun==1.2.2"] - -setup( - name="source_stripe", - description="Source implementation for Stripe.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py new file mode 100644 index 000000000000..b5a321986359 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_stripe import SourceStripe + + +def _get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + try: + return SourceStripe( + SourceStripe.read_catalog(catalog_path) if catalog_path else None, + SourceStripe.read_config(config_path) if config_path else None, + SourceStripe.read_state(state_path) if state_path else None, + ) + except Exception as error: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = _get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json index b1a68dbcb95d..36dc095652aa 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json @@ -6,6 +6,24 @@ "business_profile": { "type": ["null", "object"], "properties": { + "annual_revenue": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "fiscal_year_end": { + "type": ["null", "string"] + } + } + }, + "estimated_worker_count": { + "type": ["null", "integer"] + }, "mcc": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json index 9a1130c1c5b9..90361867fd2d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json @@ -13,6 +13,9 @@ "account_holder_type": { "type": ["null", "string"] }, + "account_type": { + "type": ["null", "string"] + }, "bank_name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json index b331e12c6427..3aec7668d5ae 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json @@ -25,6 +25,13 @@ "type": ["null", "object"], "properties": { "enabled": { "type": ["null", "boolean"] }, + "liability": { + "type": ["null", "object"], + "properties": { + "account": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] } + } + }, "status": { "type": ["null", "string"] } } }, @@ -369,6 +376,17 @@ "footer": { "type": ["null", "string"] }, + "issuer": { + "type": ["null", "object"], + "properties": { + "account": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, "metadata": { "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index 6959909cc77f..c21e5c93fd3e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -530,6 +530,14 @@ } } }, + "issuer": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + } + } + }, "latest_revision": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 1a720f6fd034..89f180cd4532 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -49,6 +49,12 @@ "billing_cycle_anchor": { "type": ["null", "number"] }, + "billing_cycle_anchor_config": { + "type": ["null", "object"] + }, + "invoice_settings": { + "type": ["null", "object"] + }, "cancel_at_period_end": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index ee85adfb2162..49479c5cc78e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -14,11 +14,14 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message.repository import InMemoryMessageRepository +from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade -from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor +from airbyte_cdk.sources.streams.concurrent.cursor import Comparable, ConcurrentCursor, CursorField, NoopCursor +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException from airbyte_protocol.models import SyncMode @@ -42,6 +45,10 @@ _MAX_CONCURRENCY = 20 _DEFAULT_CONCURRENCY = 10 _CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +_REFUND_STREAM_NAME = "refunds" +_INCREMENTAL_CONCURRENCY_EXCLUSION = { + _REFUND_STREAM_NAME, # excluded because of the upcoming changes in terms of cursor https://github.com/airbytehq/airbyte/issues/34332 +} USE_CACHE = not _CACHE_DISABLED STRIPE_TEST_ACCOUNT_PREFIX = "sk_test_" @@ -49,8 +56,12 @@ class SourceStripe(ConcurrentSourceAdapter): message_repository = InMemoryMessageRepository(entrypoint_logger.level) + _SLICE_BOUNDARY_FIELDS_BY_IMPLEMENTATION = { + Events: ("created[gte]", "created[lte]"), + CreatedCursorIncrementalStripeStream: ("created[gte]", "created[lte]"), + } - def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], **kwargs): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs): if config: concurrency_level = min(config.get("num_workers", _DEFAULT_CONCURRENCY), _MAX_CONCURRENCY) else: @@ -60,6 +71,7 @@ def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional concurrency_level, concurrency_level // 2, logger, self._slice_logger, self.message_repository ) super().__init__(concurrent_source) + self._state = state if catalog: self._streams_configured_as_full_refresh = { configured_stream.stream.name @@ -71,9 +83,8 @@ def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional self._streams_configured_as_full_refresh = set() @staticmethod - def validate_and_fill_with_defaults(config: MutableMapping) -> MutableMapping: - start_date, lookback_window_days, slice_range = ( - config.get("start_date"), + def validate_and_fill_with_defaults(config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + lookback_window_days, slice_range = ( config.get("lookback_window_days"), config.get("slice_range"), ) @@ -86,9 +97,9 @@ def validate_and_fill_with_defaults(config: MutableMapping) -> MutableMapping: internal_message=message, failure_type=FailureType.config_error, ) - if start_date: - # verifies the start_date is parseable - SourceStripe._start_date_to_timestamp(start_date) + + # verifies the start_date in the config is valid + SourceStripe._start_date_to_timestamp(config) if slice_range is None: config["slice_range"] = 365 elif not isinstance(slice_range, int) or slice_range < 1: @@ -100,7 +111,7 @@ def validate_and_fill_with_defaults(config: MutableMapping) -> MutableMapping: ) return config - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + def check_connection(self, logger: AirbyteLogger, config: MutableMapping[str, Any]) -> Tuple[bool, Any]: self.validate_and_fill_with_defaults(config) stripe.api_key = config["client_secret"] try: @@ -167,14 +178,11 @@ def get_api_call_budget(self, config: Mapping[str, Any]) -> AbstractAPIBudget: return HttpAPIBudget(policies=policies) - def streams(self, config: Mapping[str, Any]) -> List[Stream]: + def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: config = self.validate_and_fill_with_defaults(config) authenticator = TokenAuthenticator(config["client_secret"]) - if "start_date" in config: - start_timestamp = self._start_date_to_timestamp(config["start_date"]) - else: - start_timestamp = pendulum.datetime(2017, 1, 25).int_timestamp + start_timestamp = self._start_date_to_timestamp(config) args = { "authenticator": authenticator, "account_id": config["account_id"], @@ -289,7 +297,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # The Refunds stream does not utilize the Events API as it created issues with data loss during the incremental syncs. # Therefore, we're using the regular API with the `created` cursor field. A bug has been filed with Stripe. # See more at https://github.com/airbytehq/oncall/issues/3090, https://github.com/airbytehq/oncall/issues/3428 - CreatedCursorIncrementalStripeStream(name="refunds", path="refunds", **incremental_args), + CreatedCursorIncrementalStripeStream(name=_REFUND_STREAM_NAME, path="refunds", **incremental_args), UpdatedCursorIncrementalStripeStream( name="payment_methods", path="payment_methods", @@ -464,12 +472,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ), UpdatedCursorIncrementalStripeLazySubStream( name="bank_accounts", - path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice['parent']['id']}/sources", + path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice['parent']['id']}/bank_accounts", parent=self.customers(expand_items=["data.sources"], **args), event_types=["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"], legacy_cursor_field=None, sub_items_attr="sources", - extra_request_params={"object": "bank_account"}, response_filter=lambda record: record["object"] == "bank_account", **args, ), @@ -512,21 +519,44 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ), ] - return [ - StreamFacade.create_from_stream(stream, self, entrypoint_logger, self._create_empty_state(), NoopCursor()) - if stream.name in self._streams_configured_as_full_refresh - else stream - for stream in streams - ] + state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in streams}, state=self._state) + return [self._to_concurrent(stream, self._start_date_to_timestamp(config), state_manager) for stream in streams] + + def _to_concurrent(self, stream: Stream, fallback_start, state_manager: ConnectorStateManager) -> Stream: + if stream.name in self._streams_configured_as_full_refresh: + return StreamFacade.create_from_stream(stream, self, entrypoint_logger, self._create_empty_state(), NoopCursor()) + + state = state_manager.get_stream_state(stream.name, stream.namespace) + slice_boundary_fields = self._SLICE_BOUNDARY_FIELDS_BY_IMPLEMENTATION.get(type(stream)) + if slice_boundary_fields and stream.name not in _INCREMENTAL_CONCURRENCY_EXCLUSION: + cursor_field = CursorField(stream.cursor_field) if isinstance(stream.cursor_field, str) else CursorField(stream.cursor_field[0]) + converter = EpochValueConcurrentStreamStateConverter() + cursor = ConcurrentCursor( + stream.name, + stream.namespace, + state_manager.get_stream_state(stream.name, stream.namespace), + self.message_repository, + state_manager, + converter, + cursor_field, + slice_boundary_fields, + fallback_start, + ) + return StreamFacade.create_from_stream(stream, self, entrypoint_logger, state, cursor) + + return stream def _create_empty_state(self) -> MutableMapping[str, Any]: - # The state is known to be empty because concurrent CDK is currently only used for full refresh return {} @staticmethod - def _start_date_to_timestamp(start_date: str) -> int: + def _start_date_to_timestamp(config: Mapping[str, Any]) -> int: + if "start_date" not in config: + return pendulum.datetime(2017, 1, 25).int_timestamp # type: ignore # pendulum not typed + + start_date = config["start_date"] try: - return pendulum.parse(start_date).int_timestamp + return pendulum.parse(start_date).int_timestamp # type: ignore # pendulum not typed except pendulum.parsing.exceptions.ParserError as e: message = f"Invalid start date {start_date}. Please use YYYY-MM-DDTHH:MM:SSZ format." raise AirbyteTracedException( diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 109753a844d7..d8958d9453b7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -21,6 +21,7 @@ STRIPE_API_VERSION = "2022-11-15" CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +IS_TESTING = os.environ.get("DEPLOYMENT_MODE") == "testing" USE_CACHE = not CACHE_DISABLED @@ -197,6 +198,12 @@ def request_headers(self, **kwargs) -> Mapping[str, Any]: headers["Stripe-Account"] = self.account_id return headers + def retry_factor(self) -> float: + """ + Override for testing purposes + """ + return 0 if IS_TESTING else super(StripeStream, self).retry_factor + class IStreamSelector(ABC): @abstractmethod @@ -700,20 +707,6 @@ def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, items_next_pages = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, **kwargs) yield from chain(items, items_next_pages) - def stream_slices( - self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( - sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state - ) - for stream_slice in parent_stream_slices: - parent_records = self.parent.read_records( - sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state - ) - for record in parent_records: - self.logger.info(f"Fetching parent stream slices for stream {self.name}.") - yield {"parent": record} - class IncrementalStripeLazySubStreamSelector(IStreamSelector): def __init__(self, updated_cursor_incremental_stream: UpdatedCursorIncrementalStripeStream, lazy_sub_stream: StripeLazySubStream): @@ -842,7 +835,7 @@ def parse_response(self, response: requests.Response, *args, **kwargs) -> Iterab # as the events API does not support expandable items. Parent class will try getting sub-items from this object, # then from its own API. In case there are no sub-items at all for this entity, API will raise 404 error. self.logger.warning( - "Data was not found for URL: {response.request.url}. " + f"Data was not found for URL: {response.request.url}. " "If this is a path for getting child attributes like /v1/checkout/sessions//line_items when running " "the incremental sync, you may safely ignore this warning." ) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index 884543613e5f..0463e204fdb0 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -5,9 +5,12 @@ import os import pytest +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.test.state_builder import StateBuilder os.environ["CACHE_DISABLED"] = "true" +os.environ["DEPLOYMENT_MODE"] = "testing" @pytest.fixture(name="config") @@ -39,10 +42,14 @@ def stream_by_name(config): from source_stripe.source import SourceStripe def mocker(stream_name, source_config=config): - source = SourceStripe(None, source_config) + source = SourceStripe(None, source_config, StateBuilder().build()) streams = source.streams(source_config) for stream in streams: if stream.name == stream_name: + if isinstance(stream, StreamFacade): + # to avoid breaking changes for tests, we will return the legacy test. Tests that would be affected by not having this + # would probably need to be moved to integration tests or unit tests + return stream._legacy_stream return stream return mocker diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py new file mode 100644 index 000000000000..d048407320d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "client_secret": "ConfigBuilder default client secret", + "account_id": "ConfigBuilder default account id", + "start_date": "2020-05-01T00:00:00Z" + } + + def with_account_id(self, account_id: str) -> "ConfigBuilder": + self._config["account_id"] = account_id + return self + + def with_client_secret(self, client_secret: str) -> "ConfigBuilder": + self._config["client_secret"] = client_secret + return self + + def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": + self._config["start_date"] = start_datetime.isoformat()[:-13]+"Z" + return self + + def with_lookback_window_in_days(self, number_of_days: int) -> "ConfigBuilder": + self._config["lookback_window_days"] = number_of_days + return self + + def with_slice_range_in_days(self, number_of_days: int) -> "ConfigBuilder": + self._config["slice_range"] = number_of_days + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py new file mode 100644 index 000000000000..acfe9a613271 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class StripePaginationStrategy(PaginationStrategy): + @staticmethod + def update(response: Dict[str, Any]) -> None: + response["has_more"] = True diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py new file mode 100644 index 000000000000..7a2c8219c5d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import List, Optional + +from airbyte_cdk.test.mock_http import HttpRequest +from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS + + +class StripeRequestBuilder: + + @classmethod + def accounts_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("accounts", account_id, client_secret) + + @classmethod + def application_fees_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("application_fees", account_id, client_secret) + + @classmethod + def application_fees_refunds_endpoint(cls, application_fee_id: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"application_fees/{application_fee_id}/refunds", account_id, client_secret) + + @classmethod + def customers_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("customers", account_id, client_secret) + + @classmethod + def customers_bank_accounts_endpoint(cls, customer_id: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"customers/{customer_id}/bank_accounts", account_id, client_secret) + + @classmethod + def events_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("events", account_id, client_secret) + + @classmethod + def external_accounts_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"accounts/{account_id}/external_accounts", account_id, client_secret) + + @classmethod + def issuing_authorizations_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/authorizations", account_id, client_secret) + + @classmethod + def issuing_cards_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/cards", account_id, client_secret) + + @classmethod + def issuing_transactions_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/transactions", account_id, client_secret) + + @classmethod + def payment_methods_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("payment_methods", account_id, client_secret) + + @classmethod + def persons_endpoint(cls, parent_account_id: str, account_id: str, client_secret: str, ) -> "StripeRequestBuilder": + return cls(f"accounts/{parent_account_id}/persons", account_id, client_secret) + + @classmethod + def radar_early_fraud_warnings_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("radar/early_fraud_warnings", account_id, client_secret) + + @classmethod + def reviews_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("reviews", account_id, client_secret) + + @classmethod + def _for_endpoint(cls, endpoint: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(endpoint, account_id, client_secret) + + def __init__(self, resource: str, account_id: str, client_secret: str) -> None: + self._resource = resource + self._account_id = account_id + self._client_secret = client_secret + self._any_query_params = False + self._created_gte: Optional[datetime] = None + self._created_lte: Optional[datetime] = None + self._limit: Optional[int] = None + self._object: Optional[str] = None + self._starting_after_id: Optional[str] = None + self._types: List[str] = [] + self._expands: List[str] = [] + + def with_created_gte(self, created_gte: datetime) -> "StripeRequestBuilder": + self._created_gte = created_gte + return self + + def with_created_lte(self, created_lte: datetime) -> "StripeRequestBuilder": + self._created_lte = created_lte + return self + + def with_limit(self, limit: int) -> "StripeRequestBuilder": + self._limit = limit + return self + + def with_object(self, object_name: str) -> "StripeRequestBuilder": + self._object = object_name + return self + + def with_starting_after(self, starting_after_id: str) -> "StripeRequestBuilder": + self._starting_after_id = starting_after_id + return self + + def with_any_query_params(self) -> "StripeRequestBuilder": + self._any_query_params = True + return self + + def with_types(self, types: List[str]) -> "StripeRequestBuilder": + self._types = types + return self + + def with_expands(self, expands: List[str]) -> "StripeRequestBuilder": + self._expands = expands + return self + + def build(self) -> HttpRequest: + query_params = {} + if self._created_gte: + query_params["created[gte]"] = str(int(self._created_gte.timestamp())) + if self._created_lte: + query_params["created[lte]"] = str(int(self._created_lte.timestamp())) + if self._limit: + query_params["limit"] = str(self._limit) + if self._starting_after_id: + query_params["starting_after"] = self._starting_after_id + if self._types: + query_params["types[]"] = self._types + if self._object: + query_params["object"] = self._object + if self._expands: + query_params["expand[]"] = self._expands + + if self._any_query_params: + if query_params: + raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + query_params = ANY_QUERY_PARAMS + + return HttpRequest( + url=f"https://api.stripe.com/v1/{self._resource}", + query_params=query_params, + headers={"Stripe-Account": self._account_id, "Authorization": f"Bearer {self._client_secret}"}, + ) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py new file mode 100644 index 000000000000..7495bffeb4a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py @@ -0,0 +1,10 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + + +def a_response_with_status(status_code: int) -> HttpResponse: + return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py new file mode 100644 index 000000000000..b7028590f446 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py @@ -0,0 +1,376 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["application_fee.created", "application_fee.refunded"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "application_fees" +_ENDPOINT_TEMPLATE_NAME = "application_fees" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _application_fees_request() -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[List[AirbyteStateMessage]]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_application_fee() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _application_fees_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_pagination().with_record(_an_application_fee().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _application_fees_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().build(), + ) + http_mocker.get( + _application_fees_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _application_fees_response().with_record(_an_application_fee()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [a_response_with_status(500), _application_fees_response().with_record(_an_application_fee()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + _application_fees_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_application_fee().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_application_fee().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + self._an_application_fee_event() + ).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).with_record(self._an_application_fee_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # application fees endpoint + _given_application_fees_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_application_fee_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_application_fee().build()) + + def _read(self, config: ConfigBuilder, state: Optional[List[AirbyteStateMessage]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py new file mode 100644 index 000000000000..bfde5e409d11 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py @@ -0,0 +1,519 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["application_fee.refund.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_REFUNDS_FIELD = FieldPath("refunds") +_STREAM_NAME = "application_fees_refunds" +_APPLICATION_FEES_TEMPLATE_NAME = "application_fees" +_REFUNDS_TEMPLATE_NAME = "application_fees_refunds" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _application_fees_request() -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _application_fees_refunds_request(application_fee_id: str) -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_refunds_endpoint(application_fee_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_application_fee() -> RecordBuilder: + return create_record_builder( + find_template(_APPLICATION_FEES_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _application_fees_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_APPLICATION_FEES_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_refund() -> RecordBuilder: + return create_record_builder( + find_template(_REFUNDS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _refunds_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_REFUNDS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().with_record(_an_application_fee()).build() # there needs to be a record in the parent stream for the child to be available + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]: + return json.loads(response_builder.build().body) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +def _assert_not_available(output: EntrypointOutput) -> None: + # right now, no stream statuses means stream unavailable + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + .with_record(_a_refund()) + ) + ) + ) + .with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_multiple_refunds_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_pagination() + .with_record(_a_refund().with_id("latest_refund_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # we do not use slice boundaries here because: + # * there should be no duplicates parents (application fees) returned by the stripe API as it is using cursor pagination + # * it is implicitly lower bounder by the parent creation + # * the upper boundary is not configurable and is always + _application_fees_refunds_request("parent_id").with_limit(100).with_starting_after("latest_refund_id").build(), + _refunds_response().with_record(_a_refund()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_multiple_application_fees_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_pagination() + .with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + http_mocker.get( + _application_fees_request().with_starting_after("parent_id").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_parent_stream_without_refund_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None: + # events stream is not validated as application fees is validated first + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + _assert_not_available(output) + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + http_mocker.get( + _application_fees_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + + output = self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_slice_range_and_refunds_pagination_when_read_then_do_not_slice_child(self, http_mocker: HttpMocker) -> None: + """ + This means that if the user attempt to configure the slice range, it will only apply on the parent stream + """ + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().build() + ) # catching subsequent slicing request that we don't really care for this test + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_pagination() + .with_record(_a_refund().with_id("latest_refund_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # slice range is not applied here + _application_fees_refunds_request("parent_id").with_limit(100).with_starting_after("latest_refund_id").build(), + _refunds_response().with_record(_a_refund()).build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _application_fees_response().with_record(_an_application_fee().with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + )).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + request = _application_fees_request().with_any_query_params().build() + http_mocker.get( + request, + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund().with_cursor(cursor_value)))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_refund().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_refund().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).with_record(self._a_refund_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # application fees endpoint + _given_application_fees_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_refund_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_refund().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py new file mode 100644 index 000000000000..90e61aad3166 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "authorizations" +_ENDPOINT_TEMPLATE_NAME = "issuing_authorizations" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _authorizations_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_authorizations_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_authorization() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _authorizations_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_authorizations_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_authorizations_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _authorizations_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_pagination().with_record(_an_authorization().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _authorizations_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _authorizations_response().build(), + ) + http_mocker.get( + _authorizations_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _authorizations_response().with_record(_an_authorization()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + [a_response_with_status(500), _authorizations_response().with_record(_an_authorization()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + _authorizations_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_authorizations_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_authorization().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_authorization().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).with_record(self._an_authorization_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # authorizations endpoint + _given_authorizations_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_authorization_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_authorization().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py new file mode 100644 index 000000000000..db5c32d5d9a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py @@ -0,0 +1,564 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_SOURCES_FIELD = FieldPath("sources") +_STREAM_NAME = "bank_accounts" +_CUSTOMERS_TEMPLATE_NAME = "customers_expand_data_source" +_BANK_ACCOUNTS_TEMPLATE_NAME = "bank_accounts" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +# FIXME expand[] is not documented anymore in stripe API doc (see https://github.com/airbytehq/airbyte/issues/33714) +_EXPANDS = ["data.sources"] +_OBJECT = "bank_account" +_NOT_A_BANK_ACCOUNT = RecordBuilder({"object": "NOT a bank account"}, None, None) +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _customers_request() -> StripeRequestBuilder: + return StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _customers_bank_accounts_request(customer_id: str) -> StripeRequestBuilder: + return StripeRequestBuilder.customers_bank_accounts_endpoint(customer_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_customer() -> RecordBuilder: + return create_record_builder( + find_template(_CUSTOMERS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _customers_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_CUSTOMERS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_bank_account() -> RecordBuilder: + return create_record_builder( + find_template(_BANK_ACCOUNTS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _bank_accounts_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_BANK_ACCOUNTS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_customers_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _customers_response().with_record(_a_customer()).build() # there needs to be a record in the parent stream for the child to be available + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]: + return json.loads(response_builder.build().body) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +def _assert_not_available(output: EntrypointOutput) -> None: + # right now, no stream statuses means stream unavailable + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + .with_record(_a_bank_account()) + ) + ) + ) + .with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_NOT_A_BANK_ACCOUNT) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_multiple_bank_accounts_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_pagination() + .with_record(_a_bank_account().with_id("latest_bank_account_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # we do not use slice boundaries here because: + # * there should be no duplicates parents (application fees) returned by the stripe API as it is using cursor pagination + # * it is implicitly lower bounder by the parent creation + # * the upper boundary is not configurable and is always + _customers_bank_accounts_request("parent_id").with_limit(100).with_starting_after("latest_bank_account_id").build(), + _bank_accounts_response().with_record(_a_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_multiple_customers_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_pagination() + .with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_starting_after("parent_id").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_parent_stream_without_bank_accounts_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None: + # events stream is not validated as application fees is validated first + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + _assert_not_available(output) + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_slice_range_and_bank_accounts_pagination_when_read_then_do_not_slice_child(self, http_mocker: HttpMocker) -> None: + """ + This means that if the user attempt to configure the slice range, it will only apply on the parent stream + """ + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _customers_response().build() + ) # catching subsequent slicing request that we don't really care for this test + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_pagination() + .with_record(_a_bank_account().with_id("latest_bank_account_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # slice range is not applied here + _customers_bank_accounts_request("parent_id").with_limit(100).with_starting_after("latest_bank_account_id").build(), + _bank_accounts_response().with_record(_a_bank_account()).build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record(_a_customer()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _customers_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _customers_response().with_record(_a_customer().with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + )).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + request = _customers_request().with_any_query_params().build() + http_mocker.get( + request, + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker: HttpMocker) -> None: + # If stripe takes some time to ingest the data, we should recommend to use a lookback window when syncing the bank_accounts stream + # to make sure that we don't lose data between the first and the second sync + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_bank_account().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_bank_account().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).with_record(self._a_bank_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # customer endpoint + _given_customers_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + @HttpMocker() + def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None: + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, _NOT_A_BANK_ACCOUNT.build()) + ).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + def _a_bank_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_bank_account().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py new file mode 100644 index 000000000000..413c1e15d2a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_card.created", "issuing_card.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "cards" +_ENDPOINT_TEMPLATE_NAME = "issuing_cards" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _cards_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_cards_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_card() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _cards_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_cards_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_cards_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _cards_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_pagination().with_record(_a_card().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _cards_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _cards_response().build(), + ) + http_mocker.get( + _cards_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _cards_response().with_record(_a_card()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_any_query_params().build(), + [a_response_with_status(500), _cards_response().with_record(_a_card()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _cards_request().with_any_query_params().build(), + _cards_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_cards_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_card().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_card().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).with_record(self._a_card_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # cards endpoint + _given_cards_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_card_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_card().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py new file mode 100644 index 000000000000..7f8a0800b97e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py @@ -0,0 +1,342 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["radar.early_fraud_warning.created", "radar.early_fraud_warning.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "early_fraud_warnings" +_ENDPOINT_TEMPLATE_NAME = "radar_early_fraud_warnings" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _early_fraud_warnings_request() -> StripeRequestBuilder: + return StripeRequestBuilder.radar_early_fraud_warnings_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_early_fraud_warning() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _early_fraud_warnings_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_early_fraud_warnings_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.radar_early_fraud_warnings_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _early_fraud_warnings_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_pagination().with_record(_an_early_fraud_warning().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _early_fraud_warnings_request().with_starting_after("last_record_id_from_first_page").with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + [a_response_with_status(500), _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + _early_fraud_warnings_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_early_fraud_warnings_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).with_record(self._an_early_fraud_warning_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # radar/early_fraud_warnings endpoint + _given_early_fraud_warnings_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_early_fraud_warning_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py new file mode 100644 index 000000000000..14942b03c54c --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py @@ -0,0 +1,276 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_STREAM_NAME = "events" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) +_SECOND_REQUEST = timedelta(seconds=1) +_THIRD_REQUEST = timedelta(seconds=2) + + +def _a_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=73)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _a_record() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _a_response() -> HttpResponseBuilder: + return create_response_builder(find_template("events", __file__), FieldPath("data"), pagination_strategy=StripePaginationStrategy()) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).with_record(_a_record()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_pagination().with_record(_a_record().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _a_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).with_record(_a_record()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_start_date_before_30_days_stripe_limit_and_slice_range_when_read_then_perform_request_before_30_days(self, http_mocker: HttpMocker) -> None: + """ + This case is special because the source queries for a time range that is before 30 days. That being said as of 2023-12-13, the API + mentions that "We only guarantee access to events through the Retrieve Event API for 30 days." (see + https://stripe.com/docs/api/events) + """ + start_date = _NOW - timedelta(days=61) + slice_range = timedelta(days=30) + slice_datetime = start_date + slice_range + http_mocker.get( # this first request has both gte and lte before 30 days even though we know there should not be records returned + _a_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + _SECOND_REQUEST).with_created_lte(slice_datetime + slice_range + _SECOND_REQUEST).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + slice_range + _THIRD_REQUEST).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_lookback_window_when_read_then_request_before_start_date(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + lookback_window = timedelta(days=10) + http_mocker.get( + _a_request().with_created_gte(start_date - lookback_window).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_lookback_window_in_days(lookback_window.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + http_mocker.get( + _a_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + _SECOND_REQUEST).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config().with_start_date(_A_START_DATE), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _a_response().with_record(_a_record()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + [a_response_with_status(500), _a_response().with_record(_a_record()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_when_read_then_validate_availability_for_full_refresh_and_incremental(self, http_mocker: HttpMocker) -> None: + request = _a_request().with_any_query_params().build() + http_mocker.get( + request, + _a_response().build(), + ) + self._read(_config().with_start_date(_A_START_DATE)) + http_mocker.assert_number_of_calls(request, 3) # one call for full_refresh availability, one call for incremental availability and one call for the actual read + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_field(self, http_mocker: HttpMocker) -> None: + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {"events": {"created": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_use_state_for_query_params(self, http_mocker: HttpMocker) -> None: + state_value = _A_START_DATE + timedelta(seconds=1) + availability_check_requests = _a_request().with_any_query_params().build() + http_mocker.get( + availability_check_requests, + _a_response().with_record(_a_record()).build(), + ) + http_mocker.get( + _a_request().with_created_gte(state_value + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).build(), + ) + + self._read( + _config().with_start_date(_A_START_DATE), + StateBuilder().with_stream_state("events", {"created": int(state_value.timestamp())}).build() + ) + + # request matched http_mocker + + @HttpMocker() + def test_given_state_more_recent_than_cursor_when_read_then_return_state_based_on_cursor_field(self, http_mocker: HttpMocker) -> None: + """ + We do not see exactly how this case can happen in a real life scenario but it is used to see if at least one state message + would be populated given that no partitions were created. + """ + cursor_value = int(_A_START_DATE.timestamp()) + 1 + more_recent_than_record_cursor = int(_NOW.timestamp()) - 1 + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record().with_cursor(cursor_value)).build(), + ) + + output = self._read( + _config().with_start_date(_A_START_DATE), + StateBuilder().with_stream_state("events", {"created": more_recent_than_record_cursor}).build() + ) + + assert output.most_recent_state == {"events": {"created": more_recent_than_record_cursor}} + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py new file mode 100644 index 000000000000..cbd08bce1a5c --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py @@ -0,0 +1,361 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_OBJECT = "bank_account" +_STREAM_NAME = "external_account_bank_accounts" +_ENDPOINT_TEMPLATE_NAME = "external_bank_accounts" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _external_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_external_bank_account() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _external_bank_accounts_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _external_bank_accounts_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_pagination().with_record(_an_external_bank_account().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _external_accounts_request().with_starting_after("last_record_id_from_first_page").with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [a_response_with_status(500), _external_bank_accounts_response().with_record(_an_external_bank_account()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + _external_bank_accounts_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_external_bank_account().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_object_is_not_back_account_when_read_then_filter_out(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + + _given_external_accounts_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, {"object": "not a bank account"}) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_external_bank_account().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # external_accounts endpoint + _given_external_accounts_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_external_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_external_bank_account().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py new file mode 100644 index 000000000000..3635f7bd6d6d --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py @@ -0,0 +1,366 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_OBJECT = "card" +_STREAM_NAME = "external_account_cards" +_ENDPOINT_TEMPLATE_NAME = "external_account_cards" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _external_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_external_account_card() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _external_accounts_card_response() -> HttpResponseBuilder: + """ + WARNING: this response will not fully match the template as external accounts card are queried by ID and the field "url" is not updated + to match that (it is currently hardcoded to "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts"). As this has no impact on the + tests, we will leave it as is for now. + """ + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _external_accounts_card_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_pagination().with_record(_an_external_account_card().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _external_accounts_request().with_starting_after("last_record_id_from_first_page").with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [a_response_with_status(500), _external_accounts_card_response().with_record(_an_external_account_card()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + _external_accounts_card_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_external_account_card().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_object_is_not_back_account_when_read_then_filter_out(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + + _given_external_accounts_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, {"object": "not a card"}) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_external_account_card().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # external_accounts endpoint + _given_external_accounts_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_external_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_external_account_card().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py new file mode 100644 index 000000000000..10785b6c4777 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py @@ -0,0 +1,347 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = [ + "payment_method.attached", + "payment_method.automatically_updated", + "payment_method.detached", + "payment_method.updated", +] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "payment_methods" +_ENDPOINT_TEMPLATE_NAME = "payment_methods" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _payment_methods_request() -> StripeRequestBuilder: + return StripeRequestBuilder.payment_methods_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_payment_method() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _payment_methods_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_payment_methods_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.payment_methods_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _payment_methods_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_pagination().with_record(_a_payment_method().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _payment_methods_request().with_starting_after("last_record_id_from_first_page").with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _payment_methods_response().with_record(_a_payment_method()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + [a_response_with_status(500), _payment_methods_response().with_record(_a_payment_method()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + _payment_methods_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_payment_method().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_payment_method().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).with_record(self._a_payment_method_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # payment_methods endpoint + _given_payment_methods_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_payment_method_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_payment_method().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py new file mode 100644 index 000000000000..7df5857b0000 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py @@ -0,0 +1,642 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import List +from unittest import TestCase + +import freezegun +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStreamStatus, Level +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_STREAM_NAME = "persons" +_ACCOUNT_ID = "acct_1G9HZLIEn49ers" +_CLIENT_SECRET = "ConfigBuilder default client secret" +_NOW = datetime.now(timezone.utc) +_CONFIG = { + "client_secret": _CLIENT_SECRET, + "account_id": _ACCOUNT_ID, +} +_NO_STATE = StateBuilder().build() +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _create_config() -> ConfigBuilder: + return ConfigBuilder().with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="persons", sync_mode=sync_mode).build() + + +def _create_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_persons_request(parent_account_id: str = _ACCOUNT_ID) -> StripeRequestBuilder: + return StripeRequestBuilder.persons_endpoint(parent_account_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template("accounts", __file__), + records_path=FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _create_record(resource: str) -> RecordBuilder: + return create_record_builder( + find_template(resource, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created") + ) + + +def _create_persons_event_record(event_type: str) -> RecordBuilder: + event_record = create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + person_record = create_record_builder( + find_template("persons", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created") + ) + + return event_record.with_field(NestedPath(["data", "object"]), person_record.build()).with_field(NestedPath(["type"]), event_type) + + +def emits_successful_sync_status_messages(status_messages: List[AirbyteStreamStatus]) -> bool: + return (len(status_messages) == 3 and status_messages[0] == AirbyteStreamStatus.STARTED + and status_messages[1] == AirbyteStreamStatus.RUNNING and status_messages[2] == AirbyteStreamStatus.COMPLETE) + + +@freezegun.freeze_time(_NOW.isoformat()) +class PersonsTest(TestCase): + @HttpMocker() + def test_full_refresh(self, http_mocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_parent_pagination(self, http_mocker): + # First parent stream accounts first page request + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts").with_id("last_page_record_id")).with_pagination().build(), + ) + + # Second parent stream accounts second page request + http_mocker.get( + _create_accounts_request().with_limit(100).with_starting_after("last_page_record_id").build(), + _create_response().with_record(record=_create_record("accounts").with_id("last_page_record_id")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request(parent_account_id="last_page_record_id").with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # The persons stream makes a final call to events endpoint + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 4 + + @HttpMocker() + def test_substream_pagination(self, http_mocker): + # First parent stream accounts first page request + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons").with_id("last_page_record_id")).with_pagination().build(), + ) + + # Persons stream second page request + http_mocker.get( + _create_persons_request().with_limit(100).with_starting_after("last_page_record_id").build(), + _create_response().with_record(record=_create_record("persons")).with_record( + record=_create_record("persons")).build(), + ) + + # The persons stream makes a final call to events endpoint + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 4 + + @HttpMocker() + def test_accounts_400_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + a_response_with_status(400), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_persons_400_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(400), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy. They are however reported in the log messages + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_accounts_401_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + a_response_with_status(401), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + + assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_persons_401_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(401), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + + assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_persons_403_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(403), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_incremental_with_recent_state(self, http_mocker: HttpMocker): + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build() + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=state, + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_incremental_with_deleted_event(self, http_mocker: HttpMocker): + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.deleted")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.deleted")).build(), + ) + + state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build() + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=state, + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + assert actual_messages.records[0].record.data.get("is_deleted") + + @HttpMocker() + def test_incremental_with_newer_start_date(self, http_mocker): + start_datetime = _NOW - timedelta(days=7) + state_datetime = _NOW - timedelta(days=15) + config = _create_config().with_start_date(start_datetime).build() + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(start_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build() + source = SourceStripe(config=config, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state) + actual_messages = read( + source, + config=config, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=state, + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_rate_limited_parent_stream_accounts(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_record("accounts")).build(), + ], + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_rate_limited_substream_persons(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_rate_limited_incremental_events(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # Mock when check_availability is run on the persons incremental stream + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record( + record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ] + ) + + state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build() + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=state, + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_rate_limit_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + # Used to pass the initial check_availability before starting the sync + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + a_response_with_status(429), # Returns 429 on all subsequent requests to test the maximum number of retries + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.errors) == 1 + + @HttpMocker() + def test_incremental_rate_limit_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # Mock when check_availability is run on the persons incremental stream + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record( + record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + a_response_with_status(429), # Returns 429 on all subsequent requests to test the maximum number of retries + ) + + state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build() + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental), state=state) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=state, + ) + + assert len(actual_messages.errors) == 1 + + @HttpMocker() + def test_server_error_parent_stream_accounts(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + [ + a_response_with_status(500), + _create_response().with_record(record=_create_record("accounts")).build(), + ], + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_server_error_substream_persons(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + a_response_with_status(500), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_server_error_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + # Used to pass the initial check_availability before starting the sync + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + a_response_with_status(500), # Returns 429 on all subsequent requests to test the maximum number of retries + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(), state=_NO_STATE) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.errors) == 1 diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py new file mode 100644 index 000000000000..d454faec79e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["review.closed", "review.opened"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "reviews" +_ENDPOINT_TEMPLATE_NAME = "reviews" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _reviews_request() -> StripeRequestBuilder: + return StripeRequestBuilder.reviews_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_review() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _reviews_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_reviews_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.reviews_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _reviews_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_pagination().with_record(_a_review().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _reviews_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _reviews_response().build(), + ) + http_mocker.get( + _reviews_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _reviews_response().with_record(_a_review()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + [a_response_with_status(500), _reviews_response().with_record(_a_review()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + _reviews_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_reviews_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_review().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_review().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).with_record(self._a_review_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # reviews endpoint + _given_reviews_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_review_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_review().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py new file mode 100644 index 000000000000..f0a04e093760 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_transaction.created", "issuing_transaction.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "transactions" +_ENDPOINT_TEMPLATE_NAME = "issuing_transactions" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _transactions_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_transactions_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any], state: Optional[TState]) -> SourceStripe: + return SourceStripe(catalog, config, state) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_transaction() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _transactions_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_transactions_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_transactions_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _transactions_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config, state), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_pagination().with_record(_a_transaction().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _transactions_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _transactions_response().build(), + ) + http_mocker.get( + _transactions_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _transactions_response().with_record(_a_transaction()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + [a_response_with_status(500), _transactions_response().with_record(_a_transaction()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + _transactions_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_transactions_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_transaction().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_transaction().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).with_record(self._a_transaction_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # transactions endpoint + _given_transactions_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_transaction_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_transaction().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json new file mode 100644 index 000000000000..4ded0c0a8919 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json @@ -0,0 +1,7 @@ +{ + "error": { + "message": "Your account is not set up to use Issuing. Please visit https://dashboard.stripe.com/issuing/overview to get started.", + "request_log_url": "https://dashboard.stripe.com/test/logs/req_OzHOvvVQ4ALtKm?t=1702476901", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json new file mode 100644 index 000000000000..67f5dfd22e07 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "Invalid API Key provided: sk_test_*****************************************************mFeM", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json new file mode 100644 index 000000000000..9fadb9f2fe1f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json @@ -0,0 +1,8 @@ +{ + "error": { + "code": "oauth_not_supported", + "message": "This application does not have the required permissions for this endpoint on account 'acct_1G9HZLIEn49ers'.", + "request_log_url": "https://dashboard.stripe.com/acct_1IB2IIRPz4Eoy76F/test/logs/req_yfhmzM1ChMWuhX?t=1703806215", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json new file mode 100644 index 000000000000..249f882eecc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Request rate limit exceeded. Learn more about rate limits here https://stripe.com/docs/rate-limits.", + "type": "invalid_request_error", + "code": "rate_limit", + "doc_url": "https://stripe.com/docs/error-codes/rate-limit" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json new file mode 100644 index 000000000000..0077e9a45a61 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json @@ -0,0 +1,3 @@ +{ + "unknown": "maxi297: I could not reproduce the issue hence this response will not look like the actual 500 status response" +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json new file mode 100644 index 000000000000..475961a4ed4f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json @@ -0,0 +1,136 @@ +{ + "object": "list", + "url": "/v1/accounts", + "has_more": false, + "data": [ + { + "id": "acct_1G9HZLIEn49ers", + "object": "account", + "business_profile": { + "mcc": null, + "name": null, + "product_description": null, + "support_address": null, + "support_email": null, + "support_phone": null, + "support_url": null, + "url": null + }, + "business_type": null, + "capabilities": { + "card_payments": "inactive", + "transfers": "inactive" + }, + "charges_enabled": false, + "country": "US", + "created": 1695830751, + "default_currency": "usd", + "details_submitted": false, + "email": "john.lynch@49ers.com", + "external_accounts": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/accounts/acct_1G9HZLIEn49ers/external_accounts" + }, + "future_requirements": { + "alternatives": [], + "current_deadline": null, + "currently_due": [], + "disabled_reason": null, + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "metadata": {}, + "payouts_enabled": false, + "requirements": { + "alternatives": [], + "current_deadline": null, + "currently_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "disabled_reason": "requirements.past_due", + "errors": [], + "eventually_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "past_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "pending_verification": [] + }, + "settings": { + "bacs_debit_payments": {}, + "branding": { + "icon": null, + "logo": null, + "primary_color": null, + "secondary_color": null + }, + "card_issuing": { + "tos_acceptance": { + "date": null, + "ip": null + } + }, + "card_payments": { + "decline_on": { + "avs_failure": false, + "cvc_failure": false + }, + "statement_descriptor_prefix": null, + "statement_descriptor_prefix_kana": null, + "statement_descriptor_prefix_kanji": null + }, + "dashboard": { + "display_name": null, + "timezone": "Etc/UTC" + }, + "payments": { + "statement_descriptor": null, + "statement_descriptor_kana": null, + "statement_descriptor_kanji": null + }, + "payouts": { + "debit_negative_balances": false, + "schedule": { + "delay_days": 2, + "interval": "daily" + }, + "statement_descriptor": null + }, + "sepa_debit_payments": {} + }, + "tos_acceptance": { + "date": null, + "ip": null, + "user_agent": null + }, + "type": "custom" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json new file mode 100644 index 000000000000..97bb806e6bbe --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json @@ -0,0 +1,138 @@ +{ + "object": "list", + "url": "/v1/application_fees", + "has_more": false, + "data": [ + { + "id": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "object": "application_fee", + "account": "acct_164wxjKbnvuxQXGu", + "amount": 105, + "amount_refunded": 105, + "application": "ca_32D88BD1qLklliziD7gYQvctJIhWBSQ7", + "balance_transaction": "txn_1032HU2eZvKYlo2CEPtcnUvl", + "charge": "ch_1B73DOKbnvuxQXGurbwPqzsu", + "created": 1506609734, + "currency": "gbp", + "livemode": false, + "originating_transaction": null, + "refunded": true, + "refunds": { + "object": "list", + "data": [ + { + "id": "fr_1MBoV6KbnvuxQXGucP0PaPPO", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284508, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoU0KbnvuxQXGu2wCCz4Bb", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284441, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoRzKbnvuxQXGuvKkBKkSR", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284315, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoPOKbnvuxQXGueOBnke22", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284154, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoOGKbnvuxQXGu6EPQI2Zp", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284084, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoMUKbnvuxQXGu8Y0Peaoy", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670283974, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MAgZBKbnvuxQXGuLTUrgGeq", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670015681, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1JAu9EKbnvuxQXGuRdZYkxVW", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1625738880, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": { + "order_id": "6735" + } + }, + { + "id": "fr_1HZK0UKbnvuxQXGuS428gH0W", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1602005482, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_D0s7fGBKB40Twy", + "object": "fee_refund", + "amount": 138, + "balance_transaction": "txn_1CaqNg2eZvKYlo2C75cA3Euk", + "created": 1528486576, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + } + ], + "has_more": false, + "url": "/v1/application_fees/fee_1B73DOKbnvuxQXGuhY8Aw0TN/refunds" + }, + "source": { + "fee_type": "charge_application_fee", + "resource": { + "charge": "ch_1B73DOKbnvuxQXGurbwPqzsu", + "type": "charge" + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json new file mode 100644 index 000000000000..47eacf5fad9f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json @@ -0,0 +1,17 @@ +{ + "object": "list", + "url": "/v1/application_fees/fr_1MtJRpKbnvuxQXGuM6Ww0D24/refunds", + "has_more": false, + "data": [ + { + "id": "fr_1MtJRpKbnvuxQXGuM6Ww0D24", + "object": "fee_refund", + "amount": 100, + "balance_transaction": null, + "created": 1680651573, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json new file mode 100644 index 000000000000..bad75c218964 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/customers/cus_9s6XI9OFIdpjIg/bank_accounts", + "has_more": false, + "data": [ + { + "id": "ba_1MvoIJ2eZvKYlo2CO9f0MabO", + "object": "bank_account", + "account_holder_name": "Jane Austen", + "account_holder_type": "company", + "account_type": null, + "bank_name": "STRIPE TEST BANK", + "country": "US", + "currency": "usd", + "customer": "cus_9s6XI9OFIdpjIg", + "fingerprint": "1JWtPxqbdX5Gamtc", + "last4": "6789", + "metadata": {}, + "routing_number": "110000000", + "status": "new" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json new file mode 100644 index 000000000000..182aa0e44e9f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json @@ -0,0 +1,43 @@ +{ + "object": "list", + "url": "/v1/customers", + "has_more": false, + "data": [ + { + "id": "cus_NffrFeUfNV2Hib", + "object": "customer", + "address": null, + "balance": 0, + "created": 1680893993, + "currency": null, + "default_source": null, + "delinquent": false, + "description": null, + "discount": null, + "email": "jennyrosen@example.com", + "invoice_prefix": "0759376C", + "invoice_settings": { + "custom_fields": null, + "default_payment_method": null, + "footer": null, + "rendering_options": null + }, + "livemode": false, + "metadata": {}, + "name": "Jenny Rosen", + "next_invoice_sequence": 1, + "phone": null, + "preferred_locales": [], + "shipping": null, + "sources": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/customers/cus_NffrFeUfNV2Hib/sources" + }, + "tax_exempt": "none", + "test_clock": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json new file mode 100644 index 000000000000..7f62598ea161 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json @@ -0,0 +1,58 @@ +{ + "object": "list", + "data": [ + { + "id": "evt_1OEiWvEcXtiJtvvhLaQOew6V", + "object": "event", + "api_version": "2020-08-27", + "created": 1700529213, + "data": { + "object": { + "object": "balance", + "available": [ + { + "amount": 518686, + "currency": "usd", + "source_types": { + "card": 518686 + } + } + ], + "connect_reserved": [ + { + "amount": 0, + "currency": "usd" + } + ], + "issuing": { + "available": [ + { + "amount": 150000, + "currency": "usd" + } + ] + }, + "livemode": false, + "pending": [ + { + "amount": 0, + "currency": "usd", + "source_types": { + "card": 0 + } + } + ] + } + }, + "livemode": false, + "pending_webhooks": 0, + "request": { + "id": null, + "idempotency_key": null + }, + "type": "balance.available" + } + ], + "has_more": false, + "url": "/v1/events" +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json new file mode 100644 index 000000000000..c26bc36461cd --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json @@ -0,0 +1,34 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts", + "has_more": false, + "data": [ + { + "id": "card_1NAz2x2eZvKYlo2C75wJ1YUs", + "object": "card", + "address_city": null, + "address_country": null, + "address_line1": null, + "address_line1_check": null, + "address_line2": null, + "address_state": null, + "address_zip": null, + "address_zip_check": null, + "brand": "Visa", + "country": "US", + "cvc_check": "pass", + "dynamic_last4": null, + "exp_month": 8, + "exp_year": 2024, + "fingerprint": "Xt5EWLLDS7FJjR1c", + "funding": "credit", + "last4": "4242", + "metadata": {}, + "name": null, + "redaction": null, + "tokenization_method": null, + "wallet": null, + "account": "acct_1032D82eZvKYlo2C" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json new file mode 100644 index 000000000000..a8704270de2f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts", + "has_more": false, + "data": [ + { + "id": "ba_1NB1IV2eZvKYlo2CByiLrMWv", + "object": "bank_account", + "account_holder_name": "Jane Austen", + "account_holder_type": "company", + "account_type": null, + "bank_name": "STRIPE TEST BANK", + "country": "US", + "currency": "usd", + "fingerprint": "1JWtPxqbdX5Gamtc", + "last4": "6789", + "metadata": {}, + "routing_number": "110000000", + "status": "new", + "account": "acct_1032D82eZvKYlo2C" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json new file mode 100644 index 000000000000..cdf281ee2cd7 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json @@ -0,0 +1,142 @@ +{ + "object": "list", + "url": "/v1/issuing/authorizations", + "has_more": false, + "data": [ + { + "id": "iauth_1JVXl82eZvKYlo2CPIiWlzrn", + "object": "issuing.authorization", + "amount": 382, + "amount_details": { + "atm_fee": null + }, + "approved": false, + "authorization_method": "online", + "balance_transactions": [], + "card": { + "id": "ic_1JDmgz2eZvKYlo2CRXlTsXj6", + "object": "issuing.card", + "brand": "Visa", + "cancellation_reason": null, + "cardholder": { + "id": "ich_1JDmfb2eZvKYlo2CwHUgaAxU", + "object": "issuing.cardholder", + "billing": { + "address": { + "city": "San Francisco", + "country": "US", + "line1": "123 Main Street", + "line2": null, + "postal_code": "94111", + "state": "CA" + } + }, + "company": null, + "created": 1626425119, + "email": "jenny.rosen@example.com", + "individual": null, + "livemode": false, + "metadata": {}, + "name": "Jenny Rosen", + "phone_number": "+18008675309", + "redaction": null, + "requirements": { + "disabled_reason": null, + "past_due": [] + }, + "spending_controls": { + "allowed_categories": [], + "blocked_categories": [], + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "individual" + }, + "created": 1626425206, + "currency": "usd", + "exp_month": 6, + "exp_year": 2024, + "last4": "8693", + "livemode": false, + "metadata": {}, + "redaction": null, + "replaced_by": null, + "replacement_for": null, + "replacement_reason": null, + "shipping": null, + "spending_controls": { + "allowed_categories": null, + "blocked_categories": null, + "spending_limits": [ + { + "amount": 50000, + "categories": [], + "interval": "daily" + } + ], + "spending_limits_currency": "usd" + }, + "status": "active", + "type": "virtual", + "wallets": { + "apple_pay": { + "eligible": true, + "ineligible_reason": null + }, + "google_pay": { + "eligible": true, + "ineligible_reason": null + }, + "primary_account_identifier": null + } + }, + "cardholder": "ich_1JDmfb2eZvKYlo2CwHUgaAxU", + "created": 1630657706, + "currency": "usd", + "livemode": false, + "merchant_amount": 382, + "merchant_currency": "usd", + "merchant_data": { + "category": "computer_software_stores", + "category_code": "5734", + "city": "SAN FRANCISCO", + "country": "US", + "name": "STRIPE", + "network_id": "1234567890", + "postal_code": "94103", + "state": "CA" + }, + "metadata": { + "order_id": "6735" + }, + "network_data": null, + "pending_request": null, + "redaction": null, + "request_history": [ + { + "amount": 382, + "amount_details": { + "atm_fee": null + }, + "approved": false, + "created": 1630657706, + "currency": "usd", + "merchant_amount": 382, + "merchant_currency": "usd", + "reason": "verification_failed", + "reason_message": null + } + ], + "status": "closed", + "transactions": [], + "verification_data": { + "address_line1_check": "not_provided", + "address_postal_code_check": "not_provided", + "cvc_check": "mismatch", + "expiry_check": "match" + }, + "wallet": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json new file mode 100644 index 000000000000..1d5027df5457 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json @@ -0,0 +1,82 @@ +{ + "object": "list", + "url": "/v1/issuing/cards", + "has_more": false, + "data": [ + { + "id": "ic_1MvSieLkdIwHu7ixn6uuO0Xu", + "object": "issuing.card", + "brand": "Visa", + "cancellation_reason": null, + "cardholder": { + "id": "ich_1MsKAB2eZvKYlo2C3eZ2BdvK", + "object": "issuing.cardholder", + "billing": { + "address": { + "city": "Anytown", + "country": "US", + "line1": "123 Main Street", + "line2": null, + "postal_code": "12345", + "state": "CA" + } + }, + "company": null, + "created": 1680415995, + "email": null, + "individual": null, + "livemode": false, + "metadata": {}, + "name": "John Doe", + "phone_number": null, + "requirements": { + "disabled_reason": "requirements.past_due", + "past_due": [ + "individual.card_issuing.user_terms_acceptance.ip", + "individual.card_issuing.user_terms_acceptance.date", + "individual.first_name", + "individual.last_name" + ] + }, + "spending_controls": { + "allowed_categories": [], + "blocked_categories": [], + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "individual" + }, + "created": 1681163868, + "currency": "usd", + "exp_month": 8, + "exp_year": 2024, + "last4": "4242", + "livemode": false, + "metadata": {}, + "replaced_by": null, + "replacement_for": null, + "replacement_reason": null, + "shipping": null, + "spending_controls": { + "allowed_categories": null, + "blocked_categories": null, + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "virtual", + "wallets": { + "apple_pay": { + "eligible": false, + "ineligible_reason": "missing_cardholder_contact" + }, + "google_pay": { + "eligible": false, + "ineligible_reason": "missing_cardholder_contact" + }, + "primary_account_identifier": null + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json new file mode 100644 index 000000000000..bbd790f318eb --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json @@ -0,0 +1,38 @@ +{ + "object": "list", + "url": "/v1/issuing/transactions", + "has_more": false, + "data": [ + { + "id": "ipi_1MzFN1K8F4fqH0lBmFq8CjbU", + "object": "issuing.transaction", + "amount": -100, + "amount_details": { + "atm_fee": null + }, + "authorization": "iauth_1MzFMzK8F4fqH0lBc9VdaZUp", + "balance_transaction": "txn_1MzFN1K8F4fqH0lBQPtqUmJN", + "card": "ic_1MzFMxK8F4fqH0lBjIUITRYi", + "cardholder": "ich_1MzFMxK8F4fqH0lBXnFW0ROG", + "created": 1682065867, + "currency": "usd", + "dispute": null, + "livemode": false, + "merchant_amount": -100, + "merchant_currency": "usd", + "merchant_data": { + "category": "computer_software_stores", + "category_code": "5734", + "city": "SAN FRANCISCO", + "country": "US", + "name": "WWWW.BROWSEBUG.BIZ", + "network_id": "1234567890", + "postal_code": "94103", + "state": "CA" + }, + "metadata": {}, + "type": "capture", + "wallet": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json new file mode 100644 index 000000000000..59ced3939e62 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json @@ -0,0 +1,52 @@ +{ + "object": "list", + "url": "/v1/payment_methods", + "has_more": false, + "data": [ + { + "id": "pm_1NO6mA2eZvKYlo2CEydeHsKT", + "object": "payment_method", + "billing_details": { + "address": { + "city": null, + "country": null, + "line1": null, + "line2": null, + "postal_code": null, + "state": null + }, + "email": null, + "name": null, + "phone": null + }, + "card": { + "brand": "visa", + "checks": { + "address_line1_check": null, + "address_postal_code_check": null, + "cvc_check": "unchecked" + }, + "country": "US", + "exp_month": 8, + "exp_year": 2024, + "fingerprint": "Xt5EWLLDS7FJjR1c", + "funding": "credit", + "generated_from": null, + "last4": "4242", + "networks": { + "available": ["visa"], + "preferred": null + }, + "three_d_secure_usage": { + "supported": true + }, + "wallet": null + }, + "created": 1687991030, + "customer": "cus_9s6XKzkNRiz8i3", + "livemode": false, + "metadata": {}, + "type": "card" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json new file mode 100644 index 000000000000..f7ede1e42817 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json @@ -0,0 +1,65 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1G9HZLIEn49ers/persons", + "has_more": false, + "data": [ + { + "id": "person_1MqjB62eZvKYlo2CaeEJzK13", + "person": "person_1MqjB62eZvKYlo2CaeEJzK13", + "object": "person", + "account": "acct_1G9HZLIEn49ers", + "created": 1680035496, + "dob": { + "day": null, + "month": null, + "year": null + }, + "first_name": "Brock", + "future_requirements": { + "alternatives": [], + "currently_due": [], + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "id_number_provided": false, + "last_name": "Purdy", + "metadata": {}, + "relationship": { + "director": false, + "executive": true, + "owner": false, + "percent_ownership": null, + "representative": false, + "title": null + }, + "requirements": { + "alternatives": [], + "currently_due": [], + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "ssn_last_4_provided": false, + "verification": { + "additional_document": { + "back": null, + "details": null, + "details_code": null, + "front": null + }, + "details": null, + "details_code": null, + "document": { + "back": null, + "details": null, + "details_code": null, + "front": null + }, + "status": "verified" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json new file mode 100644 index 000000000000..8da264f5b475 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json @@ -0,0 +1,16 @@ +{ + "object": "list", + "url": "/v1/radar/early_fraud_warnings", + "has_more": false, + "data": [ + { + "id": "issfr_1NnrwHBw2dPENLoi9lnhV3RQ", + "object": "radar.early_fraud_warning", + "actionable": true, + "charge": "ch_1234", + "created": 123456789, + "fraud_type": "misc", + "livemode": false + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json new file mode 100644 index 000000000000..af20ee7480d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json @@ -0,0 +1,32 @@ +{ + "object": "list", + "url": "/v1/refunds", + "has_more": false, + "data": [ + { + "id": "re_1Nispe2eZvKYlo2Cd31jOCgZ", + "object": "refund", + "amount": 1000, + "balance_transaction": "txn_1Nispe2eZvKYlo2CYezqFhEx", + "charge": "ch_1NirD82eZvKYlo2CIvbtLWuY", + "created": 1692942318, + "currency": "usd", + "destination_details": { + "card": { + "reference": "123456789012", + "reference_status": "available", + "reference_type": "acquirer_reference_number", + "type": "refund" + }, + "type": "card" + }, + "metadata": {}, + "payment_intent": "pi_1GszsK2eZvKYlo2CfhZyoZLp", + "reason": null, + "receipt_number": null, + "source_transfer_reversal": null, + "status": "succeeded", + "transfer_reversal": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json new file mode 100644 index 000000000000..0e41d57d3bb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/reviews", + "has_more": false, + "data": [ + { + "id": "prv_1NVyFt2eZvKYlo2CjubqF1xm", + "object": "review", + "billing_zip": null, + "charge": null, + "closed_reason": null, + "created": 1689864901, + "ip_address": null, + "ip_address_location": null, + "livemode": false, + "open": true, + "opened_reason": "rule", + "payment_intent": "pi_3NVy8c2eZvKYlo2C055h7pkd", + "reason": "rule", + "session": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 2f2a1c0acd1e..5a2f6e06a719 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -12,13 +12,14 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils import AirbyteTracedException from source_stripe import SourceStripe logger = logging.getLogger("airbyte") _ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) _ANY_CONFIG = {} +_NO_STATE = StateBuilder().build() class CatalogBuilder: @@ -51,11 +52,11 @@ def _a_valid_config(): @patch.object(source_stripe.source, "stripe") def test_source_check_connection_ok(mocked_client, config): - assert SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=config) == (True, None) + assert SourceStripe(_ANY_CATALOG, _ANY_CONFIG, _NO_STATE).check_connection(logger, config=config) == (True, None) def test_streams_are_unique(config): - stream_names = [s.name for s in SourceStripe(_ANY_CATALOG, _ANY_CONFIG).streams(config=config)] + stream_names = [s.name for s in SourceStripe(_ANY_CATALOG, _ANY_CONFIG, _NO_STATE).streams(config=config)] assert len(stream_names) == len(set(stream_names)) == 46 @@ -72,7 +73,7 @@ def test_streams_are_unique(config): def test_config_validation(mocked_client, input_config, expected_error_msg): context = pytest.raises(AirbyteTracedException, match=expected_error_msg) if expected_error_msg else does_not_raise() with context: - SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=input_config) + SourceStripe(_ANY_CATALOG, _ANY_CONFIG, _NO_STATE).check_connection(logger, config=input_config) @pytest.mark.parametrize( @@ -85,7 +86,7 @@ def test_config_validation(mocked_client, input_config, expected_error_msg): @patch.object(source_stripe.source.stripe, "Account") def test_given_stripe_error_when_check_connection_then_connection_not_available(mocked_client, exception): mocked_client.retrieve.side_effect = exception - is_available, _ = SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=_a_valid_config()) + is_available, _ = SourceStripe(_ANY_CATALOG, _ANY_CONFIG, _NO_STATE).check_connection(logger, config=_a_valid_config()) assert not is_available @@ -93,9 +94,12 @@ def test_when_streams_return_full_refresh_as_concurrent(): streams = SourceStripe( CatalogBuilder().with_stream("bank_accounts", SyncMode.full_refresh).with_stream("customers", SyncMode.incremental).build(), _a_valid_config(), + _NO_STATE, ).streams(_a_valid_config()) - assert len(list(filter(lambda stream: isinstance(stream, StreamFacade), streams))) == 1 + # bank_accounts (as it is defined as full_refresh) + # balance_transactions, events, files, file_links and shipping_rates (as it is always concurrent now) + assert len(list(filter(lambda stream: isinstance(stream, StreamFacade), streams))) == 6 @pytest.mark.parametrize( @@ -114,7 +118,7 @@ def test_call_budget_creation(mocker, input_config, default_call_limit): policy_mock = mocker.patch("source_stripe.source.MovingWindowCallRatePolicy") matcher_mock = mocker.patch("source_stripe.source.HttpRequestMatcher") - source = SourceStripe(catalog=None, config=input_config) + source = SourceStripe(catalog=None, config=input_config, state=_NO_STATE) source.get_api_call_budget(input_config) @@ -137,7 +141,7 @@ def test_call_budget_passed_to_every_stream(mocker): """Test that each stream has call_budget passed and creates a proper session""" prod_config = {"account_id": 1, "client_secret": "secret"} - source = SourceStripe(catalog=None, config=prod_config) + source = SourceStripe(catalog=None, config=prod_config, state=_NO_STATE) get_api_call_budget_mock = mocker.patch.object(source, "get_api_call_budget") streams = source.streams(prod_config) @@ -146,7 +150,8 @@ def test_call_budget_passed_to_every_stream(mocker): get_api_call_budget_mock.assert_called_once() for stream in streams: - assert isinstance(stream, HttpStream) + if isinstance(stream, StreamFacade): + stream = stream._legacy_stream session = stream.request_session() assert isinstance(session, (CachedLimiterSession, LimiterSession)) assert session._api_budget == get_api_call_budget_mock.return_value diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index 5f942b152157..3e6ee9b1be05 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -55,12 +55,8 @@ def test_request_headers(stream_by_name): } ], }, - "https://api.stripe.com/v1/customers/cus_HezytZRkaQJC8W/sources?object=bank_account&starting_after=cs_2": { + "https://api.stripe.com/v1/customers/cus_HezytZRkaQJC8W/bank_accounts?starting_after=cs_2": { "data": [ - { - "id": "cs_3", - "object": "card", - }, { "id": "cs_4", "object": "bank_account", @@ -68,8 +64,7 @@ def test_request_headers(stream_by_name): ], "has_more": False, "object": "list", - "total_count": 4, - "url": "/v1/customers/cus_HezytZRkaQJC8W/sources", + "url": "/v1/customers/cus_HezytZRkaQJC8W/bank_accounts", }, }, "bank_accounts", @@ -339,7 +334,7 @@ def test_created_cursor_incremental_stream( for url, response in requests_mock_map.items(): requests_mock.get(url, response) - slices = list(stream.stream_slices(sync_mode, stream_state=state)) + slices = list(stream.stream_slices(sync_mode=sync_mode, stream_state=state)) assert slices == expected_slices records = read_from_stream(stream, sync_mode, state) assert records == expected_records @@ -651,7 +646,7 @@ def test_cursorless_incremental_substream(requests_mock, stream_by_name, sync_mo "has_more": False, }, ) - requests_mock.get("/v1/customers/1/sources", json={"has_more": False, "data": [{"id": 2, "object": "bank_account"}]}) + requests_mock.get("/v1/customers/1/bank_accounts", json={"has_more": False, "data": [{"id": 2, "object": "bank_account"}]}) requests_mock.get( "/v1/events", json={ diff --git a/airbyte-integrations/connectors/source-survey-sparrow/main.py b/airbyte-integrations/connectors/source-survey-sparrow/main.py index 31056359d13b..5c4977542eaf 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/main.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_survey_sparrow import SourceSurveySparrow +from source_survey_sparrow.run import run if __name__ == "__main__": - source = SourceSurveySparrow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml b/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml index 3a127ebc7732..82c2fc3cb21a 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml +++ b/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml @@ -8,6 +8,10 @@ data: icon: surveysparrow.svg license: MIT name: SurveySparrow + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-survey-sparrow registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-survey-sparrow/setup.py b/airbyte-integrations/connectors/source-survey-sparrow/setup.py index da1ca261a46e..dfa0601abbde 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/setup.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-survey-sparrow=source_survey_sparrow.run:run", + ], + }, name="source_survey_sparrow", description="Source implementation for Survey Sparrow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py b/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py new file mode 100644 index 000000000000..af39a841b3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_survey_sparrow import SourceSurveySparrow + + +def run(): + source = SourceSurveySparrow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-surveycto/main.py b/airbyte-integrations/connectors/source-surveycto/main.py index 4f26fe81785f..9f282dbc2ecd 100644 --- a/airbyte-integrations/connectors/source-surveycto/main.py +++ b/airbyte-integrations/connectors/source-surveycto/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_surveycto import SourceSurveycto +from source_surveycto.run import run if __name__ == "__main__": - source = SourceSurveycto() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-surveycto/metadata.yaml b/airbyte-integrations/connectors/source-surveycto/metadata.yaml index d8bc3a5a52f9..119a22f9ed49 100644 --- a/airbyte-integrations/connectors/source-surveycto/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveycto/metadata.yaml @@ -8,6 +8,10 @@ data: icon: surveycto.svg license: MIT name: SurveyCTO + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-surveycto registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-surveycto/setup.py b/airbyte-integrations/connectors/source-surveycto/setup.py index be5f78fbdd14..92e965e0651b 100644 --- a/airbyte-integrations/connectors/source-surveycto/setup.py +++ b/airbyte-integrations/connectors/source-surveycto/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-surveycto=source_surveycto.run:run", + ], + }, name="source_surveycto", description="Source implementation for Surveycto.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py new file mode 100644 index 000000000000..927d17c9eb1f --- /dev/null +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_surveycto import SourceSurveycto + + +def run(): + source = SourceSurveycto() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-surveymonkey/README.md b/airbyte-integrations/connectors/source-surveymonkey/README.md index 0ef47e0bcc4c..37e81d149ab0 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/README.md +++ b/airbyte-integrations/connectors/source-surveymonkey/README.md @@ -1,118 +1,55 @@ -# Surveymonkey Source +# Surveymonkey source connector + This is the repository for the Surveymonkey source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/surveymonkey). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/surveymonkey). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/surveymonkey) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_surveymonkey/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/surveymonkey) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_surveymonkey/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source surveymonkey test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-surveymonkey spec +poetry run source-surveymonkey check --config secrets/config.json +poetry run source-surveymonkey discover --config secrets/config.json +poetry run source-surveymonkey read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-surveymonkey build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-surveymonkey:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-surveymonkey:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-surveymonkey:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-surveymonkey:dev . -# Running the spec command against your patched connector -docker run airbyte/source-surveymonkey:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-surveymonkey:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-surveymonkey:dev disco docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-surveymonkey:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-surveymonkey test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-surveymonkey test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/surveymonkey.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/surveymonkey.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-surveymonkey/main.py b/airbyte-integrations/connectors/source-surveymonkey/main.py index 7fd72e7dd38f..bf4f900ad377 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/main.py +++ b/airbyte-integrations/connectors/source-surveymonkey/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_surveymonkey import SourceSurveymonkey +from source_surveymonkey.run import run if __name__ == "__main__": - source = SourceSurveymonkey() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml index 4ef8ecb31f1a..e3c8f55b0746 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: badc5925-0485-42be-8caa-b34096cb71b5 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 dockerRepository: airbyte/source-surveymonkey documentationUrl: https://docs.airbyte.com/integrations/sources/surveymonkey githubIssueLabel: source-surveymonkey icon: surveymonkey.svg license: MIT name: SurveyMonkey + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-surveymonkey registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-surveymonkey/poetry.lock b/airbyte-integrations/connectors/source-surveymonkey/poetry.lock new file mode 100644 index 000000000000..144064338eb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/poetry.lock @@ -0,0 +1,1301 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.40" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.40.tar.gz", hash = "sha256:b1bb1edecb8c27b4b1c8a313e391ac18c04259e10732867021b4c4d781f92554"}, + {file = "airbyte_cdk-0.51.40-py3-none-any.whl", hash = "sha256:4e94ca42e535fc51c2c8bda872d977cf8534fbbe49f61506d0caf414ae02cfb0"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.0" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.0" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.0-py3-none-any.whl", hash = "sha256:e6a31fcd237504198a678d02c0040a8798f281c39203da61a5abce67842c5360"}, + {file = "airbyte_protocol_models-0.4.0.tar.gz", hash = "sha256:518736015c29ac60b6b8964a1b0d9b52e40020bcbd89e2545cc781f0b37d0f2b"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "vcrpy" +version = "4.1.1" +description = "Automatically mock your HTTP interactions to simplify and speed up testing" +optional = false +python-versions = ">=3.5" +files = [ + {file = "vcrpy-4.1.1-py2.py3-none-any.whl", hash = "sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162"}, + {file = "vcrpy-4.1.1.tar.gz", hash = "sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599"}, +] + +[package.dependencies] +PyYAML = "*" +six = ">=1.5" +wrapt = "*" +yarl = {version = "*", markers = "python_version >= \"3.6\""} + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "82d1e2e98e3dcdedee09c2f9112d8df65dff18dc4f79705659c2246282ddec0e" diff --git a/airbyte-integrations/connectors/source-surveymonkey/pyproject.toml b/airbyte-integrations/connectors/source-surveymonkey/pyproject.toml new file mode 100644 index 000000000000..f78bab2e5e2b --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.2.4" +name = "source-surveymonkey" +description = "Source implementation for Surveymonkey." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/surveymonkey" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_surveymonkey" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.51.40" +vcrpy = "==4.1.1" +urllib3 = "==1.26.18" + +[tool.poetry.scripts] +source-surveymonkey = "source_surveymonkey.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-surveymonkey/requirements.txt b/airbyte-integrations/connectors/source-surveymonkey/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py deleted file mode 100644 index f70f3e894857..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy==4.1.1", "urllib3<2.0"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] - -setup( - name="source_surveymonkey", - description="Source implementation for Surveymonkey.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py new file mode 100644 index 000000000000..f3cbc028402b --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_surveymonkey import SourceSurveymonkey + + +def run(): + source = SourceSurveymonkey() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/main.py b/airbyte-integrations/connectors/source-talkdesk-explore/main.py index 33dcea10e90a..745a3f67e001 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/main.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_talkdesk_explore import SourceTalkdeskExplore +from source_talkdesk_explore.run import run if __name__ == "__main__": - source = SourceTalkdeskExplore() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml b/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml index c9c5cbe7dcf0..915acd44f1a4 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml +++ b/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml @@ -8,11 +8,15 @@ data: icon: talkdesk-explore.svg license: MIT name: TalkDesk Explore - registries: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-talkdesk-explore + registries: # Removed from registries due to LEGACY STATE cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/talkdesk-explore tags: diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py index 2694e175a333..1ec623cfb4d5 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-talkdesk-explore=source_talkdesk_explore.run:run", + ], + }, name="source_talkdesk_explore", description="Source implementation for Talkdesk Explore API.", author="Airbyte", author_email="alexandre.martins@saltpay.co", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py b/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py new file mode 100644 index 000000000000..442b84e5a278 --- /dev/null +++ b/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_talkdesk_explore import SourceTalkdeskExplore + + +def run(): + source = SourceTalkdeskExplore() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tempo/main.py b/airbyte-integrations/connectors/source-tempo/main.py index e26fc47cd806..d8f0e748fb2f 100644 --- a/airbyte-integrations/connectors/source-tempo/main.py +++ b/airbyte-integrations/connectors/source-tempo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tempo import SourceTempo +from source_tempo.run import run if __name__ == "__main__": - source = SourceTempo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tempo/metadata.yaml b/airbyte-integrations/connectors/source-tempo/metadata.yaml index d29e9d18b11c..b40b265b8b73 100644 --- a/airbyte-integrations/connectors/source-tempo/metadata.yaml +++ b/airbyte-integrations/connectors/source-tempo/metadata.yaml @@ -15,6 +15,10 @@ data: icon: tempo.svg license: MIT name: Tempo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tempo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-tempo/setup.py b/airbyte-integrations/connectors/source-tempo/setup.py index 16051a8dfa6b..37cf227b0f5b 100644 --- a/airbyte-integrations/connectors/source-tempo/setup.py +++ b/airbyte-integrations/connectors/source-tempo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tempo=source_tempo.run:run", + ], + }, name="source_tempo", description="Source implementation for Tempo.", author="Thomas van Latum", author_email="thomas@gcompany.nl", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/run.py b/airbyte-integrations/connectors/source-tempo/source_tempo/run.py new file mode 100644 index 000000000000..8883fc3f0a24 --- /dev/null +++ b/airbyte-integrations/connectors/source-tempo/source_tempo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tempo import SourceTempo + + +def run(): + source = SourceTempo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml b/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml deleted file mode 100644 index 49f39cead0ef..000000000000 --- a/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-teradata:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-teradata/build.gradle b/airbyte-integrations/connectors/source-teradata/build.gradle index 1e3ec17dcab5..d8f462e0ef10 100644 --- a/airbyte-integrations/connectors/source-teradata/build.gradle +++ b/airbyte-integrations/connectors/source-teradata/build.gradle @@ -1,26 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.teradata.TeradataSource' } @@ -29,5 +16,5 @@ dependencies { implementation 'com.teradata.jdbc:terajdbc:20.00.00.06' - testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.testcontainers:jdbc:1.19.4' } diff --git a/airbyte-integrations/connectors/source-teradata/metadata.yaml b/airbyte-integrations/connectors/source-teradata/metadata.yaml index fc063a148c33..108450bfe625 100644 --- a/airbyte-integrations/connectors/source-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/source-teradata/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: aa8ba6fd-4875-d94e-fc8d-4e1e09aa2503 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-teradata githubIssueLabel: source-teradata icon: teradata.svg diff --git a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java index 71d2ee229091..d9410b75cdd1 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java +++ b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java @@ -110,13 +110,15 @@ public JdbcDatabase createDatabase(JsonNode sourceConfig) throws SQLException { JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters(customProperties, sslConnectionProperties); final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = MoreMaps.merge(customProperties, sslConnectionProperties); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - MoreMaps.merge(customProperties, sslConnectionProperties)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); diff --git a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java index 98622f2cc227..7fc7f6eaafaf 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java +++ b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java @@ -4,8 +4,4 @@ package io.airbyte.integrations.source.teradata.envclient.dto; -public record DeleteEnvironmentRequest( - - String name - -) {} +public record DeleteEnvironmentRequest(String name) {} diff --git a/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java index 7980c0ea25ed..4f04fcaf1e3d 100644 --- a/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java @@ -32,8 +32,10 @@ import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; +@Disabled @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TeradataSourceAcceptanceTest extends SourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java index 9cad6f8b3faa..e755e01af383 100644 --- a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -16,41 +16,32 @@ import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Disabled @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class TeradataJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class TeradataJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final Logger LOGGER = LoggerFactory.getLogger(TeradataJdbcSourceAcceptanceTest.class); + private static JsonNode staticConfig; - private JsonNode staticConfig; - - static { - COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; - - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s ST_Geometry) NO PRIMARY INDEX;"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('POLYGON((1 1, 1 3, 6 3, 6 0, 1 1))');"; - - COL_TIMESTAMP = "tmstmp"; - INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, tmstmp) VALUES ('%s', '%s')"; - COL_TIMESTAMP_TYPE = "TIMESTAMP(0)"; + public static void cleanUpBeforeStarting() { + try { + cleanupEnvironment(); + } catch (final Exception ignored) {} } @BeforeAll - public void initEnvironment() throws ExecutionException, InterruptedException { + static void initEnvironment() throws ExecutionException, InterruptedException { staticConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); - TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); + cleanUpBeforeStarting(); + final TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); var request = new CreateEnvironmentRequest( staticConfig.get("env_name").asText(), staticConfig.get("env_region").asText(), @@ -62,25 +53,25 @@ public void initEnvironment() throws ExecutionException, InterruptedException { } catch (ClassNotFoundException e) { throw new RuntimeException(e); } + + COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; + + CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s ST_Geometry) NO PRIMARY INDEX;"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('POLYGON((1 1, 1 3, 6 3, 6 0, 1 1))');"; + + COL_TIMESTAMP = "tmstmp"; + INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, tmstmp) VALUES ('%s', '%s')"; + COL_TIMESTAMP_TYPE = "TIMESTAMP(0)"; } @AfterAll - public void cleanupEnvironment() throws ExecutionException, InterruptedException { - TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); - var request = new DeleteEnvironmentRequest(staticConfig.get("env_name").asText()); + public static void cleanupEnvironment() throws ExecutionException, InterruptedException { + final TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); + final var request = new DeleteEnvironmentRequest(staticConfig.get("env_name").asText()); teradataHttpClient.deleteEnvironment(request, staticConfig.get("env_token").asText()).get(); } - @BeforeEach - public void setup() throws Exception { - executeStatements(List.of( - statement -> statement.executeUpdate("CREATE DATABASE \"database_name\" AS PERMANENT = 120e6, SPOOL = 120e6;")), - staticConfig.get("host").asText(), staticConfig.get("username").asText(), staticConfig.get("password").asText()); - super.setup(); - } - - @AfterEach - public void tearDown() { + static void deleteDatabase() { executeStatements(List.of( statement -> statement.executeUpdate("DELETE DATABASE \"database_name\";"), statement -> statement.executeUpdate("DROP DATABASE \"database_name\";")), staticConfig.get("host").asText(), @@ -88,34 +79,31 @@ public void tearDown() { } @Override - public AbstractJdbcSource getSource() { - return new TeradataSource(); + protected TeradataTestDatabase createTestDatabase() { + executeStatements(List.of( + statement -> statement.executeUpdate("CREATE DATABASE \"database_name\" AS PERMANENT = 120e6, SPOOL = 120e6;")), + staticConfig.get("host").asText(), staticConfig.get("username").asText(), staticConfig.get("password").asText()); + return new TeradataTestDatabase(source().toDatabaseConfig(Jsons.clone(staticConfig))).initialized(); } @Override public boolean supportsSchemas() { - // TODO check if your db supports it and update method accordingly return false; } @Override - public JsonNode getConfig() { + public JsonNode config() { return Jsons.clone(staticConfig); } @Override - public String getDriverClass() { - return TeradataSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected TeradataSource source() { return new TeradataSource(); } @Override public String getFullyQualifiedTableName(String tableName) { - return "database_name." + tableName; + return staticConfig.get(JdbcUtils.DATABASE_KEY).asText() + "." + tableName; } private static void executeStatements(List consumers, String host, String username, String password) { diff --git a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java new file mode 100644 index 000000000000..09d9eca44028 --- /dev/null +++ b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.teradata; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.TERADATA; +import static io.airbyte.integrations.source.teradata.TeradataJdbcSourceAcceptanceTest.deleteDatabase; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class TeradataTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + private final String databaseName; + + protected TeradataTestDatabase(final JsonNode teradataConfig) { + super(new NonContainer(teradataConfig.get(JdbcUtils.USERNAME_KEY).asText(), + teradataConfig.has(JdbcUtils.PASSWORD_KEY) ? teradataConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + teradataConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), TERADATA.getDriverClassName(), "")); + this.username = teradataConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = teradataConfig.has(JdbcUtils.PASSWORD_KEY) ? teradataConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = teradataConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + this.databaseName = teradataConfig.get(JdbcUtils.SCHEMA_KEY).asText(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return TERADATA; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + deleteDatabase(); + } + + static public class TeradataDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected TeradataDbConfigBuilder(final TeradataTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-the-guardian-api/main.py b/airbyte-integrations/connectors/source-the-guardian-api/main.py index 96a88ceac666..50182e0c2da0 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/main.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_the_guardian_api import SourceTheGuardianApi +from source_the_guardian_api.run import run if __name__ == "__main__": - source = SourceTheGuardianApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml b/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml index f074b8f5153b..f6a56606c61e 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: theguardian.svg license: MIT name: The Guardian API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-the-guardian-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-the-guardian-api/setup.py b/airbyte-integrations/connectors/source-the-guardian-api/setup.py index 1fb11d947926..9627db8db884 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/setup.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-the-guardian-api=source_the_guardian_api.run:run", + ], + }, name="source_the_guardian_api", description="Source implementation for The Guardian Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py b/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py new file mode 100644 index 000000000000..eafbdfd35301 --- /dev/null +++ b/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_the_guardian_api import SourceTheGuardianApi + + +def run(): + source = SourceTheGuardianApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index 1368609a3dea..8fe8b49931d2 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -1,40 +1,22 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' + cdkVersionRequired = '0.20.4' features = ['db-sources'] useLocalCdk = false } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileTestJava { - options.compilerArgs.remove("-Werror") - } - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.source.tidb.TiDBSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { + implementation 'mysql:mysql-connector-java:8.0.33' - //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - implementation 'mysql:mysql-connector-java:8.0.22' - - // Add testcontainers and use GenericContainer for TiDB - testImplementation libs.testcontainers.tidb - - testImplementation 'org.apache.commons:commons-lang3:3.11' + testFixturesApi 'org.testcontainers:tidb:1.19.4' - integrationTestJavaImplementation libs.testcontainers.tidb + testImplementation 'org.hamcrest:hamcrest-all:1.3' } diff --git a/airbyte-integrations/connectors/source-tidb/metadata.yaml b/airbyte-integrations/connectors/source-tidb/metadata.yaml index 47a33d680ae8..1d277882a8e0 100644 --- a/airbyte-integrations/connectors/source-tidb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tidb/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: database connectorType: source definitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae - dockerImageTag: 0.2.5 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-tidb githubIssueLabel: source-tidb icon: tidb.svg diff --git a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java index 642e3a199d0e..86cb85232832 100644 --- a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java @@ -5,16 +5,10 @@ package io.airbyte.integrations.source.tidb; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.cdk.db.Database; -import io.airbyte.cdk.db.factory.DSLContextFactory; -import io.airbyte.cdk.db.factory.DatabaseDriver; -import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; @@ -25,56 +19,31 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; +import org.junit.jupiter.api.Disabled; +import org.testcontainers.tidb.TiDBContainer; +@Disabled public class TiDBSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "public.starships"; - protected GenericContainer container; - protected JsonNode config; + protected TiDBContainer container = TiDBTestDatabase.container(); + protected TiDBTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv testEnv) throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.USERNAME_KEY, "root") - .put(JdbcUtils.DATABASE_KEY, "test") - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - "", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } + testdb = new TiDBTestDatabase(container) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.with("DROP TABLE id_and_name;") + .with("DROP TABLE starships;"); } @Override @@ -89,7 +58,7 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder().build(); } @Override @@ -100,7 +69,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), @@ -109,7 +78,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME2), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME2), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java index 2d18b7bf0a35..fff46004d7e1 100755 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java @@ -5,50 +5,13 @@ package io.airbyte.integrations.source.tidb; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.mysql.cj.MysqlType; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import org.junit.jupiter.api.*; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; +import org.testcontainers.tidb.TiDBContainer; -class TiDBJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class TiDBJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - protected static GenericContainer container; - protected static String USER = "root"; - protected static String DATABASE = "test"; - - @BeforeEach - public void setup() throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "127.0.0.1") - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, USER) - .put(JdbcUtils.DATABASE_KEY, DATABASE) - // .put(JdbcUtils.SSL_KEY, true) - .build()); - - super.setup(); - } - - @AfterEach - void tearDownTiDB() throws Exception { - container.close(); - container.stop(); - super.tearDown(); - } - - @Override - public AbstractJdbcSource getSource() { - return new TiDBSource(); - } + private final TiDBContainer container = TiDBTestDatabase.container(); @Override public boolean supportsSchemas() { @@ -56,23 +19,18 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public JsonNode config() { + return Jsons.clone(testdb.testConfigBuilder().build()); } @Override - public String getDriverClass() { - return TiDBSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected TiDBSource source() { return new TiDBSource(); } - @AfterAll - static void cleanUp() { - container.close(); + @Override + protected TiDBTestDatabase createTestDatabase() { + return new TiDBTestDatabase(container).initialized(); } } diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java deleted file mode 100644 index b47f82d73672..000000000000 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.tidb; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; - -public class TiDBSourceTests { - - private JsonNode config; - private GenericContainer container; - - @Test - public void testSettingTimezones() throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "127.0.0.1") - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, "root") - .put(JdbcUtils.DATABASE_KEY, "test") - .build()); - - final AirbyteConnectionStatus check = new TiDBSource().check(config); - - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus()); - container.close(); - } - -} diff --git a/airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java b/airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java new file mode 100644 index 000000000000..17b238db5d29 --- /dev/null +++ b/airbyte-integrations/connectors/source-tidb/src/testFixtures/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.tidb; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.testutils.ContainerFactory; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.tidb.TiDBContainer; +import org.testcontainers.utility.DockerImageName; + +public class TiDBTestDatabase extends + TestDatabase { + + protected TiDBTestDatabase(final TiDBContainer container) { + super(container); + } + + @Override + public String withNamespace(String name) { + return name; + } + + @Override + public String getDatabaseName() { + return getContainer().getDatabaseName(); + } + + @Override + public String getUserName() { + return getContainer().getUsername(); + } + + @Override + public String getPassword() { + return getContainer().getPassword(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.MYSQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.MYSQL; + } + + @Override + public TiDBConfigBuilder configBuilder() { + return new TiDBConfigBuilder(this); + } + + static public class TiDBConfigBuilder extends ConfigBuilder { + + protected TiDBConfigBuilder(final TiDBTestDatabase testdb) { + super(testdb); + } + + } + + static public TiDBContainer container() { + var factory = new ContainerFactory() { + + @Override + protected TiDBContainer createNewContainer(DockerImageName dockerImageName) { + return new TiDBContainer(dockerImageName).withExposedPorts(4000); + } + + }; + return factory.exclusive("pingcap/tidb:nightly"); + } + +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/README.md b/airbyte-integrations/connectors/source-tiktok-marketing/README.md index f787ae62f943..fadf0bc2de06 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/README.md +++ b/airbyte-integrations/connectors/source-tiktok-marketing/README.md @@ -1,118 +1,55 @@ -# Tiktok Marketing Source +# Tiktok-Marketing source connector -This is the repository for the Tiktok Marketing source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/tiktok-marketing). + +This is the repository for the Tiktok-Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/tiktok-marketing). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tiktok-marketing) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tiktok_marketing/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/tiktok-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tiktok_marketing/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source tiktok-marketing test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-tiktok-marketing spec +poetry run source-tiktok-marketing check --config secrets/config.json +poetry run source-tiktok-marketing discover --config secrets/config.json +poetry run source-tiktok-marketing read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-tiktok-marketing build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-tiktok-marketing:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-tiktok-marketing:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-tiktok-marketing:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-tiktok-marketing:dev . -# Running the spec command against your patched connector -docker run airbyte/source-tiktok-marketing:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-tiktok-marketing:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tiktok-marketing:dev d docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tiktok-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-tiktok-marketing test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tiktok-marketing test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tiktok-marketing.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/tiktok-marketing.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/main.py b/airbyte-integrations/connectors/source-tiktok-marketing/main.py index d20d9e211aa8..b523ea1b0fdd 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/main.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tiktok_marketing import SourceTiktokMarketing +from source_tiktok_marketing.run import run if __name__ == "__main__": - source = SourceTiktokMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml index e18fca8923b2..e0eae9d0c7e5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml @@ -11,13 +11,17 @@ data: connectorSubtype: api connectorType: source definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 - dockerImageTag: 3.9.2 + dockerImageTag: 3.9.3 dockerRepository: airbyte/source-tiktok-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing githubIssueLabel: source-tiktok-marketing icon: tiktok.svg license: MIT name: TikTok Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tiktok-marketing registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock b/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock new file mode 100644 index 000000000000..925f4338357a --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock @@ -0,0 +1,1093 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.52.10" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.52.10.tar.gz", hash = "sha256:0daee950fe0d4453e6ceea2633090fc1d2144224e6f170b3c6cb4c6392811b47"}, + {file = "airbyte_cdk-0.52.10-py3-none-any.whl", hash = "sha256:366fd7bbbba317223edc1571d22b91c6f5bcff4ba65b3131e42f9b37e29932f4"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "timeout-decorator" +version = "0.5.0" +description = "Timeout decorator" +optional = false +python-versions = "*" +files = [ + {file = "timeout-decorator-0.5.0.tar.gz", hash = "sha256:6a2f2f58db1c5b24a2cc79de6345760377ad8bdc13813f5265f6c3e63d16b3d7"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "8cc7e89fec6f2188200b8327a969602f26fcd02a5929b5e5504c954eae99f3fc" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml b/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml new file mode 100644 index 000000000000..909a3989301d --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "3.9.3" +name = "source-tiktok-marketing" +description = "Source implementation for Tiktok Marketing." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/tiktok-marketing" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_tiktok_marketing" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.52.10" + +[tool.poetry.scripts] +source-tiktok-marketing = "source_tiktok_marketing.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "==1.9.3" +timeout-decorator = "==0.5.0" +pytest-mock = "^3.6.1" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt b/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py deleted file mode 100644 index 3a5d282355a8..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk"] - -TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock==1.9.3", "timeout-decorator==0.5.0"] - -setup( - name="source_tiktok_marketing", - description="Source implementation for Tiktok Marketing.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={"tests": TEST_REQUIREMENTS}, -) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py new file mode 100644 index 000000000000..341638b66529 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tiktok_marketing import SourceTiktokMarketing + + +def run(): + source = SourceTiktokMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-timely/main.py b/airbyte-integrations/connectors/source-timely/main.py index 674590cf1d99..1007e1945bc0 100644 --- a/airbyte-integrations/connectors/source-timely/main.py +++ b/airbyte-integrations/connectors/source-timely/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_timely import SourceTimely +from source_timely.run import run if __name__ == "__main__": - source = SourceTimely() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-timely/metadata.yaml b/airbyte-integrations/connectors/source-timely/metadata.yaml index f9b1f4fbbcfb..0b371b513fd3 100644 --- a/airbyte-integrations/connectors/source-timely/metadata.yaml +++ b/airbyte-integrations/connectors/source-timely/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.timelyapp.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-timely registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-timely/setup.py b/airbyte-integrations/connectors/source-timely/setup.py index 6dc55b4722ef..9004e784b613 100644 --- a/airbyte-integrations/connectors/source-timely/setup.py +++ b/airbyte-integrations/connectors/source-timely/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-timely=source_timely.run:run", + ], + }, name="source_timely", description="Source implementation for Timely.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-timely/source_timely/run.py b/airbyte-integrations/connectors/source-timely/source_timely/run.py new file mode 100644 index 000000000000..6ead8b999eb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-timely/source_timely/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_timely import SourceTimely + + +def run(): + source = SourceTimely() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tmdb/main.py b/airbyte-integrations/connectors/source-tmdb/main.py index 8fb1b3a13296..de1e36196011 100644 --- a/airbyte-integrations/connectors/source-tmdb/main.py +++ b/airbyte-integrations/connectors/source-tmdb/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tmdb import SourceTmdb +from source_tmdb.run import run if __name__ == "__main__": - source = SourceTmdb() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tmdb/metadata.yaml b/airbyte-integrations/connectors/source-tmdb/metadata.yaml index 7979d306d363..eb0da1758fad 100644 --- a/airbyte-integrations/connectors/source-tmdb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tmdb/metadata.yaml @@ -8,6 +8,10 @@ data: icon: tmdb.svg license: MIT name: TMDb + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tmdb registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-tmdb/setup.py b/airbyte-integrations/connectors/source-tmdb/setup.py index 35310f227836..d1b20699bcf3 100644 --- a/airbyte-integrations/connectors/source-tmdb/setup.py +++ b/airbyte-integrations/connectors/source-tmdb/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tmdb=source_tmdb.run:run", + ], + }, name="source_tmdb", description="Source implementation for Tmdb.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py b/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py new file mode 100644 index 000000000000..100edabab0b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tmdb import SourceTmdb + + +def run(): + source = SourceTmdb() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-todoist/.dockerignore b/airbyte-integrations/connectors/source-todoist/.dockerignore deleted file mode 100644 index d853b63208a0..000000000000 --- a/airbyte-integrations/connectors/source-todoist/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_todoist -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-todoist/Dockerfile b/airbyte-integrations/connectors/source-todoist/Dockerfile deleted file mode 100644 index c30309061741..000000000000 --- a/airbyte-integrations/connectors/source-todoist/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_todoist ./source_todoist - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-todoist diff --git a/airbyte-integrations/connectors/source-todoist/README.md b/airbyte-integrations/connectors/source-todoist/README.md index bd25a8590776..b68dcdc0a9bf 100644 --- a/airbyte-integrations/connectors/source-todoist/README.md +++ b/airbyte-integrations/connectors/source-todoist/README.md @@ -1,37 +1,13 @@ # Todoist Source -This is the repository for the Todoist source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/todoist). +This is the repository for the Todoist configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/todoist). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/todoist) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/todoist) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_todoist/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -39,29 +15,69 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source todoist test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name=source-todoist build +airbyte-ci connectors --name source-todoist build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-todoist:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -An image will be built with the tag `airbyte/source-todoist:dev`. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. -**Via `docker build`:** +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-todoist:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: ```bash docker build -t airbyte/source-todoist:dev . -``` +# Running the spec command against your patched connector +docker run airbyte/source-todoist:dev spec #### Run Then run any of the connector commands as follows: @@ -71,16 +87,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-todoist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - ## Testing -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=source-todoist test -``` -### Customizing acceptance Tests +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-todoist test +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml index b14d04a96904..25c0dc8dc1d4 100644 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml @@ -1,7 +1,6 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-todoist:dev -test_strictness_level: low acceptance_tests: spec: tests: @@ -22,7 +21,7 @@ acceptance_tests: empty_streams: [] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: - # path: "integration_tests/expected_records.txt" + # path: "integration_tests/expected_records.jsonl" # extra_fields: no # exact_order: no # extra_records: yes @@ -32,7 +31,8 @@ acceptance_tests: # tests: # - config_path: "secrets/config.json" # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-todoist/icon.svg b/airbyte-integrations/connectors/source-todoist/icon.svg index dbf417c99c8d..c4df54834efe 100644 --- a/airbyte-integrations/connectors/source-todoist/icon.svg +++ b/airbyte-integrations/connectors/source-todoist/icon.svg @@ -1,6 +1,14 @@ - - - - - - + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py index 1100c1c58cf5..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json index 09f16c3ccf2a..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json @@ -1,37 +1,5 @@ { - "tasks": { - "id": false, - "project_id": 10, - "section_id": -10, - "content": 10, - "description": true, - "is_completed": "not so true", - "labels": [true, false], - "parent_id": -50, - "order": true, - "priority": "10", - "due": true, - "url": 50, - "comment_count": "10", - "created_at": { "when": true }, - "creator_id": -1, - "assignee_id": 20, - "assigner_id": 50 - }, - "projects": { - "id": { - "number": "2203306141" - }, - "name": 50, - "comment_count": false, - "order": "1", - "color": 100, - "is_shared": [false], - "is_favorite": "world", - "parent_id": 100, - "is_inbox_project": [true], - "is_team_inbox": "false", - "view_style": ["list"], - "url": ["https://todoist.com/showProject?id=2203306141"] + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 275f28ac14f5..8fed59fd4d9e 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -1,34 +1,22 @@ { "streams": [ { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "tasks", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "projects", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index 37cbe64ae615..244ec5755c74 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "token": "INVALID TOKEN" + "token": "INVALID_API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index 55e640b7e5c3..6dbfca1a0354 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "token": "VALID TOKEN" + "token": "API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/main.py b/airbyte-integrations/connectors/source-todoist/main.py index 21ae17e05de7..159e7dcdc036 100644 --- a/airbyte-integrations/connectors/source-todoist/main.py +++ b/airbyte-integrations/connectors/source-todoist/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_todoist import SourceTodoist +from source_todoist.run import run if __name__ == "__main__": - source = SourceTodoist() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index f32560daec66..a7e8abc8a55d 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -1,24 +1,34 @@ data: + allowedHosts: + hosts: + - api.todoist.com/rest/v2 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-todoist + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source - definitionId: 7d272065-c316-4c04-a433-cd4ee143f83e - dockerImageTag: 0.1.0 + definitionId: 1a3d38e4-dc6b-4154-b56b-582f9e978ecd + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-todoist githubIssueLabel: source-todoist icon: todoist.svg license: MIT name: Todoist - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-12-10 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/todoist tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-todoist/requirements.txt b/airbyte-integrations/connectors/source-todoist/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-todoist/requirements.txt +++ b/airbyte-integrations/connectors/source-todoist/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index cf672aeb4515..601df38fd542 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -6,24 +6,40 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", + "pytest~=6.2", + "pytest-mock~=3.6.1", ] setup( + entry_points={ + "console_scripts": [ + "source-todoist=source_todoist.run:run", + ], + }, name="source_todoist", description="Source implementation for Todoist.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py index f04f17ce4a6f..62d0c357fa93 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml new file mode 100644 index 000000000000..8460493fe200 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml @@ -0,0 +1,62 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.todoist.com/rest/v2" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + $parameters: + path: "/tasks" + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "/projects" + +streams: + - "#/definitions/tasks_stream" + - "#/definitions/projects_stream" + +check: + type: CheckStream + stream_names: + - "tasks" + - "projects" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/source-todolist + connection_specification: + title: Source Todolist Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + type: string + description: API authorization bearer token for authenticating the API + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/run.py b/airbyte-integrations/connectors/source-todoist/source_todoist/run.py new file mode 100644 index 000000000000..b20e5a2d580f --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_todoist import SourceTodoist + + +def run(): + source = SourceTodoist() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json new file mode 100644 index 000000000000..c9bce00c9315 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json @@ -0,0 +1,82 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "assignee_id": { + "type": ["null", "string"] + }, + "assigner_id": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "due": { + "anyOf": [ + { + "type": ["null", "object"] + }, + { + "properties": { + "date": { + "type": ["null", "string"] + }, + "is_recurring": { + "type": ["null", "boolean"] + }, + "lang": { + "type": ["null", "string"] + }, + "string": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "is_completed": { + "type": ["null", "boolean"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "string"] + }, + "section_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json index cf22f8da8a4b..f5d926e1d087 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, "type": "object", "properties": { @@ -51,6 +51,9 @@ "id": { "type": ["null", "string"] }, + "duration": { + "type": ["null", "string"] + }, "is_completed": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py index 517c10ae7383..b363f5fb2623 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py @@ -2,66 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class TodoistStream(HttpStream): - """ - Stream for Todoist REST API : https://developer.todoist.com/rest/v2/#overview - """ - @property - def url_base(self) -> str: - return "https://api.todoist.com/rest/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.name.title().lower() - - -class Tasks(TodoistStream): - - primary_key = "id" - - -class Projects(TodoistStream): - - primary_key = "id" - - -# Source -class SourceTodoist(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config["token"] - authenticator = TokenAuthenticator(token=token) - task_stream = Tasks(authenticator) - task_records = task_stream.read_records(sync_mode="full_refresh") - next(task_records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - token = config["token"] - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - return [Tasks(authenticator=auth), Projects(authenticator=auth)] +# Declarative Source +class SourceTodoist(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml deleted file mode 100644 index 19e7f9dc2e42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.io/integrations/sources/todoist -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Todoist Spec - type: object - required: - - token - properties: - token: - type: string - description: >- - Your API Token. See here. The token is - case sensitive. - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py deleted file mode 100644 index d4dbafa573be..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_todoist.source import SourceTodoist - - -def test_check_connection(mocker): - source = SourceTodoist() - fake_info_record = {"collection": "is_mocked"} - with patch("source_todoist.source.Tasks.read_records", MagicMock(return_value=iter([fake_info_record]))): - logger_mock = MagicMock() - config_mock = {"token": "test"} - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceTodoist() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py deleted file mode 100644 index af1ed528da42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_todoist.source import TodoistStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TodoistStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = TodoistStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = TodoistStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = TodoistStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = TodoistStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-toggl/main.py b/airbyte-integrations/connectors/source-toggl/main.py index bd1b4defb2de..a52cb2ff9370 100644 --- a/airbyte-integrations/connectors/source-toggl/main.py +++ b/airbyte-integrations/connectors/source-toggl/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_toggl import SourceToggl +from source_toggl.run import run if __name__ == "__main__": - source = SourceToggl() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-toggl/metadata.yaml b/airbyte-integrations/connectors/source-toggl/metadata.yaml index cac927f82274..4edc40421588 100644 --- a/airbyte-integrations/connectors/source-toggl/metadata.yaml +++ b/airbyte-integrations/connectors/source-toggl/metadata.yaml @@ -8,6 +8,10 @@ data: icon: toggl.svg license: MIT name: Toggl + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-toggl registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-toggl/setup.py b/airbyte-integrations/connectors/source-toggl/setup.py index bb481cee5328..cdd1baad6ba7 100644 --- a/airbyte-integrations/connectors/source-toggl/setup.py +++ b/airbyte-integrations/connectors/source-toggl/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-toggl=source_toggl.run:run", + ], + }, name="source_toggl", description="Source implementation for Toggl.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-toggl/source_toggl/run.py b/airbyte-integrations/connectors/source-toggl/source_toggl/run.py new file mode 100644 index 000000000000..fd06c41abef3 --- /dev/null +++ b/airbyte-integrations/connectors/source-toggl/source_toggl/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_toggl import SourceToggl + + +def run(): + source = SourceToggl() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tplcentral/main.py b/airbyte-integrations/connectors/source-tplcentral/main.py index e8e5f30138f4..c5e7b8a95ec4 100644 --- a/airbyte-integrations/connectors/source-tplcentral/main.py +++ b/airbyte-integrations/connectors/source-tplcentral/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tplcentral import SourceTplcentral +from source_tplcentral.run import run if __name__ == "__main__": - source = SourceTplcentral() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tplcentral/metadata.yaml b/airbyte-integrations/connectors/source-tplcentral/metadata.yaml index e47c7d06d92e..cb14f697ec39 100644 --- a/airbyte-integrations/connectors/source-tplcentral/metadata.yaml +++ b/airbyte-integrations/connectors/source-tplcentral/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-tplcentral license: MIT name: TPLcentral + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tplcentral registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-tplcentral/setup.py b/airbyte-integrations/connectors/source-tplcentral/setup.py index 602d12056cb3..b3d6343e0d1c 100644 --- a/airbyte-integrations/connectors/source-tplcentral/setup.py +++ b/airbyte-integrations/connectors/source-tplcentral/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tplcentral=source_tplcentral.run:run", + ], + }, name="source_tplcentral", description="Source implementation for Tplcentral.", author="Labanoras Tech", author_email="jv@labanoras.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py new file mode 100644 index 000000000000..443a194ed6c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tplcentral import SourceTplcentral + + +def run(): + source = SourceTplcentral() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-trello/main.py b/airbyte-integrations/connectors/source-trello/main.py index c0657c0065fc..7b0b502fdb1e 100644 --- a/airbyte-integrations/connectors/source-trello/main.py +++ b/airbyte-integrations/connectors/source-trello/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_trello import SourceTrello +from source_trello.run import run if __name__ == "__main__": - source = SourceTrello() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-trello/metadata.yaml b/airbyte-integrations/connectors/source-trello/metadata.yaml index 51d7da393c99..2f7a89819189 100644 --- a/airbyte-integrations/connectors/source-trello/metadata.yaml +++ b/airbyte-integrations/connectors/source-trello/metadata.yaml @@ -5,6 +5,10 @@ data: allowedHosts: hosts: - api.trello.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-trello registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-trello/setup.py b/airbyte-integrations/connectors/source-trello/setup.py index 38d12ba64785..85fd6920b1af 100644 --- a/airbyte-integrations/connectors/source-trello/setup.py +++ b/airbyte-integrations/connectors/source-trello/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-trello=source_trello.run:run", + ], + }, name="source_trello", description="Source implementation for Trello.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-trello/source_trello/run.py b/airbyte-integrations/connectors/source-trello/source_trello/run.py new file mode 100644 index 000000000000..628d22faf0db --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/source_trello/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_trello import SourceTrello + + +def run(): + source = SourceTrello() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-trustpilot/main.py b/airbyte-integrations/connectors/source-trustpilot/main.py index f0cd7b4265e5..27e409441e33 100644 --- a/airbyte-integrations/connectors/source-trustpilot/main.py +++ b/airbyte-integrations/connectors/source-trustpilot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_trustpilot import SourceTrustpilot +from source_trustpilot.run import run if __name__ == "__main__": - source = SourceTrustpilot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-trustpilot/metadata.yaml b/airbyte-integrations/connectors/source-trustpilot/metadata.yaml index 96e957b54a11..7ac069f60bdd 100644 --- a/airbyte-integrations/connectors/source-trustpilot/metadata.yaml +++ b/airbyte-integrations/connectors/source-trustpilot/metadata.yaml @@ -8,6 +8,10 @@ data: icon: trustpilot.svg license: MIT name: TrustPilot + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-trustpilot registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-trustpilot/setup.py b/airbyte-integrations/connectors/source-trustpilot/setup.py index 4b8001807ec1..7c61c708d96e 100644 --- a/airbyte-integrations/connectors/source-trustpilot/setup.py +++ b/airbyte-integrations/connectors/source-trustpilot/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-trustpilot=source_trustpilot.run:run", + ], + }, name="source_trustpilot", description="Source implementation for Trustpilot.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py new file mode 100644 index 000000000000..40c488dbf9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_trustpilot import SourceTrustpilot + + +def run(): + source = SourceTrustpilot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py index 3a530f427a6f..be6abd8ef07c 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tvmaze_schedule import SourceTvmazeSchedule +from source_tvmaze_schedule.run import run if __name__ == "__main__": - source = SourceTvmazeSchedule() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml index 0da8f2802ab0..a51eed571207 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml @@ -8,6 +8,10 @@ data: icon: tvmazeschedule.svg license: MIT name: TVMaze Schedule + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tvmaze-schedule registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py index 77c97057564e..800761e02c23 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tvmaze-schedule=source_tvmaze_schedule.run:run", + ], + }, name="source_tvmaze_schedule", description="Source implementation for Tvmaze Schedule.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py new file mode 100644 index 000000000000..24cdb98bf699 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tvmaze_schedule import SourceTvmazeSchedule + + +def run(): + source = SourceTvmazeSchedule() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/main.py b/airbyte-integrations/connectors/source-twilio-taskrouter/main.py index 792fa7db3284..94a3a9659fe9 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/main.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twilio_taskrouter import SourceTwilioTaskrouter +from source_twilio_taskrouter.run import run if __name__ == "__main__": - source = SourceTwilioTaskrouter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml b/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml index 69d1bbb070b4..e8a42439382d 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml @@ -8,6 +8,10 @@ data: icon: twilio.svg license: MIT name: Twilio Taskrouter + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-twilio-taskrouter registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py index be86feef816f..2d7ea97c9833 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-twilio-taskrouter=source_twilio_taskrouter.run:run", + ], + }, name="source_twilio_taskrouter", description="Source implementation for Twilio Taskrouter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py b/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py new file mode 100644 index 000000000000..6c677af9dd2c --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twilio_taskrouter import SourceTwilioTaskrouter + + +def run(): + source = SourceTwilioTaskrouter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twilio/README.md b/airbyte-integrations/connectors/source-twilio/README.md index 95b155fb069d..21f3011ac732 100644 --- a/airbyte-integrations/connectors/source-twilio/README.md +++ b/airbyte-integrations/connectors/source-twilio/README.md @@ -1,117 +1,55 @@ -# Twilio Source +# Twilio source connector + This is the repository for the Twilio source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/twilio). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/twilio). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/twilio) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_twilio/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/twilio) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_twilio/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source twilio test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-twilio spec +poetry run source-twilio check --config secrets/config.json +poetry run source-twilio discover --config secrets/config.json +poetry run source-twilio read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: - -```bash -airbyte-ci connectors --name source-twilio build +### Running unit tests +To run unit tests locally, from the connector directory run: ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-twilio:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +poetry run pytest unit_tests ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-twilio:latest +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-twilio build +``` -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code +An image will be available on your host with the tag `airbyte/source-twilio:dev`. -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. -2. Build your image: -```bash -docker build -t airbyte/source-twilio:dev . -# Running the spec command against your patched connector -docker run airbyte/source-twilio:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-twilio:dev spec @@ -120,28 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twilio:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-twilio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-twilio test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-twilio test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/twilio.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/twilio.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-twilio/main.py b/airbyte-integrations/connectors/source-twilio/main.py index 8f91bc04c56f..0999d1e67f26 100644 --- a/airbyte-integrations/connectors/source-twilio/main.py +++ b/airbyte-integrations/connectors/source-twilio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twilio import SourceTwilio +from source_twilio.run import run if __name__ == "__main__": - source = SourceTwilio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index db8a6e53432d..0037ed37d93e 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -13,13 +13,17 @@ data: connectorSubtype: api connectorType: source definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 - dockerImageTag: 0.10.1 + dockerImageTag: 0.10.2 dockerRepository: airbyte/source-twilio documentationUrl: https://docs.airbyte.com/integrations/sources/twilio githubIssueLabel: source-twilio icon: twilio.svg license: MIT name: Twilio + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-twilio registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-twilio/poetry.lock b/airbyte-integrations/connectors/source-twilio/poetry.lock new file mode 100644 index 000000000000..92d0ef9ca49d --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/poetry.lock @@ -0,0 +1,1048 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.55.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.55.0.tar.gz", hash = "sha256:b75ffcc9c94453b41c49e25c12404e5741c238ac2d42949420101496aa4c143e"}, + {file = "airbyte_cdk-0.55.0-py3-none-any.whl", hash = "sha256:f9b7ea2484c7f62ad3557dbf29cdec8f7783e146513f29810f7fbacc1aeeb303"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "fcc0ecc10e4b4573a02085f518f90ca99bf5794d301d0c73e1a94258cffad925" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml new file mode 100644 index 000000000000..94c5b71102ce --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.10.2" +name = "source-twilio" +description = "Source implementation for Twilio." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/twilio" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_twilio" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.55.0" +pendulum = "==2.1.2" +requests = "==2.31.0" + +[tool.poetry.scripts] +source-twilio = "source_twilio.run:run" + +[tool.poetry.group.dev.dependencies] +pytest-mock = "^3.12.0" +freezegun = "^1.4.0" +requests-mock = "^1.9.3" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-twilio/requirements.txt b/airbyte-integrations/connectors/source-twilio/requirements.txt deleted file mode 100644 index ecf975e2fa63..000000000000 --- a/airbyte-integrations/connectors/source-twilio/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-twilio/setup.py b/airbyte-integrations/connectors/source-twilio/setup.py deleted file mode 100644 index 66a2cb8a7ee0..000000000000 --- a/airbyte-integrations/connectors/source-twilio/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "pendulum", - "requests", -] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock", "freezegun"] - -setup( - name="source_twilio", - description="Source implementation for Twilio.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/run.py b/airbyte-integrations/connectors/source-twilio/source_twilio/run.py new file mode 100644 index 000000000000..bb95086d7eba --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twilio import SourceTwilio + + +def run(): + source = SourceTwilio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twitter/main.py b/airbyte-integrations/connectors/source-twitter/main.py index 3e6dae60da34..f50219e6455c 100644 --- a/airbyte-integrations/connectors/source-twitter/main.py +++ b/airbyte-integrations/connectors/source-twitter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twitter import SourceTwitter +from source_twitter.run import run if __name__ == "__main__": - source = SourceTwitter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twitter/metadata.yaml b/airbyte-integrations/connectors/source-twitter/metadata.yaml index b80e85f83dbf..8e4b662909de 100644 --- a/airbyte-integrations/connectors/source-twitter/metadata.yaml +++ b/airbyte-integrations/connectors/source-twitter/metadata.yaml @@ -15,6 +15,10 @@ data: icon: twitter.svg license: MIT name: Twitter + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-twitter registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-twitter/setup.py b/airbyte-integrations/connectors/source-twitter/setup.py index bcc921fcba46..e85c1d404a2a 100644 --- a/airbyte-integrations/connectors/source-twitter/setup.py +++ b/airbyte-integrations/connectors/source-twitter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-twitter=source_twitter.run:run", + ], + }, name="source_twitter", description="Source implementation for Twitter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-twitter/source_twitter/run.py b/airbyte-integrations/connectors/source-twitter/source_twitter/run.py new file mode 100644 index 000000000000..4c18e4636da4 --- /dev/null +++ b/airbyte-integrations/connectors/source-twitter/source_twitter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twitter import SourceTwitter + + +def run(): + source = SourceTwitter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tyntec-sms/main.py b/airbyte-integrations/connectors/source-tyntec-sms/main.py index 37e104247846..142de37101e6 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/main.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tyntec_sms import SourceTyntecSms +from source_tyntec_sms.run import run if __name__ == "__main__": - source = SourceTyntecSms() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml b/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml index e2b450284998..3a5278d572b4 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml +++ b/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml @@ -8,6 +8,10 @@ data: icon: tyntec.svg license: MIT name: Tyntec SMS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-tyntec-sms registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-tyntec-sms/setup.py b/airbyte-integrations/connectors/source-tyntec-sms/setup.py index 1a15c6223694..832b4139ef2d 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/setup.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tyntec-sms=source_tyntec_sms.run:run", + ], + }, name="source_tyntec_sms", description="Source implementation for Tyntec Sms.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py b/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py new file mode 100644 index 000000000000..cd08f8c0c7ee --- /dev/null +++ b/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tyntec_sms import SourceTyntecSms + + +def run(): + source = SourceTyntecSms() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-typeform/.coveragerc b/airbyte-integrations/connectors/source-typeform/.coveragerc new file mode 100644 index 000000000000..2c98af4de877 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_typeform/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-typeform/README.md b/airbyte-integrations/connectors/source-typeform/README.md index b414ad7e136a..0157f45bbbd8 100644 --- a/airbyte-integrations/connectors/source-typeform/README.md +++ b/airbyte-integrations/connectors/source-typeform/README.md @@ -1,86 +1,55 @@ -# Typeform Source +# Typeform source connector -This is the repository for the Typeform configuration based source connector. + +This is the repository for the Typeform source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/typeform). ## Local development -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/typeform) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_typeform/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source typeform test creds` -and place them into `secrets/config.json`. - -### Locally running the connector docker image - - +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Installing the connector +From this connector directory, run: ```bash -airbyte-ci connectors --name=source-typeform build +poetry install --with dev ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-typeform:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/typeform) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_typeform/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +### Locally running the connector +``` +poetry run source-typeform spec +poetry run source-typeform check --config secrets/config.json +poetry run source-typeform discover --config secrets/config.json +poetry run source-typeform read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-typeform:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests ``` -Please use this as an example. This is not optimized. -2. Build your image: +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -docker build -t airbyte/source-typeform:dev . -# Running the spec command against your patched connector -docker run airbyte/source-typeform:dev spec +airbyte-ci connectors --name=source-typeform build ``` -#### Run + +An image will be available on your host with the tag `airbyte/source-typeform:dev`. + + +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-typeform:dev spec @@ -89,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-typeform:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-typeform:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-typeform test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-typeform test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/typeform.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/typeform.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-typeform/main.py b/airbyte-integrations/connectors/source-typeform/main.py index 332857e87cdd..126dc556ff7d 100644 --- a/airbyte-integrations/connectors/source-typeform/main.py +++ b/airbyte-integrations/connectors/source-typeform/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_typeform import SourceTypeform +from source_typeform.run import run if __name__ == "__main__": - source = SourceTypeform() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-typeform/metadata.yaml b/airbyte-integrations/connectors/source-typeform/metadata.yaml index edeb34d13425..c8c2bb6379e4 100644 --- a/airbyte-integrations/connectors/source-typeform/metadata.yaml +++ b/airbyte-integrations/connectors/source-typeform/metadata.yaml @@ -6,17 +6,21 @@ data: hosts: - api.typeform.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: e7eff203-90bf-43e5-a240-19ea3056c474 - dockerImageTag: 1.2.1 + dockerImageTag: 1.2.5 dockerRepository: airbyte/source-typeform documentationUrl: https://docs.airbyte.com/integrations/sources/typeform githubIssueLabel: source-typeform icon: typeform.svg license: MIT name: Typeform + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-typeform registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-typeform/poetry.lock b/airbyte-integrations/connectors/source-typeform/poetry.lock new file mode 100644 index 000000000000..cfbe169883b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.59.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.59.0.tar.gz", hash = "sha256:2f7bc07556cc7f42f0daf41d09be08fd22102864d087a27c8999f6f13fe67aad"}, + {file = "airbyte_cdk-0.59.0-py3-none-any.whl", hash = "sha256:94c561c053b8be3a66bfefe420812ced9237403441249408e2af5445214a6f7b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "1a1acacff3cde56bc593bf94bef4f697cff3c1d4eb0599c70cc9e8307df9cb5b" diff --git a/airbyte-integrations/connectors/source-typeform/pyproject.toml b/airbyte-integrations/connectors/source-typeform/pyproject.toml new file mode 100644 index 000000000000..ee42093d03d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "1.2.5" +name = "source-typeform" +description = "Source implementation for Typeform." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/typeform" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_typeform" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.59.0" + +[tool.poetry.scripts] +source-typeform = "source_typeform.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6.1" +pytest = "^6.2" diff --git a/airbyte-integrations/connectors/source-typeform/requirements.txt b/airbyte-integrations/connectors/source-typeform/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-typeform/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-typeform/setup.py b/airbyte-integrations/connectors/source-typeform/setup.py deleted file mode 100644 index 282003bf9077..000000000000 --- a/airbyte-integrations/connectors/source-typeform/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] - -setup( - name="source_typeform", - description="Source implementation for Typeform.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/components.py b/airbyte-integrations/connectors/source-typeform/source_typeform/components.py index 3966a856ca2c..d7e12b70396d 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/components.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/components.py @@ -20,9 +20,7 @@ class TypeformAuthenticator(DeclarativeAuthenticator): oauth2: DeclarativeSingleUseRefreshTokenOauth2Authenticator def __new__(cls, token_auth, oauth2, config, *args, **kwargs): - if config["credentials"]["access_token"]: - return token_auth - return oauth2 + return token_auth if config["credentials"]["auth_type"] == "access_token" else oauth2 @dataclass diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml index 7a36969afcb0..6fd25154e5e6 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml @@ -26,6 +26,7 @@ definitions: client_id: "{{ config['credentials']['client_id'] }}" client_secret: "{{ config['credentials']['client_secret'] }}" refresh_token: "{{ config['credentials']['refresh_token'] }}" + refresh_token_updater: {} requester: type: HttpRequester url_base: https://api.typeform.com/ @@ -124,7 +125,7 @@ definitions: pagination_strategy: type: CursorPagination cursor_value: "{{ last_records[-1]['token'] }}" - stop_condition: "{{ not response['total_items'] }}" + stop_condition: "{{ not last_records }}" page_size: 1000 partition_router: $ref: "#/definitions/form_id_partition_router" diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py new file mode 100644 index 000000000000..2ebf804b4940 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_typeform import SourceTypeform + + +def run(): + source = SourceTypeform() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py index cda8bc7a95bb..a841b1d266cf 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py @@ -6,8 +6,8 @@ def test_typeform_authenticator(): - config = {"credentials": {"access_token": "access_token", "client_id": None, "client_secret": None}} - oauth_config = {"credentials": {"access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} + config = {"credentials": {"auth_type": "access_token", "access_token": "access_token"}} + oauth_config = {"credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} class TokenProvider: def get_token(self) -> str: @@ -16,12 +16,12 @@ def get_token(self) -> str: auth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") ) assert isinstance(auth, BearerAuthenticator) oauth = TypeformAuthenticator( - token_auth=BearerAuthenticator(config=oauth_config, token_provider=TokenProvider(), parameters={}), + token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=oauth_config, oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") ) diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py index 0b906e9e1ba6..c5c3f1508e76 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py @@ -45,20 +45,3 @@ def test_stream_slices(form_ids, parent_stream_configs, expected_slices): slices = list(router.stream_slices()) assert slices == expected_slices - -@pytest.mark.parametrize("token_auth, oauth2, config, expected", [ - ( - "token_auth", - None, - {"credentials": { "auth_type": True, "access_token": True }}, - "token_auth" - ), - ( - None, - "oauth2", - {"credentials": { "auth_type": False, "access_token": False}}, - "oauth2" - ) -]) -def test_new_typeformauthenticator(token_auth, oauth2, config, expected): - assert TypeformAuthenticator(token_auth, oauth2, config) == expected \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-unleash/main.py b/airbyte-integrations/connectors/source-unleash/main.py index 9e57236c979a..72ec2c888dcb 100644 --- a/airbyte-integrations/connectors/source-unleash/main.py +++ b/airbyte-integrations/connectors/source-unleash/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_unleash import SourceUnleash +from source_unleash.run import run if __name__ == "__main__": - source = SourceUnleash() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-unleash/metadata.yaml b/airbyte-integrations/connectors/source-unleash/metadata.yaml index 84e08f649241..e3b6c2cf07a4 100644 --- a/airbyte-integrations/connectors/source-unleash/metadata.yaml +++ b/airbyte-integrations/connectors/source-unleash/metadata.yaml @@ -8,6 +8,11 @@ data: icon: unleash.svg license: MIT name: Unleash + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-unleash registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-unleash/setup.py b/airbyte-integrations/connectors/source-unleash/setup.py index 8c00746f3fd4..9822815f03ce 100644 --- a/airbyte-integrations/connectors/source-unleash/setup.py +++ b/airbyte-integrations/connectors/source-unleash/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-unleash=source_unleash.run:run", + ], + }, name="source_unleash", description="Source implementation for Unleash.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-unleash/source_unleash/run.py b/airbyte-integrations/connectors/source-unleash/source_unleash/run.py new file mode 100644 index 000000000000..0de14599b8b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-unleash/source_unleash/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_unleash import SourceUnleash + + +def run(): + source = SourceUnleash() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-us-census/Dockerfile b/airbyte-integrations/connectors/source-us-census/Dockerfile index 8ce00031e8b7..49be16577813 100644 --- a/airbyte-integrations/connectors/source-us-census/Dockerfile +++ b/airbyte-integrations/connectors/source-us-census/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-us-census diff --git a/airbyte-integrations/connectors/source-us-census/main.py b/airbyte-integrations/connectors/source-us-census/main.py index 95978af82418..c46fd28c9ce6 100644 --- a/airbyte-integrations/connectors/source-us-census/main.py +++ b/airbyte-integrations/connectors/source-us-census/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_us_census import SourceUsCensus +from source_us_census.run import run if __name__ == "__main__": - source = SourceUsCensus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-us-census/metadata.yaml b/airbyte-integrations/connectors/source-us-census/metadata.yaml index 47c5e9e77e8b..81721da1cc16 100644 --- a/airbyte-integrations/connectors/source-us-census/metadata.yaml +++ b/airbyte-integrations/connectors/source-us-census/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: c4cfaeda-c757-489a-8aba-859fb08b6970 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-us-census githubIssueLabel: source-us-census icon: uscensus.svg license: MIT name: US Census + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-us-census registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-us-census/setup.py b/airbyte-integrations/connectors/source-us-census/setup.py index 6b00267e73c8..a88a817b95a9 100644 --- a/airbyte-integrations/connectors/source-us-census/setup.py +++ b/airbyte-integrations/connectors/source-us-census/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-us-census=source_us_census.run:run", + ], + }, name="source_us_census", description="Source implementation for Us Census.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-us-census/source_us_census/run.py b/airbyte-integrations/connectors/source-us-census/source_us_census/run.py new file mode 100644 index 000000000000..20b8b7eedd0c --- /dev/null +++ b/airbyte-integrations/connectors/source-us-census/source_us_census/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_us_census import SourceUsCensus + + +def run(): + source = SourceUsCensus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json b/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json index b8c67bbff6af..1c2115ecdd30 100644 --- a/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json +++ b/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json @@ -5,7 +5,7 @@ "title": "https://api.census.gov/ Source Spec", "type": "object", "required": ["api_key", "query_path"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "query_params": { "type": "string", diff --git a/airbyte-integrations/connectors/source-vantage/main.py b/airbyte-integrations/connectors/source-vantage/main.py index fdec95252d78..eab761b3302d 100644 --- a/airbyte-integrations/connectors/source-vantage/main.py +++ b/airbyte-integrations/connectors/source-vantage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_vantage import SourceVantage +from source_vantage.run import run if __name__ == "__main__": - source = SourceVantage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-vantage/metadata.yaml b/airbyte-integrations/connectors/source-vantage/metadata.yaml index 968cef66b541..1c76b2dcf45e 100644 --- a/airbyte-integrations/connectors/source-vantage/metadata.yaml +++ b/airbyte-integrations/connectors/source-vantage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: vantage.svg license: MIT name: Vantage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-vantage registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-vantage/setup.py b/airbyte-integrations/connectors/source-vantage/setup.py index 11fe68724009..6f3c091a99c3 100644 --- a/airbyte-integrations/connectors/source-vantage/setup.py +++ b/airbyte-integrations/connectors/source-vantage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-vantage=source_vantage.run:run", + ], + }, name="source_vantage", description="Source implementation for Vantage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-vantage/source_vantage/run.py b/airbyte-integrations/connectors/source-vantage/source_vantage/run.py new file mode 100644 index 000000000000..a9c2ee3eb36a --- /dev/null +++ b/airbyte-integrations/connectors/source-vantage/source_vantage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_vantage import SourceVantage + + +def run(): + source = SourceVantage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-visma-economic/main.py b/airbyte-integrations/connectors/source-visma-economic/main.py index f5a5d1e8f883..ae896df1e48f 100644 --- a/airbyte-integrations/connectors/source-visma-economic/main.py +++ b/airbyte-integrations/connectors/source-visma-economic/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_visma_economic import SourceVismaEconomic +from source_visma_economic.run import run if __name__ == "__main__": - source = SourceVismaEconomic() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml index ec378e98d2b4..9fb3c2bce09b 100644 --- a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml +++ b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "restapi.e-conomic.com" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-visma-economic registries: oss: enabled: true @@ -21,5 +25,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/visma-economic tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-visma-economic/setup.py b/airbyte-integrations/connectors/source-visma-economic/setup.py index 9574bffc86ff..aca046a78f08 100644 --- a/airbyte-integrations/connectors/source-visma-economic/setup.py +++ b/airbyte-integrations/connectors/source-visma-economic/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-visma-economic=source_visma_economic.run:run", + ], + }, name="source_visma_economic", description="Source implementation for Visma Economic.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py new file mode 100644 index 000000000000..4c8146aa54ec --- /dev/null +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_visma_economic import SourceVismaEconomic + + +def run(): + source = SourceVismaEconomic() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-vitally/main.py b/airbyte-integrations/connectors/source-vitally/main.py index d03f28ec1f1b..1410ead03c39 100644 --- a/airbyte-integrations/connectors/source-vitally/main.py +++ b/airbyte-integrations/connectors/source-vitally/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_vitally import SourceVitally +from source_vitally.run import run if __name__ == "__main__": - source = SourceVitally() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-vitally/metadata.yaml b/airbyte-integrations/connectors/source-vitally/metadata.yaml index d233f0fcd5c2..3098c58d83eb 100644 --- a/airbyte-integrations/connectors/source-vitally/metadata.yaml +++ b/airbyte-integrations/connectors/source-vitally/metadata.yaml @@ -8,6 +8,10 @@ data: icon: vitally.svg license: MIT name: Vitally + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-vitally registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-vitally/setup.py b/airbyte-integrations/connectors/source-vitally/setup.py index f7091d2171d5..d0adc3fc2c1f 100644 --- a/airbyte-integrations/connectors/source-vitally/setup.py +++ b/airbyte-integrations/connectors/source-vitally/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-vitally=source_vitally.run:run", + ], + }, name="source_vitally", description="Source implementation for Vitally.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-vitally/source_vitally/run.py b/airbyte-integrations/connectors/source-vitally/source_vitally/run.py new file mode 100644 index 000000000000..1c6c2f841ed9 --- /dev/null +++ b/airbyte-integrations/connectors/source-vitally/source_vitally/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_vitally import SourceVitally + + +def run(): + source = SourceVitally() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-waiteraid/main.py b/airbyte-integrations/connectors/source-waiteraid/main.py index 0ba9d249cc5a..50171148de01 100644 --- a/airbyte-integrations/connectors/source-waiteraid/main.py +++ b/airbyte-integrations/connectors/source-waiteraid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_waiteraid import SourceWaiteraid +from source_waiteraid.run import run if __name__ == "__main__": - source = SourceWaiteraid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-waiteraid/metadata.yaml b/airbyte-integrations/connectors/source-waiteraid/metadata.yaml index bc77862fa906..36ae2230265c 100644 --- a/airbyte-integrations/connectors/source-waiteraid/metadata.yaml +++ b/airbyte-integrations/connectors/source-waiteraid/metadata.yaml @@ -8,6 +8,10 @@ data: icon: waiteraid.svg license: MIT name: Waiteraid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-waiteraid registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-waiteraid/setup.py b/airbyte-integrations/connectors/source-waiteraid/setup.py index fe5d754e3df5..e2dc21a8895f 100644 --- a/airbyte-integrations/connectors/source-waiteraid/setup.py +++ b/airbyte-integrations/connectors/source-waiteraid/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-waiteraid=source_waiteraid.run:run", + ], + }, name="source_waiteraid", description="Source implementation for Waiteraid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py new file mode 100644 index 000000000000..5219abc38724 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_waiteraid import SourceWaiteraid + + +def run(): + source = SourceWaiteraid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-weatherstack/main.py b/airbyte-integrations/connectors/source-weatherstack/main.py index 1924e100bb8d..83c59afceaf5 100644 --- a/airbyte-integrations/connectors/source-weatherstack/main.py +++ b/airbyte-integrations/connectors/source-weatherstack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_weatherstack import SourceWeatherstack +from source_weatherstack.run import run if __name__ == "__main__": - source = SourceWeatherstack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-weatherstack/metadata.yaml b/airbyte-integrations/connectors/source-weatherstack/metadata.yaml index 2317e207aacf..d7583620b2c9 100644 --- a/airbyte-integrations/connectors/source-weatherstack/metadata.yaml +++ b/airbyte-integrations/connectors/source-weatherstack/metadata.yaml @@ -8,6 +8,10 @@ data: icon: weatherstack.svg license: MIT name: Weatherstack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-weatherstack registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-weatherstack/setup.py b/airbyte-integrations/connectors/source-weatherstack/setup.py index 98887751b488..207fe4d3c13d 100644 --- a/airbyte-integrations/connectors/source-weatherstack/setup.py +++ b/airbyte-integrations/connectors/source-weatherstack/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-weatherstack=source_weatherstack.run:run", + ], + }, name="source_weatherstack", description="Source implementation for Weatherstack.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py b/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py new file mode 100644 index 000000000000..07f3930dd2b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_weatherstack import SourceWeatherstack + + +def run(): + source = SourceWeatherstack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-webflow/Dockerfile b/airbyte-integrations/connectors/source-webflow/Dockerfile index c5d8ef311742..487098ab5c11 100644 --- a/airbyte-integrations/connectors/source-webflow/Dockerfile +++ b/airbyte-integrations/connectors/source-webflow/Dockerfile @@ -34,5 +34,5 @@ COPY source_webflow ./source_webflow ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-webflow diff --git a/airbyte-integrations/connectors/source-webflow/README.md b/airbyte-integrations/connectors/source-webflow/README.md index e36dbbd9ead2..807cec095b47 100644 --- a/airbyte-integrations/connectors/source-webflow/README.md +++ b/airbyte-integrations/connectors/source-webflow/README.md @@ -10,7 +10,7 @@ A detailed tutorial has been written about this implementation. See: [Build a co ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +- Webflow v1 API Key #### Minimum Python version required `= 3.9.11` diff --git a/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json index c2887e81f620..3891ee24dd07 100644 --- a/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json @@ -2,8 +2,9 @@ "streams": [ { "stream": { - "name": "Blog Authors", - "json_schema": {} + "name": "Ebooks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-webflow/main.py b/airbyte-integrations/connectors/source-webflow/main.py index f35348e338c3..4d481e07151b 100644 --- a/airbyte-integrations/connectors/source-webflow/main.py +++ b/airbyte-integrations/connectors/source-webflow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_webflow import SourceWebflow +from source_webflow.run import run if __name__ == "__main__": - source = SourceWebflow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-webflow/metadata.yaml b/airbyte-integrations/connectors/source-webflow/metadata.yaml index 2f37cd559298..c88f34c1a656 100644 --- a/airbyte-integrations/connectors/source-webflow/metadata.yaml +++ b/airbyte-integrations/connectors/source-webflow/metadata.yaml @@ -2,12 +2,16 @@ data: connectorSubtype: api connectorType: source definitionId: ef580275-d9a9-48bb-af5e-db0f5855be04 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-webflow githubIssueLabel: source-webflow icon: webflow.svg license: MIT name: Webflow + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-webflow registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-webflow/setup.py b/airbyte-integrations/connectors/source-webflow/setup.py index 8c550837d124..1d604e8b4ea1 100644 --- a/airbyte-integrations/connectors/source-webflow/setup.py +++ b/airbyte-integrations/connectors/source-webflow/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-webflow=source_webflow.run:run", + ], + }, name="source_webflow", description="Source implementation for Webflow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py b/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py index 722c06f4080d..132db64bf5b8 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py @@ -23,6 +23,6 @@ def get_auth_header(self) -> Mapping[str, Any]: class WebflowTokenAuthenticator(WebflowAuthMixin, TokenAuthenticator): """ - Auth class for Personal Access Token - https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens + Authentication class information + https://docs.developers.webflow.com/reference/authorization """ diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/run.py b/airbyte-integrations/connectors/source-webflow/source_webflow/run.py new file mode 100644 index 000000000000..a71297a1d6a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_webflow import SourceWebflow + + +def run(): + source = SourceWebflow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py index 21f7d0bc63dd..f0209639ae09 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py @@ -61,10 +61,10 @@ def request_params( class CollectionSchema(WebflowStream): """ - Gets the schema of the current collection - see: https://developers.webflow.com/#get-collection-with-full-schema, and + Gets the schema of the current collection - see: https://docs.developers.webflow.com/v1.0.0/reference/get-collection, and then converts that schema to a json-schema.org-compatible schema that uses supported Airbyte types. - More info about Webflow schema: https://developers.webflow.com/#get-collection-with-full-schema + More info about Webflow schema: https://docs.developers.webflow.com/v1.0.0/reference/get-collection Airbyte data types: https://docs.airbyte.com/understanding-airbyte/supported-data-types/ """ @@ -77,8 +77,8 @@ def __init__(self, collection_id: str = None, **kwargs): def path(self, **kwargs) -> str: """ - See: https://developers.webflow.com/#list-collections - Returns a list which contains high-level information about each collection. + See: https://docs.developers.webflow.com/v1.0.0/reference/get-collection + Returns a collection with full schema by collection_id """ path = f"collections/{self.collection_id}" @@ -243,6 +243,7 @@ def get_json_schema(self) -> Mapping[str, Any]: extra_fields = { "_id": {"type": ["null", "string"]}, "_cid": {"type": ["null", "string"]}, + "_locale": {"type": ["null", "string"]}, } json_schema.update(extra_fields) @@ -283,7 +284,7 @@ def get_authenticator(config): which overloads that standard authentication to include additional headers that are required by Webflow. """ api_key = config.get("api_key", None) - accept_version = WEBFLOW_ACCEPT_VERSION + accept_version = config.get("accept_version", WEBFLOW_ACCEPT_VERSION) if not api_key: raise Exception("Config validation error: 'api_key' is a required property") diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml b/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml index 0fd66a820c1a..5d3be8b1984d 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml @@ -6,7 +6,7 @@ connectionSpecification: required: - api_key - site_id - additionalProperties: false + additionalProperties: true properties: site_id: title: Site id @@ -21,3 +21,9 @@ connectionSpecification: example: "a very long hex sequence" order: 1 airbyte_secret: true + accept_version: + title: Accept Version + type: string + description: "The version of the Webflow API to use. See https://developers.webflow.com/#versioning" + example: "1.0.0" + order: 2 diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py index 39c3e709b4ff..ea40dc0ab320 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py @@ -30,4 +30,5 @@ class WebflowToAirbyteMapping: "RichText": {"type": ["null", "string"]}, "User": {"type": ["null", "string"]}, "Video": {"type": ["null", "string"]}, + "FileRef": {"type": ["null", "object"]}, } diff --git a/airbyte-integrations/connectors/source-whisky-hunter/main.py b/airbyte-integrations/connectors/source-whisky-hunter/main.py index b9e7c8a1c682..76cf72379579 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/main.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_whisky_hunter import SourceWhiskyHunter +from source_whisky_hunter.run import run if __name__ == "__main__": - source = SourceWhiskyHunter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml b/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml index 145617426b9b..b82c59277318 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml +++ b/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml @@ -8,6 +8,10 @@ data: icon: whiskyhunter.svg license: MIT name: Whisky Hunter + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-whisky-hunter registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-whisky-hunter/setup.py b/airbyte-integrations/connectors/source-whisky-hunter/setup.py index 6ea55fe522d9..6b48b33f68c8 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/setup.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-whisky-hunter=source_whisky_hunter.run:run", + ], + }, name="source_whisky_hunter", description="Source implementation for Whisky Hunter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py new file mode 100644 index 000000000000..2af9bbe54e12 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_whisky_hunter import SourceWhiskyHunter + + +def run(): + source = SourceWhiskyHunter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py index 3f284fc2e06f..1ba1f8403240 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_wikipedia_pageviews import SourceWikipediaPageviews +from source_wikipedia_pageviews.run import run if __name__ == "__main__": - source = SourceWikipediaPageviews() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml b/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml index a553cff05092..f32b4074975a 100644 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml @@ -8,6 +8,10 @@ data: icon: wikipediapageviews.svg license: MIT name: Wikipedia Pageviews + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-wikipedia-pageviews registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py index 9716b7daa4ee..f946706e21f9 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-wikipedia-pageviews=source_wikipedia_pageviews.run:run", + ], + }, name="source_wikipedia_pageviews", description="Source implementation for Wikipedia Pageviews.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py new file mode 100755 index 000000000000..d77093ecb81b --- /dev/null +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_wikipedia_pageviews import SourceWikipediaPageviews + + +def run(): + source = SourceWikipediaPageviews() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-woocommerce/main.py b/airbyte-integrations/connectors/source-woocommerce/main.py index 606cdff539ba..1fae4df4545e 100644 --- a/airbyte-integrations/connectors/source-woocommerce/main.py +++ b/airbyte-integrations/connectors/source-woocommerce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_woocommerce import SourceWoocommerce +from source_woocommerce.run import run if __name__ == "__main__": - source = SourceWoocommerce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml index 9e34f36b4e55..d66ba0449d4a 100644 --- a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml @@ -15,6 +15,10 @@ data: icon: woocommerce.svg license: MIT name: WooCommerce + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-woocommerce registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-woocommerce/setup.py b/airbyte-integrations/connectors/source-woocommerce/setup.py index 40945a630f15..c69b2f251935 100644 --- a/airbyte-integrations/connectors/source-woocommerce/setup.py +++ b/airbyte-integrations/connectors/source-woocommerce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-woocommerce=source_woocommerce.run:run", + ], + }, name="source_woocommerce", description="Source implementation for Woocommerce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py new file mode 100644 index 000000000000..b53f1bed41bb --- /dev/null +++ b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_woocommerce import SourceWoocommerce + + +def run(): + source = SourceWoocommerce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workable/main.py b/airbyte-integrations/connectors/source-workable/main.py index 9ca2a8507540..7c63dc17331b 100644 --- a/airbyte-integrations/connectors/source-workable/main.py +++ b/airbyte-integrations/connectors/source-workable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_workable import SourceWorkable +from source_workable.run import run if __name__ == "__main__": - source = SourceWorkable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-workable/metadata.yaml b/airbyte-integrations/connectors/source-workable/metadata.yaml index 6862708b8a23..0257dc2802d5 100644 --- a/airbyte-integrations/connectors/source-workable/metadata.yaml +++ b/airbyte-integrations/connectors/source-workable/metadata.yaml @@ -8,6 +8,10 @@ data: icon: workable.svg license: MIT name: Workable + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-workable registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-workable/setup.py b/airbyte-integrations/connectors/source-workable/setup.py index 67f123878709..e35ab25501c6 100644 --- a/airbyte-integrations/connectors/source-workable/setup.py +++ b/airbyte-integrations/connectors/source-workable/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-workable=source_workable.run:run", + ], + }, name="source_workable", description="Source implementation for Workable.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-workable/source_workable/run.py b/airbyte-integrations/connectors/source-workable/source_workable/run.py new file mode 100644 index 000000000000..5b838949c4ef --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workable import SourceWorkable + + +def run(): + source = SourceWorkable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workramp/main.py b/airbyte-integrations/connectors/source-workramp/main.py index c18f064ccead..7b2a20cedcac 100644 --- a/airbyte-integrations/connectors/source-workramp/main.py +++ b/airbyte-integrations/connectors/source-workramp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_workramp import SourceWorkramp +from source_workramp.run import run if __name__ == "__main__": - source = SourceWorkramp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-workramp/metadata.yaml b/airbyte-integrations/connectors/source-workramp/metadata.yaml index 0fff3108cfc5..d704e70619cb 100644 --- a/airbyte-integrations/connectors/source-workramp/metadata.yaml +++ b/airbyte-integrations/connectors/source-workramp/metadata.yaml @@ -8,6 +8,10 @@ data: icon: workramp.svg license: MIT name: WorkRamp + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-workramp registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-workramp/setup.py b/airbyte-integrations/connectors/source-workramp/setup.py index a2b4fc952a19..eb12a1398b9c 100644 --- a/airbyte-integrations/connectors/source-workramp/setup.py +++ b/airbyte-integrations/connectors/source-workramp/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-workramp=source_workramp.run:run", + ], + }, name="source_workramp", description="Source implementation for Workramp.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-workramp/source_workramp/run.py b/airbyte-integrations/connectors/source-workramp/source_workramp/run.py new file mode 100644 index 000000000000..6b56635b93cb --- /dev/null +++ b/airbyte-integrations/connectors/source-workramp/source_workramp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workramp import SourceWorkramp + + +def run(): + source = SourceWorkramp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-wrike/main.py b/airbyte-integrations/connectors/source-wrike/main.py index 5f853be26873..4c1d0cd0f5fa 100644 --- a/airbyte-integrations/connectors/source-wrike/main.py +++ b/airbyte-integrations/connectors/source-wrike/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_wrike import SourceWrike +from source_wrike.run import run if __name__ == "__main__": - source = SourceWrike() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-wrike/metadata.yaml b/airbyte-integrations/connectors/source-wrike/metadata.yaml index 9e57391670c9..d8c02979a9aa 100644 --- a/airbyte-integrations/connectors/source-wrike/metadata.yaml +++ b/airbyte-integrations/connectors/source-wrike/metadata.yaml @@ -4,6 +4,10 @@ data: - app-us*.wrike.com - app-eu*.wrike.com - www.wrike.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-wrike registries: oss: enabled: true @@ -23,5 +27,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/wrike tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-wrike/setup.py b/airbyte-integrations/connectors/source-wrike/setup.py index a3c314147ca7..63714da0ca09 100644 --- a/airbyte-integrations/connectors/source-wrike/setup.py +++ b/airbyte-integrations/connectors/source-wrike/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-wrike=source_wrike.run:run", + ], + }, name="source_wrike", description="Source implementation for Wrike.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/run.py b/airbyte-integrations/connectors/source-wrike/source_wrike/run.py new file mode 100644 index 000000000000..5795da526154 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_wrike import SourceWrike + + +def run(): + source = SourceWrike() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-xero/Dockerfile b/airbyte-integrations/connectors/source-xero/Dockerfile index f3099f93deb2..b634bf69f6a1 100644 --- a/airbyte-integrations/connectors/source-xero/Dockerfile +++ b/airbyte-integrations/connectors/source-xero/Dockerfile @@ -34,5 +34,5 @@ COPY source_xero ./source_xero ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-xero diff --git a/airbyte-integrations/connectors/source-xero/main.py b/airbyte-integrations/connectors/source-xero/main.py index ecb627ec90dd..d765f10d2093 100644 --- a/airbyte-integrations/connectors/source-xero/main.py +++ b/airbyte-integrations/connectors/source-xero/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_xero import SourceXero +from source_xero.run import run if __name__ == "__main__": - source = SourceXero() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-xero/metadata.yaml b/airbyte-integrations/connectors/source-xero/metadata.yaml index da8edb234071..164c3987663f 100644 --- a/airbyte-integrations/connectors/source-xero/metadata.yaml +++ b/airbyte-integrations/connectors/source-xero/metadata.yaml @@ -5,12 +5,16 @@ data: connectorSubtype: api connectorType: source definitionId: 6fd1e833-dd6e-45ec-a727-ab917c5be892 - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-xero githubIssueLabel: source-xero icon: xero.svg license: MIT name: Xero + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-xero registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-xero/setup.py b/airbyte-integrations/connectors/source-xero/setup.py index f44d404f36c2..31cd43ef5838 100644 --- a/airbyte-integrations/connectors/source-xero/setup.py +++ b/airbyte-integrations/connectors/source-xero/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-xero=source_xero.run:run", + ], + }, name="source_xero", description="Source implementation for Xero.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-xero/source_xero/run.py b/airbyte-integrations/connectors/source-xero/source_xero/run.py new file mode 100644 index 000000000000..fb8d5955af03 --- /dev/null +++ b/airbyte-integrations/connectors/source-xero/source_xero/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_xero import SourceXero + + +def run(): + source = SourceXero() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-xkcd/main.py b/airbyte-integrations/connectors/source-xkcd/main.py index c0de98487459..ea20bdf50104 100644 --- a/airbyte-integrations/connectors/source-xkcd/main.py +++ b/airbyte-integrations/connectors/source-xkcd/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_xkcd import SourceXkcd +from source_xkcd.run import run if __name__ == "__main__": - source = SourceXkcd() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-xkcd/metadata.yaml b/airbyte-integrations/connectors/source-xkcd/metadata.yaml index 6fb89af41897..cd6153d0b8da 100644 --- a/airbyte-integrations/connectors/source-xkcd/metadata.yaml +++ b/airbyte-integrations/connectors/source-xkcd/metadata.yaml @@ -8,6 +8,10 @@ data: icon: xkcd.svg license: MIT name: xkcd + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-xkcd registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-xkcd/setup.py b/airbyte-integrations/connectors/source-xkcd/setup.py index 7fe43ebd17d6..f6c8db1ffa16 100644 --- a/airbyte-integrations/connectors/source-xkcd/setup.py +++ b/airbyte-integrations/connectors/source-xkcd/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-xkcd=source_xkcd.run:run", + ], + }, name="source_xkcd", description="Source implementation for Xkcd.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py b/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py new file mode 100644 index 000000000000..3e056bd11961 --- /dev/null +++ b/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_xkcd import SourceXkcd + + +def run(): + source = SourceXkcd() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/main.py b/airbyte-integrations/connectors/source-yahoo-finance-price/main.py index 7f376eec167c..3b00718b8391 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/main.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yahoo_finance_price import SourceYahooFinancePrice +from source_yahoo_finance_price.run import run if __name__ == "__main__": - source = SourceYahooFinancePrice() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml b/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml index 83b5ea098897..081e0d18c685 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml @@ -8,6 +8,10 @@ data: icon: yahoo-finance-price.svg license: MIT name: Yahoo Finance Price + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-yahoo-finance-price registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py index 862dc4e0083d..e010bdc6daee 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-yahoo-finance-price=source_yahoo_finance_price.run:run", + ], + }, name="source_yahoo_finance_price", description="Source implementation for Yahoo Finance Price.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py new file mode 100644 index 000000000000..d24a645a4183 --- /dev/null +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yahoo_finance_price import SourceYahooFinancePrice + + +def run(): + source = SourceYahooFinancePrice() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/main.py b/airbyte-integrations/connectors/source-yandex-metrica/main.py index 6c6a7edcc353..a84b23e0a261 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/main.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yandex_metrica import SourceYandexMetrica +from source_yandex_metrica.run import run if __name__ == "__main__": - source = SourceYandexMetrica() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml index 31c05d9a4759..c2cb2e985886 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml +++ b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml @@ -15,6 +15,10 @@ data: icon: yandexmetrica.svg license: MIT name: Yandex Metrica + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-yandex-metrica registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-yandex-metrica/setup.py b/airbyte-integrations/connectors/source-yandex-metrica/setup.py index ce5d5b361010..adebbc0e9bb2 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/setup.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "freezegun", "pytest~=6.1", "pytest-mock", "requests_mock"] setup( + entry_points={ + "console_scripts": [ + "source-yandex-metrica=source_yandex_metrica.run:run", + ], + }, name="source_yandex_metrica", description="Source implementation for Yandex Metrica.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py new file mode 100644 index 000000000000..69e34c3d1740 --- /dev/null +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yandex_metrica import SourceYandexMetrica + + +def run(): + source = SourceYandexMetrica() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yotpo/main.py b/airbyte-integrations/connectors/source-yotpo/main.py index d456ca2abab3..968a262f7f26 100644 --- a/airbyte-integrations/connectors/source-yotpo/main.py +++ b/airbyte-integrations/connectors/source-yotpo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yotpo import SourceYotpo +from source_yotpo.run import run if __name__ == "__main__": - source = SourceYotpo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yotpo/metadata.yaml b/airbyte-integrations/connectors/source-yotpo/metadata.yaml index 56487a95fd70..efb0e929ba31 100644 --- a/airbyte-integrations/connectors/source-yotpo/metadata.yaml +++ b/airbyte-integrations/connectors/source-yotpo/metadata.yaml @@ -8,6 +8,10 @@ data: icon: yotpo.svg license: MIT name: Yotpo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-yotpo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-yotpo/setup.py b/airbyte-integrations/connectors/source-yotpo/setup.py index ed41819fb6d9..4d1d2163dbbd 100644 --- a/airbyte-integrations/connectors/source-yotpo/setup.py +++ b/airbyte-integrations/connectors/source-yotpo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-yotpo=source_yotpo.run:run", + ], + }, name="source_yotpo", description="Source implementation for Yotpo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py b/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py new file mode 100644 index 000000000000..d302e5b2f4dd --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yotpo import SourceYotpo + + +def run(): + source = SourceYotpo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-younium/main.py b/airbyte-integrations/connectors/source-younium/main.py index 311b077eb084..8fcc0a655630 100644 --- a/airbyte-integrations/connectors/source-younium/main.py +++ b/airbyte-integrations/connectors/source-younium/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_younium import SourceYounium +from source_younium.run import run if __name__ == "__main__": - source = SourceYounium() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-younium/metadata.yaml b/airbyte-integrations/connectors/source-younium/metadata.yaml index 892a1ee423c0..c1b23187fcec 100644 --- a/airbyte-integrations/connectors/source-younium/metadata.yaml +++ b/airbyte-integrations/connectors/source-younium/metadata.yaml @@ -1,4 +1,8 @@ data: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-younium registries: oss: enabled: true @@ -23,5 +27,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/younium tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py index 2a8872be5287..25ca4c963925 100644 --- a/airbyte-integrations/connectors/source-younium/setup.py +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-younium=source_younium.run:run", + ], + }, name="source_younium", description="Source implementation for Younium.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-younium/source_younium/run.py b/airbyte-integrations/connectors/source-younium/source_younium/run.py new file mode 100644 index 000000000000..5250f8d9bd94 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_younium import SourceYounium + + +def run(): + source = SourceYounium() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/main.py b/airbyte-integrations/connectors/source-youtube-analytics/main.py index 046af990bcb0..f2542cccc965 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/main.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_youtube_analytics import SourceYoutubeAnalytics +from source_youtube_analytics.run import run if __name__ == "__main__": - source = SourceYoutubeAnalytics() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml index 9cd1fe065eaa..6168a00de77f 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml +++ b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml @@ -15,6 +15,10 @@ data: icon: youtube-analytics.svg license: MIT name: YouTube Analytics + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-youtube-analytics registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-youtube-analytics/setup.py b/airbyte-integrations/connectors/source-youtube-analytics/setup.py index 795c2044a1ca..1a4c92407af1 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/setup.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-youtube-analytics=source_youtube_analytics.run:run", + ], + }, name="source_youtube_analytics", description="Source implementation for Youtube Analytics.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py new file mode 100644 index 000000000000..27f8967eff43 --- /dev/null +++ b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_youtube_analytics import SourceYoutubeAnalytics + + +def run(): + source = SourceYoutubeAnalytics() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/main.py b/airbyte-integrations/connectors/source-zapier-supported-storage/main.py index b65c91f88848..8de0f27f55af 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/main.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zapier_supported_storage import SourceZapierSupportedStorage +from source_zapier_supported_storage.run import run if __name__ == "__main__": - source = SourceZapierSupportedStorage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml b/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml index d5f9864b86ab..c3a192df8c1a 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: zapiersupportedstorage.svg license: MIT name: Zapier Supported Storage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zapier-supported-storage registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py index 1ffccc828857..9feb0c8cde94 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zapier-supported-storage=source_zapier_supported_storage.run:run", + ], + }, name="source_zapier_supported_storage", description="Source implementation for Zapier Supported Storage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py b/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py new file mode 100644 index 000000000000..a3777475b5b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zapier_supported_storage import SourceZapierSupportedStorage + + +def run(): + source = SourceZapierSupportedStorage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/README.md b/airbyte-integrations/connectors/source-zendesk-chat/README.md index 5e8b7fd035dc..f7d40d3e06a8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/README.md +++ b/airbyte-integrations/connectors/source-zendesk-chat/README.md @@ -1,100 +1,91 @@ -# XKCD Source +# Zendesk-Chat source connector -This is the repository for the Xkcd source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/xkcd). + +This is the repository for the Zendesk-Chat source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-chat). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python3 -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/xkcd) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_xkcd/spec.yaml` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-chat) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_chat/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source xkcd test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-zendesk-chat spec +poetry run source-zendesk-chat check --config secrets/config.json +poetry run source-zendesk-chat discover --config secrets/config.json +poetry run source-zendesk-chat read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash -airbyte-ci connectors --name source-xkcd build +airbyte-ci connectors --name=source-zendesk-chat build ``` -An image will be built with the tag `airbyte/source-xkcd:dev`. +An image will be available on your host with the tag `airbyte/source-zendesk-chat:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-xkcd:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-xkcd:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-xkcd:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-zendesk-chat:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-chat:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-chat:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-chat:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-zendesk-chat test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-chat test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-chat.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-chat.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py index e85c96df52a9..f3342e5b52cb 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py @@ -8,7 +8,6 @@ if TYPE_CHECKING: from dagger import Container -MAIN_FILE_NAME = "main_dev.py" async def pre_connector_install(base_image_container: Container) -> Container: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt index 33c52b2289bf..10f75ba0af99 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt @@ -8,17 +8,17 @@ {"stream": "bans", "data": {"type": "visitor", "id": 75411401, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 62959049", "visitor_id": "10414779.13ojzHu7at4VKcG"}, "emitted_at": 1672828433831} {"stream": "bans", "data": {"created_at": "2021-04-27T15:52:32Z", "visitor_id": "10414779.13ojzHu7at4VKcG", "id": 75411401, "reason": "Spammer", "visitor_name": "Visitor 62959049", "type": "visitor"}, "emitted_at": 1672828434000} {"stream": "bans", "data": {"created_at": "2021-04-27T15:52:33Z", "visitor_id": "10414779.13ojzHu7s9YwIjz", "id": 75411441, "reason": "Spammer", "visitor_name": "Visitor 97350211", "type": "visitor"}, "emitted_at": 1672828434001} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "0.83900", "name": "Fake user - chat 116", "email": "fake_user_chat_116@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:28Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:28Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsV9Tu0Ynj", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "6.42465", "name": "Fake user - chat 117", "email": "fake_user_chat_117@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsbJTqVJsF", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "8.89712", "name": "Fake user - chat 118", "email": "fake_user_chat_118@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsgcJUJbVN", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "9.61246", "name": "Fake user - chat 119", "email": "fake_user_chat_119@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WslzhLr3zm", "zendesk_ticket_id": null}, "emitted_at": 1672828434385} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "update_timestamp": "2021-04-27T15:09:17Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:54:02Z", "unread": false, "id": "2104.10414779.SVhDCJ9flq79a", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730189} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "update_timestamp": "2021-04-30T11:06:19Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-21T14:36:55Z", "unread": false, "id": "2104.10414779.SVE9Mo9bE4wR8", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "update_timestamp": "2021-04-30T11:08:12Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:53:30Z", "unread": false, "id": "2104.10414779.SVhD3v7I1LBOq", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "7.34502", "name": "Fake user - chat 2", "email": "fake_user_chat_2@doe.com"}, "update_timestamp": "2021-04-30T13:32:27Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-30T13:32:27Z", "unread": true, "id": "2104.10414779.SW4VrjJpOq6gk", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730191} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282640316815}, "members": [361084605116], "name": "Airbyte Department 1", "enabled": true, "description": "A sample department", "id": 7282640316815}, "emitted_at": 1688547521914} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282618889231}, "members": [360786799676], "name": "Department 1", "enabled": true, "description": "A sample department", "id": 7282618889231}, "emitted_at": 1688547521914} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282630247567}, "members": [361089721035, 361084605116], "name": "Department 2", "enabled": true, "description": "A sample department 2", "id": 7282630247567}, "emitted_at": 1688547521914} -{"stream": "goals", "data": {"description": "A new goal", "id": 513481, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal 3", "enabled": true, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434873} -{"stream": "goals", "data": {"description": "A new goal - 1", "id": 529641, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal one", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} -{"stream": "goals", "data": {"description": "A new goal - 2", "id": 529681, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal two", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} -{"stream": "goals", "data": {"description": "Test goal", "id": 537121, "attribution_model": "last_touch", "attribution_period": 30, "name": "Test goal", "enabled": true, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1672828434874} +{"stream": "goals", "data": {"enabled": true, "id": 513481, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal 3", "description": "A new goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031915} +{"stream": "goals", "data": {"enabled": false, "id": 529641, "attribution_period": 5, "attribution_model": "first_touch", "name": "Goal one", "description": "A new goal - 1", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} +{"stream": "goals", "data": {"enabled": false, "id": 529681, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal two", "description": "A new goal - 2", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} +{"stream": "goals", "data": {"enabled": true, "id": 537121, "attribution_period": 30, "attribution_model": "last_touch", "name": "Test goal", "description": "Test goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1701453031916} {"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "enabled": true, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "id": 360002848996, "name": "Administrator", "members_count": 1}, "emitted_at": 1672828435141} {"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "enabled": true, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "id": 360002848976, "name": "Agent", "members_count": 2}, "emitted_at": 1672828435142} {"stream": "shortcuts", "data": {"name": "goodbye", "id": "goodbye", "options": "Yes/No", "tags": ["goodbye_survey"], "scope": "all", "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1672828435386} @@ -31,4 +31,4 @@ {"stream": "triggers", "data": {"name": "Product Discounts", "enabled": true, "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page.", "id": 66052801, "definition": {"event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} {"stream": "triggers", "data": {"name": "Request Contact Details", "enabled": true, "description": "When your account is set to away, ask customer's requesting a chat to leave their email address.", "id": 66052841, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]], "actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} {"stream": "triggers", "data": {"name": "Tag Repeat Visitors", "enabled": true, "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand.", "id": 66052881, "definition": {"event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]], "actions": [["addTag", "5times"]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}}, "emitted_at": 1688547526146} +{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1701453336379} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main.py b/airbyte-integrations/connectors/source-zendesk-chat/main.py new file mode 100644 index 000000000000..c2c8d74d092a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_zendesk_chat.run import run + +if __name__ == "__main__": + run() diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py b/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py deleted file mode 100644 index 56361251988d..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_chat import SourceZendeskChat - -if __name__ == "__main__": - source = SourceZendeskChat() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 7620e39f8aeb..1aa58d5146b5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -1,6 +1,6 @@ data: ab_internal: - ql: 400 + ql: 200 sl: 200 allowedHosts: hosts: @@ -10,13 +10,17 @@ data: connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-zendesk-chat documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat githubIssueLabel: source-zendesk-chat icon: zendesk-chat.svg license: MIT name: Zendesk Chat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zendesk-chat registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock new file mode 100644 index 000000000000..4035ad70602b --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -0,0 +1,1018 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.41" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.41.tar.gz", hash = "sha256:cce614d67872cf66a151e5b72d70f4bf26e2a1ce672c7abfc15a5cb4e45d8429"}, + {file = "airbyte_cdk-0.51.41-py3-none-any.whl", hash = "sha256:bbf82a45d9ec97c4a92b85e3312b327f8060fffec1f7c7ea7dfa720f9adcc13b"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "e55b65b435ed00315a8288393c1fb2adde5904ae32b5aed66f133bdb721a6991" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml new file mode 100644 index 000000000000..f47dbc02c81d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.2.2" +name = "source-zendesk-chat" +description = "Source implementation for Zendesk Chat." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/zendesk-chat" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_zendesk_chat" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.51.41" +pendulum = "==2.1.2" + +[tool.poetry.scripts] +source-zendesk-chat = "source_zendesk_chat.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.12.0" +requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt b/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py deleted file mode 100644 index 8e1732196deb..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum"] - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock"] - -setup( - name="source_zendesk_chat", - description="Source implementation for Zendesk Chat.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/run.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/run.py new file mode 100644 index 000000000000..f87a47735571 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_chat import SourceZendeskChat + + +def run(): + source = SourceZendeskChat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/main.py b/airbyte-integrations/connectors/source-zendesk-sell/main.py index 6f01a3b8f7a8..e57dccf7d35e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/main.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_sell import SourceZendeskSell +from source_zendesk_sell.run import run if __name__ == "__main__": - source = SourceZendeskSell() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml index 803cca0d539b..511885e7ea86 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.getbase.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zendesk-sell registries: oss: enabled: false @@ -19,7 +23,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index 054742fb41d4..a789b08aeb99 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-sell=source_zendesk_sell.run:run", + ], + }, name="source_zendesk_sell", description="Source implementation for Zendesk Sell.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py new file mode 100644 index 000000000000..0e238308da46 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sell import SourceZendeskSell + + +def run(): + source = SourceZendeskSell() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/main.py b/airbyte-integrations/connectors/source-zendesk-sunshine/main.py index 0a86066993fc..4b7507ee396c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/main.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_sunshine import SourceZendeskSunshine +from source_zendesk_sunshine.run import run if __name__ == "__main__": - source = SourceZendeskSunshine() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index 003390c81a7a..f226bcd50be1 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${subdomain}.zendesk.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zendesk-sunshine registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py index 1c47ce8ab0a9..d59a2a4793b4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-sunshine=source_zendesk_sunshine.run:run", + ], + }, name="source_zendesk_sunshine", description="Source implementation for Zendesk Sunshine.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py new file mode 100644 index 000000000000..be323283bc95 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sunshine import SourceZendeskSunshine + + +def run(): + source = SourceZendeskSunshine() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-support/README.md b/airbyte-integrations/connectors/source-zendesk-support/README.md index 11b8aa2fd4cb..a1b9fd4aabbd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/README.md +++ b/airbyte-integrations/connectors/source-zendesk-support/README.md @@ -1,118 +1,55 @@ -# Source Zendesk Support Source +# Zendesk-Support source connector -This is the repository for the Source Zendesk Support source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-support). + +This is the repository for the Zendesk-Support source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-support). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-support) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_support/spec.json` file. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-support) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_support/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source source-zendesk-support test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-zendesk-support spec +poetry run source-zendesk-support check --config secrets/config.json +poetry run source-zendesk-support discover --config secrets/config.json +poetry run source-zendesk-support read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - - - -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-zendesk-support build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-zendesk-support:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container +An image will be available on your host with the tag `airbyte/source-zendesk-support:dev`. -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-zendesk-support:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: -```bash -docker build -t airbyte/source-zendesk-support:dev . -# Running the spec command against your patched connector -docker run airbyte/source-zendesk-support:dev spec -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-zendesk-support:dev spec @@ -121,29 +58,34 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-support:dev di docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-support:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-zendesk-support test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-support test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-support.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-support.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index c20d59cbd37a..e08e4169e436 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -1,15 +1,15 @@ {"stream": "articles", "data": {"id": 7253351877519, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253351877519.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253351877519-Sample-article-Stellar-Skyonomy-refund-policies", "author_id": 360786799676, "comments_disabled": false, "draft": true, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394933775, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "Sample article: Stellar Skyonomy refund policies", "title": "Sample article: Stellar Skyonomy refund policies", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      All Stellar Skyonomy merchandise purchases are backed by our 30-day satisfaction guarantee, no questions asked. We even pay to have it shipped back to us. Additionally, you can cancel your Stellar Skyonomy subscription at any time. Before you cancel, review our refund policies in this article.


      Refund policy

      We automatically issue a full refund when you initiate a return within 30 days of delivery.

      To cancel an annual website subscription you can do so at any time and your refund will be prorated based on the cancellation date.


      Request a refund

      If you believe you\u2019re eligible for a refund but haven\u2019t received one, contact us by completing a refund request form. We review every refund and aim to respond within two business days.

      If you haven't received a refund you're expecting, note that it can take up to 10 business days to appear on your card statement.

      "}, "emitted_at": 1697714809846} -{"stream": "articles", "data": {"id": 7253391134863, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253391134863.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253391134863-How-can-agents-leverage-knowledge-to-help-customers-", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How can agents leverage knowledge to help customers?", "title": "How can agents leverage knowledge to help customers?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can use our Knowledge Capture app to leverage your team\u2019s collective knowledge.

      \n

      Using the app, agents can:\n

        \n
      • Search the Help Center without leaving the ticket
        • \n
      • Insert links to relevant Help Center articles in ticket comments
        • \n
      • Add inline feedback to existing articles that need updates
        • \n
      • Create new articles while answering tickets using a pre-defined template
        • \n
      \n\n\n

      Agents never have to leave the ticket interface to share, flag, or create knowledge, so they can help the customer, while also improving your self-service offerings for other customers.

      \n\n

      To get started, see our Knowledge Capture documentation.

      \n\n

      And before your agents can start creating new knowledge directly from tickets, you\u2019ll need to create a template for them to use. To help you along, we\u2019ve provided some template ideas below. You can copy and paste any sample template below into a new article, add the KCTemplate label to the article, and you\u2019ll be all set.

      \n\n

      Q&A template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Question

      \nwrite the question here.\n\n\n

      \n

      \n

      Answer

      \nwrite the answer here.\n\n\n
      \n\n

      Solution template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Symptoms

      \nwrite the symptoms here.\n\n\n

      \n

      \n

      Resolution

      \nwrite the resolution here.\n\n\n

      \n

      \n

      Cause

      \nwrite the cause here.\n\n\n
      \n\n

      How-to template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Objective

      \nwrite the purpose or task here.\n\n\n

      \n

      \n

      Procedure

      \nwrite the steps here.\n\n\n
      \n"}, "emitted_at": 1697714809848} -{"stream": "articles", "data": {"id": 7253394952591, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394952591.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394952591-How-do-I-customize-my-Help-Center-", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How do I customize my Help Center?", "title": "How do I customize my Help Center?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can modify the look and feel of your Help Center by changing colors and fonts. See Branding your Help Center to learn how.

      \n\n

      You can also change the design of your Help Center. If you're comfortable working with page code, you can dig in to the site's HTML, CSS, and Javascript to customize your theme. To get started, see Customizing the Help Center.

      "}, "emitted_at": 1697714809849} +{"stream": "articles", "data": {"id": 7253391134863, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253391134863.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253391134863-How-can-agents-leverage-knowledge-to-help-customers", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How can agents leverage knowledge to help customers?", "title": "How can agents leverage knowledge to help customers?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can use our Knowledge Capture app to leverage your team\u2019s collective knowledge.

      \n

      Using the app, agents can:\n

        \n
      • Search the Help Center without leaving the ticket
        • \n
      • Insert links to relevant Help Center articles in ticket comments
        • \n
      • Add inline feedback to existing articles that need updates
        • \n
      • Create new articles while answering tickets using a pre-defined template
        • \n
      \n\n\n

      Agents never have to leave the ticket interface to share, flag, or create knowledge, so they can help the customer, while also improving your self-service offerings for other customers.

      \n\n

      To get started, see our Knowledge Capture documentation.

      \n\n

      And before your agents can start creating new knowledge directly from tickets, you\u2019ll need to create a template for them to use. To help you along, we\u2019ve provided some template ideas below. You can copy and paste any sample template below into a new article, add the KCTemplate label to the article, and you\u2019ll be all set.

      \n\n

      Q&A template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Question

      \nwrite the question here.\n\n\n

      \n

      \n

      Answer

      \nwrite the answer here.\n\n\n
      \n\n

      Solution template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Symptoms

      \nwrite the symptoms here.\n\n\n

      \n

      \n

      Resolution

      \nwrite the resolution here.\n\n\n

      \n

      \n

      Cause

      \nwrite the cause here.\n\n\n
      \n\n

      How-to template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Objective

      \nwrite the purpose or task here.\n\n\n

      \n

      \n

      Procedure

      \nwrite the steps here.\n\n\n
      \n"}, "emitted_at": 1697714809848} +{"stream": "articles", "data": {"id": 7253394952591, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394952591.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394952591-How-do-I-customize-my-Help-Center", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How do I customize my Help Center?", "title": "How do I customize my Help Center?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can modify the look and feel of your Help Center by changing colors and fonts. See Branding your Help Center to learn how.

      \n\n

      You can also change the design of your Help Center. If you're comfortable working with page code, you can dig in to the site's HTML, CSS, and Javascript to customize your theme. To get started, see Customizing the Help Center.

      "}, "emitted_at": 1697714809849} {"stream": "article_comments", "data": {"id": 7253381447311, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394935055/comments/7253381447311.json", "body": "

      Test comment 2

      ", "author_id": 360786799676, "source_id": 7253394935055, "source_type": "Article", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394935055/comments/7253381447311", "locale": "en-us", "created_at": "2023-06-22T00:33:36Z", "updated_at": "2023-06-22T00:33:42Z", "vote_sum": -1, "vote_count": 1, "non_author_editor_id": null, "non_author_updated_at": null}, "emitted_at": 1697714814160} {"stream": "article_comments", "data": {"id": 7253366869647, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394935055/comments/7253366869647.json", "body": "

      Test comment

      ", "author_id": 360786799676, "source_id": 7253394935055, "source_type": "Article", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394935055/comments/7253366869647", "locale": "en-us", "created_at": "2023-06-22T00:33:29Z", "updated_at": "2023-06-22T00:33:40Z", "vote_sum": 1, "vote_count": 1, "non_author_editor_id": null, "non_author_updated_at": null}, "emitted_at": 1697714814162} {"stream": "article_comment_votes", "data": {"id": 7253393200655, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7253393200655.json", "user_id": 360786799676, "value": -1, "item_id": 7253381447311, "item_type": "Comment", "created_at": "2023-06-22T00:33:42Z", "updated_at": "2023-06-22T00:33:42Z"}, "emitted_at": 1697714823072} {"stream": "article_comment_votes", "data": {"id": 7253381522703, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7253381522703.json", "user_id": 360786799676, "value": 1, "item_id": 7253366869647, "item_type": "Comment", "created_at": "2023-06-22T00:33:40Z", "updated_at": "2023-06-22T00:33:40Z"}, "emitted_at": 1697714823501} {"stream": "article_votes", "data": {"id": 7816935174287, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7816935174287.json", "user_id": 360786799676, "value": 1, "item_id": 7253394935055, "item_type": "Article", "created_at": "2023-09-04T13:52:38Z", "updated_at": "2023-09-04T13:52:38Z"}, "emitted_at": 1697714827544} {"stream": "article_votes", "data": {"id": 7816935384335, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7816935384335.json", "user_id": 360786799676, "value": 1, "item_id": 7253391120527, "item_type": "Article", "created_at": "2023-09-04T13:52:58Z", "updated_at": "2023-09-04T13:52:58Z"}, "emitted_at": 1697714828540} -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8170722077839.json", "id": 8170722077839, "action_label": "Updated", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "update", "change_description": "Password: Changed", "ip_address": null, "created_at": "2023-10-19T11:20:04Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829754} -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8156156463759.json", "id": 8156156463759, "action_label": "Updated", "actor_id": -1, "source_id": 8156194806799, "source_type": "account_setting", "source_label": "Agent Workspace Auto Activation opt out", "action": "create", "change_description": "Turned on", "ip_address": null, "created_at": "2023-10-17T22:00:13Z", "actor_name": "Zendesk"}, "emitted_at": 1697714829755} -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8154367957263.json", "id": 8154367957263, "action_label": "Updated", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "update", "change_description": "Password: Changed", "ip_address": null, "created_at": "2023-10-17T14:01:52Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829755} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8841266675343.json", "id": 8841266675343, "action_label": "Updated", "actor_id": 360786799676, "source_id": 8841127505167, "source_type": "user", "source_label": "Dylan Dominguez499", "action": "update", "change_description": "Organization: Test998 is assigned", "ip_address": "24.228.86.152", "created_at": "2024-01-19T15:55:46Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829754} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8841266672271.json", "id": 8841266672271, "action_label": "Updated", "actor_id": 360786799676, "source_id": 8841127505039, "source_type": "user", "source_label": "Dylan Dominguez498", "action": "update", "change_description": "Organization: Test997 is assigned", "ip_address": "24.228.86.152", "created_at": "2024-01-19T15:55:46Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829755} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8841239171855.json", "id": 8841239171855, "action_label": "Updated", "actor_id": 360786799676, "source_id": 8841127505423, "source_type": "user", "source_label": "Customer: Dylan Dominguez501", "action": "update", "change_description": "Primary email changed from not set to dylandominguez501@gmail.com", "ip_address": "24.228.86.152", "created_at": "2024-01-19T15:55:46Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829755} {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1697714830912} {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1697714830913} {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1697714830914} @@ -24,8 +24,8 @@ {"stream": "organization_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_fields/7376673274511.json", "id": 7376673274511, "type": "checkbox", "key": "test_check_box_field_1", "title": "Test Check box field 1", "description": "Description for a Test Check box field", "raw_title": "Test Check box field 1", "raw_description": "Description for a Test Check box field", "position": 2, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-07-10T08:36:58Z", "updated_at": "2023-07-10T08:36:58Z", "tag": "check_box_1"}, "emitted_at": 1697714836211} {"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/360057705196.json", "id": 360057705196, "user_id": 360786799676, "organization_id": 360033549136, "default": true, "created_at": "2020-12-11T18:34:05Z", "organization_name": "Airbyte", "updated_at": "2020-12-11T18:34:05Z", "view_tickets": true}, "emitted_at": 1697714837426} {"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/7282880134671.json", "id": 7282880134671, "user_id": 7282634891791, "organization_id": 360033549136, "default": true, "created_at": "2023-06-26T11:03:38Z", "organization_name": "Airbyte", "updated_at": "2023-06-26T11:03:38Z", "view_tickets": true}, "emitted_at": 1697714837428} -{"stream": "posts", "data": {"id": 7253351904271, "title": "How do I get around the community?", "details": "

      You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838032} -{"stream": "posts", "data": {"id": 7253375870607, "title": "Which topics should I add to my community?", "details": "

      That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.

      A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.

      \n\n

      To create your own topics, see Adding community discussion topics.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838034} +{"stream": "posts", "data": {"id": 7253351904271, "title": "How do I get around the community?", "details": "

      You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838032} +{"stream": "posts", "data": {"id": 7253375870607, "title": "Which topics should I add to my community?", "details": "

      That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.

      A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.

      \n\n

      To create your own topics, see Adding community discussion topics.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838034} {"stream": "posts", "data": {"id": 7253375879055, "title": "I'd like a way for users to submit feature requests", "details": "

      You can add a topic like this one in your community. End users can add feature requests and describe their use cases. Other users can comment on the requests and vote for them. Product managers can review feature requests and provide feedback.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253394974479, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838034} {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/7235633102607.json", "id": 7235633102607, "assignee_id": null, "group_id": null, "requester_id": 361089721035, "ticket_id": 146, "score": "offered", "created_at": "2023-06-19T18:01:40Z", "updated_at": "2023-06-19T18:01:40Z", "comment": null}, "emitted_at": 1697714848277} {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5909514818319.json", "id": 5909514818319, "assignee_id": null, "group_id": null, "requester_id": 360786799676, "ticket_id": 25, "score": "offered", "created_at": "2022-11-22T17:02:04Z", "updated_at": "2022-11-22T17:02:04Z", "comment": null}, "emitted_at": 1697714848279} @@ -54,13 +54,13 @@ {"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1658140562}, "emitted_at": 1697714865824} {"stream": "topics", "data": {"id": 7253394974479, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253394974479.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253394974479-Feature-Requests", "name": "Feature Requests", "description": null, "position": 0, "follower_count": 1, "community_id": 7253391140495, "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "manageable_by": "managers", "user_segment_id": null}, "emitted_at": 1697714866838} {"stream": "topics", "data": {"id": 7253351897871, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253351897871.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253351897871-General-Discussion", "name": "General Discussion", "description": null, "position": 0, "follower_count": 1, "community_id": 7253391140495, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "manageable_by": "managers", "user_segment_id": null}, "emitted_at": 1697714866839} -{"stream": "users", "data": {"id": 4992781783439, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json", "name": "Caller +1 (689) 689-8023", "email": null, "created_at": "2022-06-17T14:49:19Z", "updated_at": "2022-06-17T14:49:19Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16896898023", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "chat_only": false, "default_group_id": null, "report_csv": false, "user_fields": {"test_display_name_checkbox_field": false, "test_display_name_decimal_field": null, "test_display_name_text_field": null}}, "emitted_at": 1697714871902} -{"stream": "users", "data": {"id": 4993467856015, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json", "name": "Caller +1 (912) 420-0314", "email": null, "created_at": "2022-06-17T19:52:38Z", "updated_at": "2022-06-17T19:52:38Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+19124200314", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "chat_only": false, "default_group_id": null, "report_csv": false, "user_fields": {"test_display_name_checkbox_field": false, "test_display_name_decimal_field": null, "test_display_name_text_field": null}}, "emitted_at": 1697714871904} -{"stream": "users", "data": {"id": 5137812260495, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json", "name": "Caller +1 (607) 210-9549", "email": null, "created_at": "2022-07-13T14:34:04Z", "updated_at": "2022-07-13T14:34:04Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16072109549", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "chat_only": false, "default_group_id": null, "report_csv": false, "user_fields": {"test_display_name_checkbox_field": false, "test_display_name_decimal_field": null, "test_display_name_text_field": null}}, "emitted_at": 1697714871905} +{"stream":"users","data":{"id":4992781783439,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json","name":"Caller +1 (689) 689-8023","email":null,"created_at":"2022-06-17T14:49:19Z","updated_at":"2022-06-17T14:49:19Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16896898023","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960493} +{"stream":"users","data":{"id":4993467856015,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json","name":"Caller +1 (912) 420-0314","email":null,"created_at":"2022-06-17T19:52:38Z","updated_at":"2022-06-17T19:52:38Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+19124200314","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960494} +{"stream":"users","data":{"id":5137812260495,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json","name":"Caller +1 (607) 210-9549","email":null,"created_at":"2022-07-13T14:34:04Z","updated_at":"2022-07-13T14:34:04Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16072109549","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960494} {"stream": "brands", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/brands/360000358316.json", "id": 360000358316, "name": "Airbyte", "brand_url": "https://d3v-airbyte.zendesk.com", "subdomain": "d3v-airbyte", "host_mapping": null, "has_help_center": true, "help_center_state": "enabled", "active": true, "default": true, "is_deleted": false, "logo": null, "ticket_form_ids": [360000084116], "signature_template": "{{agent.signature}}", "created_at": "2020-12-11T18:34:04Z", "updated_at": "2020-12-11T18:34:09Z"}, "emitted_at": 1697714873604} -{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_automations": true, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": true, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854337} -{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_automations": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": false, "manage_slas": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": false, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854338} -{"stream": "custom_roles", "data": {"id": 360000210616, "name": "Team lead", "description": "Can manage all tickets and forums", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2023-06-26T11:06:24Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": true, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "all", "ticket_comment_access": "public", "ticket_deletion": true, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": true, "voice_access": true, "group_access": true, "organization_editing": true, "organization_notes_editing": true, "assign_tickets_to_any_group": false, "end_user_profile_access": "full", "explore_access": "edit", "forum_access": "full", "macro_access": "full", "report_access": "full", "ticket_editing": true, "ticket_merge": true, "user_view_access": "full", "view_access": "playonly", "voice_dashboard_access": true, "manage_automations": true, "manage_contextual_workspaces": true, "manage_organization_fields": true, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": true, "manage_ticket_forms": true, "manage_user_fields": true, "ticket_redaction": true, "manage_roles": "all-except-self", "manage_groups": true, "manage_group_memberships": true, "manage_organizations": true, "manage_suspended_tickets": true, "manage_triggers": true, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 2}, "emitted_at": 1698749854339} +{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_automations": true, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": true, "manage_slas": true, "manage_suspended_tickets": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_triggers": true, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_deletion_schedules": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854337} +{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_automations": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": false, "manage_slas": false, "manage_suspended_tickets": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_triggers": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_deletion_schedules": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854338} +{"stream": "custom_roles", "data": {"id": 360000210616, "name": "Team lead", "description": "Can manage all tickets and forums", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2023-06-26T11:06:24Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": true, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "all", "ticket_comment_access": "public", "ticket_deletion": true, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": true, "voice_access": true, "group_access": true, "organization_editing": true, "organization_notes_editing": true, "assign_tickets_to_any_group": false, "end_user_profile_access": "full", "explore_access": "edit", "forum_access": "full", "macro_access": "full", "report_access": "full", "ticket_editing": true, "ticket_merge": true, "user_view_access": "full", "view_access": "playonly", "voice_dashboard_access": true, "manage_automations": true, "manage_contextual_workspaces": true, "manage_organization_fields": true, "manage_skills": true, "manage_slas": true, "manage_suspended_tickets": true, "manage_ticket_fields": true, "manage_ticket_forms": true, "manage_triggers": true, "manage_user_fields": true, "ticket_redaction": true, "manage_roles": "all-except-self", "manage_deletion_schedules": "none", "manage_groups": true, "manage_group_memberships": true, "manage_organizations": true, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 2}, "emitted_at": 1698749854339} {"stream": "schedules", "data": {"id": 4567312249615, "name": "Test Schedule", "time_zone": "New Caledonia", "created_at": "2022-03-25T10:23:34Z", "updated_at": "2022-03-25T10:23:34Z", "intervals": [{"start_time": 1980, "end_time": 2460}, {"start_time": 3420, "end_time": 3900}, {"start_time": 4860, "end_time": 5340}, {"start_time": 6300, "end_time": 6780}, {"start_time": 7740, "end_time": 8220}]}, "emitted_at": 1697714875775} {"stream": "user_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/user_fields/7761239926287.json", "id": 7761239926287, "type": "text", "key": "test_display_name_text_field", "title": "test Display Name text field", "description": "test Display Name text field", "raw_title": "test Display Name text field", "raw_description": "test Display Name text field", "position": 0, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-08-28T10:10:46Z", "updated_at": "2023-08-28T10:10:46Z"}, "emitted_at": 1697714876719} {"stream": "user_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/user_fields/7761264848527.json", "id": 7761264848527, "type": "checkbox", "key": "test_display_name_checkbox_field", "title": "test Display Name Checkbox field", "description": "", "raw_title": "test Display Name Checkbox field", "raw_description": "", "position": 1, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-08-28T10:11:16Z", "updated_at": "2023-08-28T10:11:16Z", "tag": null}, "emitted_at": 1697714876720} diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json index 08b3d403363c..a77590cbd4ed 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json @@ -204,18 +204,6 @@ "sync_mode": "incremental", "destination_sync_mode": "append" }, - { - "stream": { - "name": "posts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, { "stream": { "name": "organization_memberships", diff --git a/airbyte-integrations/connectors/source-zendesk-support/main.py b/airbyte-integrations/connectors/source-zendesk-support/main.py index d3b005c42d35..88eed5ec56af 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/main.py +++ b/airbyte-integrations/connectors/source-zendesk-support/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_support import SourceZendeskSupport +from source_zendesk_support.run import run if __name__ == "__main__": - source = SourceZendeskSupport() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml index e608c6186930..8f6493cfd6c7 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml @@ -7,11 +7,11 @@ data: - ${subdomain}.zendesk.com - zendesk.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 - dockerImageTag: 2.2.2 + dockerImageTag: 2.2.8 dockerRepository: airbyte/source-zendesk-support documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support githubIssueLabel: source-zendesk-support @@ -19,6 +19,10 @@ data: license: ELv2 maxSecondsBetweenMessages: 10800 name: Zendesk Support + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zendesk-support registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-zendesk-support/poetry.lock b/airbyte-integrations/connectors/source-zendesk-support/poetry.lock new file mode 100644 index 000000000000..457d40ddb6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/poetry.lock @@ -0,0 +1,1059 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.60.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.60.2.tar.gz", hash = "sha256:11cae56d77ae28dff228016373d8573d5fe7b9f65f7b984586283bb904f628ea"}, + {file = "airbyte_cdk-0.60.2-py3-none-any.whl", hash = "sha256:c34d601a50de2b8e0a4732bf5f7c08eeb9b41972df816e1fb6164eea250c8928"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "92ec2b4ac91287d9aed5533a5929649943394a41c5dca9427fd7278c956490ae" diff --git a/airbyte-integrations/connectors/source-zendesk-support/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-support/pyproject.toml new file mode 100644 index 000000000000..e0944429cb23 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.2.8" +name = "source-zendesk-support" +description = "Source implementation for Zendesk Support." +authors = [ "Airbyte ",] +license = "ELv2" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/zendesk-support" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_zendesk_support" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.60.2" +pytz = "==2024.1" + +[tool.poetry.scripts] +source-zendesk-support = "source_zendesk_support.run:run" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.1" +pytest-mock = "^3.6" +requests-mock = "==1.9.3" +freezegun = "^1.4.0" diff --git a/airbyte-integrations/connectors/source-zendesk-support/requirements.txt b/airbyte-integrations/connectors/source-zendesk-support/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py deleted file mode 100644 index 519b1350c914..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz"] - -TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock~=3.6", "requests-mock==1.9.3"] - -setup( - version="0.1.0", - name="source_zendesk_support", - description="Source implementation for Zendesk Support.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py new file mode 100644 index 000000000000..95b88323a18c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_support import SourceZendeskSupport + + +def run(): + source = SourceZendeskSupport() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json index b2167cc3f446..976fca82707f 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json @@ -60,6 +60,18 @@ }, "confition": { "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "collection_key": { + "type": ["null", "string"] + }, + "item_key": { + "type": ["null", "string"] + } + }, + "additionalProperties": true } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json index 442a7c65f39a..1909b117b645 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json @@ -24,6 +24,56 @@ "chat_access": { "type": ["null", "boolean"] }, + "manage_automations": { + "type": ["null", "boolean"] + }, + "manage_group_memberships": { + "type": ["null", "boolean"] + }, + "manage_groups": { + "type": ["null", "boolean"] + }, + "manage_macro_content_suggestions": { + "type": ["null", "boolean"] + }, + "manage_organizations": { + "type": ["null", "boolean"] + }, + "manage_skills": { + "type": ["null", "boolean"] + }, + "manage_suspended_tickets": { + "type": ["null", "boolean"] + }, + "manage_triggers": { + "type": ["null", "boolean"] + }, + "manage_slas": { + "type": ["null", "boolean"] + }, + "read_macro_content_suggestions": { + "type": ["null", "boolean"] + }, + "ticket_redaction": { + "type": ["null", "boolean"] + }, + "view_filter_tickets": { + "type": ["null", "boolean"] + }, + "view_reduced_count": { + "type": ["null", "boolean"] + }, + "manage_roles": { + "type": ["null", "string"] + }, + "custom_objects": { + "type": ["null", "object"], + "properties": {}, + "additionalProperties": true + }, + "manage_deletion_schedules": { + "type": ["null", "string"] + }, "end_user_list_access": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/attachments.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/attachments.json index 5c235ba83a1c..28ed648b86f1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/attachments.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/attachments.json @@ -11,9 +11,18 @@ "url": { "type": ["null", "string"] }, + "malware_scan_result": { + "type": ["null", "string"] + }, "inline": { "type": ["null", "boolean"] }, + "deleted": { + "type": ["null", "boolean"] + }, + "malware_access_override": { + "type": ["null", "boolean"] + }, "height": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json index cea500a4e061..33658c964d6d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json @@ -128,6 +128,9 @@ "ticket_form_id": { "type": ["null", "integer"] }, + "deleted_ticket_form_id": { + "type": ["null", "integer"] + }, "satisfaction_rating": { "type": ["null", "object", "string"], "properties": { diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via.json index 4fb4506bb191..dabb3367ef1b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via.json @@ -19,6 +19,9 @@ "subject": { "type": ["null", "string"] }, + "channel": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via_channel.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via_channel.json index d37cc65685bf..a53ec6f045b9 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via_channel.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/via_channel.json @@ -28,6 +28,9 @@ "subject": { "type": ["null", "string"] }, + "channel": { + "type": ["null", "string"] + }, "id": { "type": ["null", "integer"] }, @@ -66,6 +69,9 @@ "to": { "type": ["null", "object"], "properties": { + "brand_id": { + "type": ["null", "integer"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json index 664f9ba4f330..453cf1fb6dda 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json @@ -76,6 +76,15 @@ "type": ["null", "object"] }, "type": ["null", "array"] + }, + "deleted": { + "type": ["null", "boolean"] + }, + "malware_access_override": { + "type": ["null", "boolean"] + }, + "malware_scan_result": { + "type": ["null", "string"] } }, "type": ["null", "object"] @@ -89,6 +98,36 @@ "data": { "type": ["null", "object"], "properties": { + "author_id": { + "type": ["null", "integer"] + }, + "brand_id": { + "type": ["null", "integer"] + }, + "call_id": { + "type": ["null", "integer"] + }, + "line_type": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "public": { + "type": ["null", "boolean"] + }, + "recorded": { + "type": ["null", "boolean"] + }, + "recording_consent_action": { + "type": ["null", "string"] + }, + "recording_type": { + "type": ["null", "string"] + }, + "via_id": { + "type": ["null", "integer"] + }, "transcription_status": { "type": ["null", "string"] }, @@ -119,6 +158,18 @@ } } }, + "current_followers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "previous_followers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "formatted_from": { "type": ["null", "string"] }, @@ -274,6 +325,9 @@ "metadata": { "type": ["null", "object"], "properties": { + "suspension_type_id": { + "type": ["null", "integer"] + }, "custom": { "type": ["null", "object"], "properties": {}, @@ -288,6 +342,51 @@ "type": ["null", "integer"] } }, + "decoration": { + "type": ["null", "object"], + "properties": { + "channels": { + "type": ["null", "object"], + "properties": { + "allow_channelback": { + "type": ["null", "boolean"] + } + }, + "additionalProperties": true + }, + "source": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "zendesk_id": { + "type": ["null", "integer"] + } + }, + "additionalProperties": true + }, + "created_at": { + "type": ["null", "string"] + }, + "external_id": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "integer"] + } + }, + "additionalProperties": true + }, "flags_options": { "type": ["null", "object"], "properties": { @@ -314,6 +413,22 @@ } } } + }, + "15": { + "type": ["null", "object"], + "properties": { + "trusted": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "object"], + "properties": { + "user": { + "type": ["null", "string"] + } + } + } + } } } }, @@ -349,6 +464,12 @@ }, "latitude": { "type": ["null", "number"] + }, + "email_id": { + "type": ["null", "string"] + }, + "eml_redacted": { + "type": ["null", "boolean"] } } } @@ -385,12 +506,21 @@ "subject": { "type": ["null", "string"] }, + "channel": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"] }, "address": { "type": ["null", "string"] }, + "formatted_phone": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, "original_recipients": { "type": ["null", "array"], "items": { @@ -417,8 +547,17 @@ "name": { "type": ["null", "string"] }, + "brand_id": { + "type": ["null", "integer"] + }, "address": { "type": ["null", "string"] + }, + "formatted_phone": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] } } }, @@ -428,6 +567,62 @@ } } } + }, + "attachments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "content_type": { + "type": ["null", "string"] + }, + "content_url": { + "type": ["null", "string"] + }, + "deleted": { + "type": ["null", "boolean"] + }, + "file_name": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "inline": { + "type": ["null", "boolean"] + }, + "malware_access_override": { + "type": ["null", "boolean"] + }, + "malware_scan_result": { + "type": ["null", "string"] + }, + "mapped_content_url": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "thumbnails": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": {}, + "additionalProperties": true + } + }, + "url": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "additionalProperties": true + } } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 4b62fa45ae12..ab5316725f1a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -256,7 +256,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late new_value = str((latest_record or {}).get(self.cursor_field, "")) return {self.cursor_field: max(new_value, old_value)} - def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + def get_stream_state_value(self, stream_state: Mapping[str, Any] = None) -> int: """ Returns the state value, if exists. Otherwise, returns user defined `Start Date`. """ @@ -281,7 +281,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = { - "start_time": self.check_stream_state(stream_state), + "start_time": self.get_stream_state_value(stream_state), "page[size]": self.page_size, } if next_page_token: @@ -306,7 +306,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) + parsed_state = self.get_stream_state_value(stream_state) if self.cursor_field: params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} else: @@ -319,15 +319,19 @@ class SourceZendeskIncrementalExportStream(IncrementalZendeskSupportStream): https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-export-time-based @ param response_list_name: the main nested entity to look at inside of response, default = response_list_name - @ param sideload_param : parameter variable to include various information to response more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints """ - response_list_name: str = None - sideload_param: str = None + @property + def response_list_name() -> str: + raise NotImplementedError("The `response_list_name` must be implemented") + + @property + def next_page_field() -> str: + raise NotImplementedError("The `next_page_field` varies depending on stream and must be set individually") @staticmethod - def check_start_time_param(requested_start_time: int, value: int = 1) -> int: + def validate_start_time(requested_start_time: int, value: int = 1) -> int: """ Requesting tickets in the future is not allowed, hits 400 - bad request. We get current UNIX timestamp minus `value` from now(), default = 1 (minute). @@ -347,7 +351,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, if self._ignore_pagination: return None response_json = response.json() - return None if response_json.get(END_OF_STREAM_KEY, True) else {"cursor": response_json.get("after_cursor")} + if END_OF_STREAM_KEY in response_json and response_json[END_OF_STREAM_KEY]: + return None + return dict(parse_qsl(urlparse(response_json.get(self.next_page_field, "")).query)) def request_params( self, @@ -355,16 +361,14 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param + """ + Request params are based on parsed query params of next page url. + `start_time` will be included as the initial request parameter and will never be changed unless it is itself a next page token. + """ if next_page_token: - params.update(next_page_token) - return params + return next_page_token + start_time = self.get_stream_state_value(stream_state) + return {"start_time": self.validate_start_time(start_time)} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: for record in response.json().get(self.response_list_name, []): @@ -379,29 +383,24 @@ class SourceZendeskSupportTicketEventsExportStream(SourceZendeskIncrementalExpor @ param response_target_entity: nested property inside of `response_list_name`, default = "child_events" @ param list_entities_from_event : the list of nested child_events entities to include from parent record @ param event_type : specific event_type to check ["Audit", "Change", "Comment", etc] + @ param sideload_param : parameter variable to include various information to response """ cursor_field = "created_at" + event_type: str = None + list_entities_from_event: List[str] = None response_list_name: str = "ticket_events" response_target_entity: str = "child_events" - list_entities_from_event: List[str] = None - event_type: str = None - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return f"incremental/{self.response_list_name}.json" + sideload_param: str = None + next_page_field: str = "next_page" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Returns next_page_token based on `end_of_stream` parameter inside of response - """ - response_json = response.json() - return None if response_json.get(END_OF_STREAM_KEY, True) else {"start_time": response_json.get("end_time")} + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + if self.sideload_param: + params["include"] = self.sideload_param + return params @property def update_event_from_record(self) -> bool: @@ -439,6 +438,7 @@ class Users(SourceZendeskIncrementalExportStream): """Users stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-user-export""" response_list_name: str = "users" + next_page_field: str = "after_url" def path(self, **kwargs) -> str: return "incremental/users/cursor.json" @@ -448,6 +448,7 @@ class Organizations(SourceZendeskIncrementalExportStream): """Organizations stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/""" response_list_name: str = "organizations" + next_page_field: str = "next_page" class Posts(CursorPaginationZendeskSupportStream): @@ -468,41 +469,28 @@ class Tickets(SourceZendeskIncrementalExportStream): transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) cursor_field = "generated_timestamp" + next_page_field = "after_url" def path(self, **kwargs) -> str: return "incremental/tickets/cursor.json" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": self.check_start_time_param(parsed_state)} - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: old_value = (current_stream_state or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) new_value = (latest_record or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) return {self.cursor_field: max(new_value, old_value)} - def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + def get_stream_state_value(self, stream_state: Mapping[str, Any] = None) -> int: """ Returns the state value, if exists. Otherwise, returns user defined `Start Date`. """ return stream_state.get(self.cursor_field) if stream_state else pendulum.parse(self._start_date).int_timestamp - def check_start_time_param(self, requested_start_time: int, value: int = 1) -> int: + def validate_start_time(self, requested_start_time: int, value: int = 1) -> int: """ The stream returns 400 Bad Request StartTimeTooRecent when requesting tasks 1 second before now. Figured out during experiments that the most recent time needed for request to be successful is 3 seconds before now. """ - return super().check_start_time_param(requested_start_time, value=3) + return super().validate_start_time(requested_start_time, value=3) class TicketComments(SourceZendeskSupportTicketEventsExportStream): @@ -594,7 +582,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = { - "start_time": self.check_stream_state(stream_state), + "start_time": self.get_stream_state_value(stream_state), "page[size]": self.page_size, } if next_page_token: # need keep start_time for this stream @@ -610,7 +598,7 @@ class TicketAudits(IncrementalZendeskSupportStream): """TicketAudits stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_audits/""" # can request a maximum of 1,000 results - page_size = 1000 + page_size = 200 # ticket audits doesn't have the 'updated_by' field cursor_field = "created_at" @@ -638,6 +626,23 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, response_json = response.json() return {"cursor": response.json().get("before_cursor")} if response_json.get("before_cursor") else None + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + except requests.exceptions.HTTPError as e: + if e.response.status_code == requests.codes.GATEWAY_TIMEOUT: + self.logger.error(f"Skipping stream `{self.name}`. Timed out waiting for response: {e.response.text}...") + else: + raise e + class Tags(FullRefreshZendeskSupportStream): """Tags stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/tags/""" @@ -838,6 +843,7 @@ class Articles(SourceZendeskIncrementalExportStream): """Articles Stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/articles/#list-articles""" response_list_name: str = "articles" + next_page_field: str = "next_page" def path(self, **kwargs) -> str: return "help_center/incremental/articles" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py new file mode 100644 index 000000000000..cdd05d55f442 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import base64 +from typing import Any, Dict + +from pendulum.datetime import DateTime + + +class ConfigBuilder: + def __init__(self) -> None: + self._subdomain: str = None + self._start_date: str = None + self._credentials: Dict[str, str] = {} + + def with_subdomain(self, subdomain: str) -> "ConfigBuilder": + self._subdomain = subdomain + return self + + def with_oauth_credentials(self, access_token: str) -> "ConfigBuilder": + self._credentials["access_token"] = access_token + self._credentials["credentials"] = "oauth2.0" + return self + + def with_basic_auth_credentials(self, email: str, password: str) -> "ConfigBuilder": + self._credentials["api_token"] = password + self._credentials["credentials"] = "api_token" + self._credentials["email"] = email + return self + + def with_start_date(self, start_date: DateTime) -> "ConfigBuilder": + self._start_date = start_date.format("YYYY-MM-DDTHH:mm:ss[Z]") + return self + + def build(self) -> Dict[str, Any]: + config = {} + if self._subdomain: + config["subdomain"] = self._subdomain + if self._start_date: + config["start_date"] = self._start_date + if self._credentials: + config["credentials"] = self._credentials + return config diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py new file mode 100644 index 000000000000..ef657b2c7637 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from pendulum.datetime import DateTime + +from .utils import datetime_to_string +from .zs_requests import PostsCommentsRequestBuilder, PostsRequestBuilder, TicketFormsRequestBuilder +from .zs_requests.request_authenticators import ApiTokenAuthenticator +from .zs_responses import PostsCommentsResponseBuilder, PostsResponseBuilder, TicketFormsResponseBuilder +from .zs_responses.records import PostsCommentsRecordBuilder, PostsRecordBuilder, TicketFormsRecordBuilder + + +def given_ticket_forms(http_mocker: HttpMocker, start_date: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketFormsRecordBuilder: + """ + Ticket Forms reqests + """ + ticket_forms_record_builder = TicketFormsRecordBuilder.ticket_forms_record().with_field(FieldPath("updated_at"), datetime_to_string(start_date.add(seconds=1))) + http_mocker.get( + TicketFormsRequestBuilder.ticket_forms_endpoint(api_token_authenticator).with_start_time(datetime_to_string(start_date)).build(), + TicketFormsResponseBuilder.ticket_forms_response().with_record(ticket_forms_record_builder).build() + ) + return ticket_forms_record_builder + + +def given_posts(http_mocker: HttpMocker, start_date: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> PostsRecordBuilder: + """ + Posts requests setup + """ + posts_record_builder = PostsRecordBuilder.posts_record().with_field(FieldPath("updated_at"), datetime_to_string(start_date.add(seconds=1))) + http_mocker.get( + PostsRequestBuilder.posts_endpoint(api_token_authenticator).with_start_time(datetime_to_string(start_date)).with_page_size(100).build(), + PostsResponseBuilder.posts_response().with_record(posts_record_builder).build() + ) + return posts_record_builder + + +def given_post_comments(http_mocker: HttpMocker, start_date: DateTime, post_id: int, api_token_authenticator: ApiTokenAuthenticator) -> PostsCommentsRecordBuilder: + """ + Post Comments requests setup + """ + post_comments_record_builder = PostsCommentsRecordBuilder.posts_commetns_record().with_field(FieldPath("updated_at"), datetime_to_string(start_date.add(seconds=1))) + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post_id).with_start_time(datetime_to_string(start_date)).with_page_size(100).build(), + PostsCommentsResponseBuilder.posts_comments_response().with_record(post_comments_record_builder).build() + ) + return post_comments_record_builder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py new file mode 100644 index 000000000000..8444c4888020 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py @@ -0,0 +1,246 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timezone +from unittest import TestCase +from unittest.mock import patch + +import freezegun +import pendulum +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .config import ConfigBuilder +from .helpers import given_post_comments, given_posts, given_ticket_forms +from .utils import datetime_to_string, get_log_messages_by_log_level, read_stream, string_to_datetime +from .zs_requests import PostCommentVotesRequestBuilder +from .zs_requests.request_authenticators import ApiTokenAuthenticator +from .zs_responses import ErrorResponseBuilder, PostCommentVotesResponseBuilder +from .zs_responses.records import PostCommentVotesRecordBuilder + +_NOW = datetime.now(timezone.utc) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsCommentsVouteStreamFullRefresh(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_one_page_when_read_posts_comments_votes_then_return_records(self, http_mocker): + """ + A normal full refresh sync without pagination + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + posts_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = posts_comments_record_builder.build() + + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostCommentVotesResponseBuilder.post_comment_votes_response().with_record(PostCommentVotesRecordBuilder.post_commetn_votes_record()).build() + ) + + output = read_stream("post_comment_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 403 error and then skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + posts_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = posts_comments_record_builder.build() + + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(403).build() + ) + + output = read_stream("post_comment_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 403 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 404 error and then skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + posts_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = posts_comments_record_builder.build() + + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(404).build() + ) + + output = read_stream("post_comment_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 404 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): + """ + Get a 500 error and then stop syncing + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + posts_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = posts_comments_record_builder.build() + + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("post_comment_votes", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 500 error" in error for error in error_logs]) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsCommentsStreamIncremental(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def _get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker): + """ + A normal incremental sync without pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + posts_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = posts_comments_record_builder.build() + + post_comment_votes_record_builder = PostCommentVotesRecordBuilder.post_commetn_votes_record() + post_comment_votes = post_comment_votes_record_builder.build() + + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostCommentVotesResponseBuilder.post_comment_votes_response().with_record(PostCommentVotesRecordBuilder.post_commetn_votes_record()).build() + ) + + output = read_stream("post_comment_votes", SyncMode.incremental, self._config) + assert len(output.records) == 1 + + assert output.most_recent_state == {"post_comment_votes": {"updated_at": post_comment_votes["updated_at"]}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker): + """ + A normal incremental sync with state and pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + + # Ticket Forms mock. Will be the same for check availability and read requests + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + # Posts mock for check availability request + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + post = posts_record_builder.build() + + # Post comment mock for check availability request + post_comments_record_builder = given_post_comments(http_mocker, string_to_datetime(self._config["start_date"]), post["id"], api_token_authenticator) + post_comment = post_comments_record_builder.build() + + # Post comment votes mock for check availability request + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]) \ + .with_start_time(self._config["start_date"]) \ + .with_page_size(100) \ + .build(), + PostCommentVotesResponseBuilder.post_comment_votes_response().with_record(PostCommentVotesRecordBuilder.post_commetn_votes_record()).build() + ) + + state_start_date = pendulum.parse(self._config["start_date"]).add(years=1) + first_page_record_updated_at = state_start_date.add(months=1) + last_page_record_updated_at = first_page_record_updated_at.add(months=2) + + state = {"updated_at": datetime_to_string(state_start_date)} + + posts_record_builder = given_posts(http_mocker, state_start_date, api_token_authenticator) + post = posts_record_builder.build() + + post_comments_record_builder = given_post_comments(http_mocker, state_start_date, post["id"], api_token_authenticator) + post_comment = post_comments_record_builder.build() + + post_comment_votes_first_record_builder = PostCommentVotesRecordBuilder.post_commetn_votes_record() \ + .with_field(FieldPath("updated_at"), datetime_to_string(first_page_record_updated_at)) + + # Read first page request mock + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]) \ + .with_start_time(datetime_to_string(state_start_date)) \ + .with_page_size(100) \ + .build(), + PostCommentVotesResponseBuilder.post_comment_votes_response().with_pagination().with_record(post_comment_votes_first_record_builder).build() + ) + + post_comment_votes_last_record_builder = PostCommentVotesRecordBuilder.post_commetn_votes_record() \ + .with_id("last_record_id_from_last_page") \ + .with_field(FieldPath("updated_at"), datetime_to_string(last_page_record_updated_at)) + + # Read second page request mock + http_mocker.get( + PostCommentVotesRequestBuilder.post_comment_votes_endpoint(api_token_authenticator, post["id"], post_comment["id"]) \ + .with_page_after("after-cursor") \ + .with_page_size(100) \ + .build(), + PostCommentVotesResponseBuilder.post_comment_votes_response().with_record(post_comment_votes_last_record_builder).build() + ) + + output = read_stream("post_comment_votes", SyncMode.incremental, self._config, StateBuilder().with_stream_state("post_comment_votes", state).build()) + assert len(output.records) == 2 + + assert output.most_recent_state == {"post_comment_votes": {"updated_at": datetime_to_string(last_page_record_updated_at)}} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py new file mode 100644 index 000000000000..710dcf3abc64 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py @@ -0,0 +1,218 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timezone +from unittest import TestCase +from unittest.mock import patch + +import freezegun +import pendulum +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .config import ConfigBuilder +from .helpers import given_posts, given_ticket_forms +from .utils import datetime_to_string, get_log_messages_by_log_level, read_stream, string_to_datetime +from .zs_requests import PostsCommentsRequestBuilder +from .zs_requests.request_authenticators import ApiTokenAuthenticator +from .zs_responses import ErrorResponseBuilder, PostsCommentsResponseBuilder +from .zs_responses.records import PostsCommentsRecordBuilder + +_NOW = datetime.now(timezone.utc) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsCommentsStreamFullRefresh(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_one_page_when_read_posts_comments_then_return_records(self, http_mocker): + """ + A normal full refresh sync without pagination + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsCommentsResponseBuilder.posts_comments_response().with_record(PostsCommentsRecordBuilder.posts_commetns_record()).build() + ) + + output = read_stream("post_comments", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 403 error and then skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(403).build() + ) + + output = read_stream("post_comments", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 403 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 404 error and then skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(404).build() + ) + + output = read_stream("post_comments", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 404 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): + """ + Get a 500 error and then stop the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("post_comments", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 500 error" in error for error in error_logs]) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsCommentsStreamIncremental(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def _get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker): + """ + A normal incremental sync without pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + post_comments_record_builder = PostsCommentsRecordBuilder.posts_commetns_record() + + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsCommentsResponseBuilder.posts_comments_response().with_record(post_comments_record_builder).build() + ) + + output = read_stream("post_comments", SyncMode.incremental, self._config) + assert len(output.records) == 1 + + post_comment = post_comments_record_builder.build() + assert output.most_recent_state == {"post_comments": {"updated_at": post_comment["updated_at"]}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker): + """ + A normal incremental sync with state and pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + + # Ticket Forms mock. Will be the same for check availability and read requests + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + # Posts mock for check availability request + _ = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + state_start_date = pendulum.parse(self._config["start_date"]).add(years=1) + first_page_record_updated_at = state_start_date.add(months=1) + last_page_record_updated_at = first_page_record_updated_at.add(months=2) + + state = {"updated_at": datetime_to_string(state_start_date)} + + posts_record_builder = given_posts(http_mocker, state_start_date, api_token_authenticator) + post = posts_record_builder.build() + + post_comments_first_record_builder = PostsCommentsRecordBuilder.posts_commetns_record() \ + .with_field(FieldPath("updated_at"), datetime_to_string(first_page_record_updated_at)) + + # Check availability request mock + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsCommentsResponseBuilder.posts_comments_response().with_record(PostsCommentsRecordBuilder.posts_commetns_record()).build() + ) + + # Read first page request mock + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]) \ + .with_start_time(datetime_to_string(state_start_date)) \ + .with_page_size(100) \ + .build(), + PostsCommentsResponseBuilder.posts_comments_response().with_pagination().with_record(post_comments_first_record_builder).build() + ) + + post_comments_last_record_builder = PostsCommentsRecordBuilder.posts_commetns_record() \ + .with_id("last_record_id_from_last_page") \ + .with_field(FieldPath("updated_at"), datetime_to_string(last_page_record_updated_at)) + + # Read second page request mock + http_mocker.get( + PostsCommentsRequestBuilder.posts_comments_endpoint(api_token_authenticator, post["id"]) \ + .with_page_after("after-cursor") \ + .with_page_size(100) \ + .build(), + PostsCommentsResponseBuilder.posts_comments_response().with_record(post_comments_last_record_builder).build() + ) + + output = read_stream("post_comments", SyncMode.incremental, self._config, StateBuilder().with_stream_state("post_comments", state).build()) + assert len(output.records) == 2 + + assert output.most_recent_state == {"post_comments": {"updated_at": datetime_to_string(last_page_record_updated_at)}} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py new file mode 100644 index 000000000000..a98444bad526 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py @@ -0,0 +1,218 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timezone +from unittest import TestCase +from unittest.mock import patch + +import freezegun +import pendulum +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import FieldPath +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import Level as LogLevel +from airbyte_protocol.models import SyncMode + +from .config import ConfigBuilder +from .helpers import given_posts, given_ticket_forms +from .utils import datetime_to_string, get_log_messages_by_log_level, read_stream, string_to_datetime +from .zs_requests import PostsVotesRequestBuilder +from .zs_requests.request_authenticators import ApiTokenAuthenticator +from .zs_responses import ErrorResponseBuilder, PostsVotesResponseBuilder +from .zs_responses.records import PostsVotesRecordBuilder + +_NOW = datetime.now(timezone.utc) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsVotesStreamFullRefresh(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_one_page_when_read_posts_comments_then_return_records(self, http_mocker): + """ + A normal full refresh sync without pagination + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsVotesResponseBuilder.posts_votes_response().with_record(PostsVotesRecordBuilder.posts_votes_record()).build() + ) + + output = read_stream("post_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 403 error and then skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(403).build() + ) + + output = read_stream("post_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 403 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): + """ + Get a 404 error and skip the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(404).build() + ) + + output = read_stream("post_votes", SyncMode.full_refresh, self._config) + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 404 error" in error for error in error_logs]) + + @HttpMocker() + def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): + """ + Get a 500 error and stop the stream + """ + api_token_authenticator = self.get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + ErrorResponseBuilder.response_with_status(500).build() + ) + + with patch('time.sleep', return_value=None): + output = read_stream("post_votes", SyncMode.full_refresh, self._config) + + assert len(output.records) == 0 + + error_logs = get_log_messages_by_log_level(output.logs, LogLevel.ERROR) + assert any(["the 500 error" in error for error in error_logs]) + + +@freezegun.freeze_time(_NOW.isoformat()) +class TestPostsVotesStreamIncremental(TestCase): + @property + def _config(self): + return ConfigBuilder() \ + .with_basic_auth_credentials("user@example.com", "password") \ + .with_subdomain("d3v-airbyte") \ + .with_start_date(pendulum.now(tz="UTC").subtract(years=2)) \ + .build() + + def _get_authenticator(self, config): + return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) + + @HttpMocker() + def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker): + """ + A normal incremental sync without pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + posts_record_builder = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + post = posts_record_builder.build() + post_comments_record_builder = PostsVotesRecordBuilder.posts_votes_record() + + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsVotesResponseBuilder.posts_votes_response().with_record(post_comments_record_builder).build() + ) + + output = read_stream("post_votes", SyncMode.incremental, self._config) + assert len(output.records) == 1 + + post_comment = post_comments_record_builder.build() + assert output.most_recent_state == {"post_votes": {"updated_at": post_comment["updated_at"]}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker): + """ + A normal incremental sync with state and pagination + """ + api_token_authenticator = self._get_authenticator(self._config) + + # Ticket Forms mock. Will be the same for check availability and read requests + _ = given_ticket_forms(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + # Posts mock for check availability request + _ = given_posts(http_mocker, string_to_datetime(self._config["start_date"]), api_token_authenticator) + + state_start_date = pendulum.parse(self._config["start_date"]).add(years=1) + first_page_record_updated_at = state_start_date.add(months=1) + last_page_record_updated_at = first_page_record_updated_at.add(months=2) + + state = {"updated_at": datetime_to_string(state_start_date)} + + posts_record_builder = given_posts(http_mocker, state_start_date, api_token_authenticator) + post = posts_record_builder.build() + + post_comments_first_record_builder = PostsVotesRecordBuilder.posts_votes_record() \ + .with_field(FieldPath("updated_at"), datetime_to_string(first_page_record_updated_at)) + + # Check availability request mock + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]).with_start_time(self._config["start_date"]).with_page_size(100).build(), + PostsVotesResponseBuilder.posts_votes_response().with_record(PostsVotesRecordBuilder.posts_votes_record()).build() + ) + + # Read first page request mock + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]) \ + .with_start_time(datetime_to_string(state_start_date)) \ + .with_page_size(100) \ + .build(), + PostsVotesResponseBuilder.posts_votes_response().with_pagination().with_record(post_comments_first_record_builder).build() + ) + + post_comments_last_record_builder = PostsVotesRecordBuilder.posts_votes_record() \ + .with_id("last_record_id_from_last_page") \ + .with_field(FieldPath("updated_at"), datetime_to_string(last_page_record_updated_at)) + + # Read second page request mock + http_mocker.get( + PostsVotesRequestBuilder.posts_votes_endpoint(api_token_authenticator, post["id"]) \ + .with_page_after("after-cursor") \ + .with_page_size(100) \ + .build(), + PostsVotesResponseBuilder.posts_votes_response().with_record(post_comments_last_record_builder).build() + ) + + output = read_stream("post_votes", SyncMode.incremental, self._config, StateBuilder().with_stream_state("post_votes", state).build()) + assert len(output.records) == 2 + + assert output.most_recent_state == {"post_votes": {"updated_at": datetime_to_string(last_page_record_updated_at)}} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py new file mode 100644 index 000000000000..d5f3a3d01419 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import operator +from typing import Any, Dict, List, Optional + +import pendulum +from airbyte_cdk.models import AirbyteMessage +from airbyte_cdk.models import Level as LogLevel +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_protocol.models import SyncMode +from pendulum.datetime import DateTime +from source_zendesk_support import SourceZendeskSupport + + +def read_stream( + stream_name: str, + sync_mode: SyncMode, + config: Dict[str, Any], + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build() + return read(SourceZendeskSupport(), config, catalog, state, expecting_exception) + + +def get_log_messages_by_log_level(logs: List[AirbyteMessage], log_level: LogLevel) -> List[str]: + return map(operator.attrgetter("log.message"), filter(lambda x: x.log.level == log_level, logs)) + + +def datetime_to_string(dt: DateTime) -> str: + return dt.format("YYYY-MM-DDTHH:mm:ss[Z]") + + +def string_to_datetime(dt_string: str) -> DateTime: + return pendulum.parse(dt_string) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/__init__.py new file mode 100644 index 000000000000..a992aa2a95cb --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/__init__.py @@ -0,0 +1,5 @@ +from .post_comments_request_builder import PostsCommentsRequestBuilder +from .posts_request_builder import PostsRequestBuilder +from .ticket_forms_request_bilder import TicketFormsRequestBuilder +from .post_votes_request_builder import PostsVotesRequestBuilder +from .post_comment_votes_request_builder import PostCommentVotesRequestBuilder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py new file mode 100644 index 000000000000..a185dd225a2f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import abc +from typing import Any, Dict, Optional + +from airbyte_cdk.test.mock_http import HttpRequest + +from .request_authenticators.authenticator import Authenticator + + +class ZendeskSuppportRequestBuilder(abc.ABC): + @property + @abc.abstractmethod + def url(self) -> str: + """A url""" + + @property + @abc.abstractmethod + def query_params(self) -> Dict[str, Any]: + """Query params""" + + @property + @abc.abstractmethod + def headers(self) -> Dict[str, Any]: + """Headers""" + + @property + @abc.abstractmethod + def request_body(self) -> Optional[str]: + """A request body""" + + def build(self) -> HttpRequest: + return HttpRequest( + url=self.url, + query_params=self.query_params, + headers=self.headers, + body=self.request_body + ) + + +class ZendeskSupportBaseRequestBuilder(ZendeskSuppportRequestBuilder): + def __init__(self, subdomain: str, resource: str) -> None: + self._resource: str = resource + self._subdomain: str = subdomain + self._authenticator: str = None + + @property + def url(self) -> str: + return f"https://{self._subdomain}.zendesk.com/api/v2/{self._resource}" + + @property + def headers(self) -> Dict[str, Any]: + return (super().headers or {}) | { + "Authorization": self._authenticator.client_access_token, + } + + @property + def request_body(self): + return super().request_body + + def with_authenticator(self, authenticator: Authenticator) -> "ZendeskSupportBaseRequestBuilder": + self._authenticator: Authenticator = authenticator + return self + + def with_subdomain(self, subdomain: str) -> "ZendeskSupportBaseRequestBuilder": + self._subdomain: str = subdomain + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py new file mode 100644 index 000000000000..3d224cd79456 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import calendar + +import pendulum + +from .base_request_builder import ZendeskSupportBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class PostCommentVotesRequestBuilder(ZendeskSupportBaseRequestBuilder): + @classmethod + def post_comment_votes_endpoint(cls, authenticator: Authenticator, post_id: int, post_comment_id: int) -> "PostCommentVotesRequestBuilder": + return cls("d3v-airbyte", f"community/posts/{post_id}/comments/{post_comment_id}/votes").with_authenticator(authenticator) + + def __init__(self, subdomain: str, resource: str) -> None: + super().__init__(subdomain, resource) + self._start_time: int = None + self._page_size: int = None + self._page_after: str = None + + @property + def query_params(self): + params = super().query_params or {} + if self._start_time: + params["start_time"] = self._start_time + if self._page_size: + params["page[size]"] = self._page_size + if self._page_after: + params["page[after]"] = self._page_after + return params + + def with_start_time(self, start_time: int) -> "PostCommentVotesRequestBuilder": + self._start_time: int = calendar.timegm(pendulum.parse(start_time).utctimetuple()) + return self + + def with_page_size(self, page_size: int) -> "PostCommentVotesRequestBuilder": + self._page_size: int = page_size + return self + + def with_page_after(self, next_page_token: str) -> "PostCommentVotesRequestBuilder": + self._page_after = next_page_token + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comments_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comments_request_builder.py new file mode 100644 index 000000000000..a905be90bb15 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comments_request_builder.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import calendar + +import pendulum + +from .base_request_builder import ZendeskSupportBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class PostsCommentsRequestBuilder(ZendeskSupportBaseRequestBuilder): + @classmethod + def posts_comments_endpoint(cls, authenticator: Authenticator, post_id: int) -> "PostsCommentsRequestBuilder": + return cls("d3v-airbyte", f"community/posts/{post_id}/comments").with_authenticator(authenticator) + + def __init__(self, subdomain: str, resource: str) -> None: + super().__init__(subdomain, resource) + self._start_time: int = None + self._page_size: int = None + self._page_after: str = None + + @property + def query_params(self): + params = super().query_params or {} + if self._start_time: + params["start_time"] = self._start_time + if self._page_size: + params["page[size]"] = self._page_size + if self._page_after: + params["page[after]"] = self._page_after + return params + + def with_start_time(self, start_time: int) -> "PostsCommentsRequestBuilder": + self._start_time: int = calendar.timegm(pendulum.parse(start_time).utctimetuple()) + return self + + def with_page_size(self, page_size: int) -> "PostsCommentsRequestBuilder": + self._page_size: int = page_size + return self + + def with_page_after(self, next_page_token: str) -> "PostsCommentsRequestBuilder": + self._page_after = next_page_token + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_votes_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_votes_request_builder.py new file mode 100644 index 000000000000..30891ce3818a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_votes_request_builder.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import calendar + +import pendulum + +from .base_request_builder import ZendeskSupportBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class PostsVotesRequestBuilder(ZendeskSupportBaseRequestBuilder): + @classmethod + def posts_votes_endpoint(cls, authenticator: Authenticator, post_id: int) -> "PostsVotesRequestBuilder": + return cls("d3v-airbyte", f"community/posts/{post_id}/votes").with_authenticator(authenticator) + + def __init__(self, subdomain: str, resource: str) -> None: + super().__init__(subdomain, resource) + self._start_time: int = None + self._page_size: int = None + self._page_after: str = None + + @property + def query_params(self): + params = super().query_params or {} + if self._start_time: + params["start_time"] = self._start_time + if self._page_size: + params["page[size]"] = self._page_size + if self._page_after: + params["page[after]"] = self._page_after + return params + + def with_start_time(self, start_time: int) -> "PostsVotesRequestBuilder": + self._start_time: int = calendar.timegm(pendulum.parse(start_time).utctimetuple()) + return self + + def with_page_size(self, page_size: int) -> "PostsVotesRequestBuilder": + self._page_size: int = page_size + return self + + def with_page_after(self, next_page_token: str) -> "PostsVotesRequestBuilder": + self._page_after = next_page_token + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/posts_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/posts_request_builder.py new file mode 100644 index 000000000000..72b5529b9829 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/posts_request_builder.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import calendar + +import pendulum + +from .base_request_builder import ZendeskSupportBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class PostsRequestBuilder(ZendeskSupportBaseRequestBuilder): + @classmethod + def posts_endpoint(cls, authenticator: Authenticator) -> "PostsRequestBuilder": + return cls("d3v-airbyte", "community/posts").with_authenticator(authenticator) + + def __init__(self, subdomain: str, resource: str) -> None: + super().__init__(subdomain, resource) + self._start_time: int = None + self._page_size: int = None + + @property + def query_params(self): + params = super().query_params or {} + if self._start_time: + params["start_time"] = self._start_time + if self._page_size: + params["page[size]"] = self._page_size + return params + + def with_start_time(self, start_time: int) -> "PostsRequestBuilder": + self._start_time: int = calendar.timegm(pendulum.parse(start_time).utctimetuple()) + return self + + def with_page_size(self, page_size: int) -> "PostsRequestBuilder": + self._page_size: int = page_size + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/__init__.py new file mode 100644 index 000000000000..c670d8c3e6c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/__init__.py @@ -0,0 +1 @@ +from .api_token_authenticator import ApiTokenAuthenticator diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/api_token_authenticator.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/api_token_authenticator.py new file mode 100644 index 000000000000..787ec6ba2ad1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/api_token_authenticator.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import base64 + +from .authenticator import Authenticator + + +class ApiTokenAuthenticator(Authenticator): + def __init__(self, email: str, password: str) -> None: + super().__init__() + self._email = f"{email}/token" + self._password = password + + @property + def client_access_token(self) -> str: + api_token = base64.b64encode(f"{self._email}:{self._password}".encode("utf-8")) + return f"Basic {api_token.decode('utf-8')}" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/authenticator.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/authenticator.py new file mode 100644 index 000000000000..0d85143998b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/request_authenticators/authenticator.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import abc + + +class Authenticator(abc.ABC): + @abc.abstractproperty + def client_access_token(self) -> str: + """""" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/ticket_forms_request_bilder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/ticket_forms_request_bilder.py new file mode 100644 index 000000000000..2beab753157e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/ticket_forms_request_bilder.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import calendar + +import pendulum + +from .base_request_builder import ZendeskSupportBaseRequestBuilder +from .request_authenticators.authenticator import Authenticator + + +class TicketFormsRequestBuilder(ZendeskSupportBaseRequestBuilder): + @classmethod + def ticket_forms_endpoint(cls, authenticator: Authenticator) -> "TicketFormsRequestBuilder": + return cls("d3v-airbyte", "ticket_forms").with_authenticator(authenticator) + + def __init__(self, subdomain: str, resource: str) -> None: + super().__init__(subdomain, resource) + self._start_time: int = None + + @property + def query_params(self): + params = super().query_params or {} + if self._start_time: + params["start_time"] = self._start_time + return params + + def with_start_time(self, start_time: int) -> "TicketFormsRequestBuilder": + self._start_time: int = calendar.timegm(pendulum.parse(start_time).utctimetuple()) + return self diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/__init__.py new file mode 100644 index 000000000000..d30abbef3984 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/__init__.py @@ -0,0 +1,6 @@ +from .error_response_builder import ErrorResponseBuilder +from .posts_response_builder import PostsResponseBuilder +from .ticket_forms_response_builder import TicketFormsResponseBuilder +from .post_comments_response_builder import PostsCommentsResponseBuilder +from .post_votes_response_builder import PostsVotesResponseBuilder +from .post_comment_votes_response_builder import PostCommentVotesResponseBuilder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/error_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/error_response_builder.py new file mode 100644 index 000000000000..a746a8d68a05 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/error_response_builder.py @@ -0,0 +1,18 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + + +class ErrorResponseBuilder: + def __init__(self, status_code: int): + self._status_code: int = status_code + + @classmethod + def response_with_status(cls, status_code) -> "ErrorResponseBuilder": + return cls(status_code) + + def build(self) -> HttpResponse: + return HttpResponse(json.dumps(find_template(str(self._status_code), __file__)), self._status_code) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/__init__.py new file mode 100644 index 000000000000..36019e353fec --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/__init__.py @@ -0,0 +1 @@ +from .cursor_based_pagination_strategy import CursorBasedPaginationStrategy diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/cursor_based_pagination_strategy.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/cursor_based_pagination_strategy.py new file mode 100644 index 000000000000..3afea4fa4881 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/pagination_strategies/cursor_based_pagination_strategy.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class CursorBasedPaginationStrategy(PaginationStrategy): + @staticmethod + def update(response: Dict[str, Any]) -> None: + response["meta"]["has_more"] = True + response["meta"]["after_cursor"] = "after-cursor" + response["meta"]["before_cursor"] = "before-cursor" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comment_votes_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comment_votes_response_builder.py new file mode 100644 index 000000000000..a551f4756fcf --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comment_votes_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + +from .pagination_strategies import CursorBasedPaginationStrategy + + +class PostCommentVotesResponseBuilder(HttpResponseBuilder): + @classmethod + def post_comment_votes_response(cls) -> "PostCommentVotesResponseBuilder": + return cls(find_template("votes", __file__), FieldPath("votes"), CursorBasedPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comments_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comments_response_builder.py new file mode 100644 index 000000000000..43466cb187b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_comments_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + +from .pagination_strategies import CursorBasedPaginationStrategy + + +class PostsCommentsResponseBuilder(HttpResponseBuilder): + @classmethod + def posts_comments_response(cls) -> "PostsCommentsResponseBuilder": + return cls(find_template("post_comments", __file__), FieldPath("comments"), CursorBasedPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_votes_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_votes_response_builder.py new file mode 100644 index 000000000000..8cc6d129651a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/post_votes_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + +from .pagination_strategies import CursorBasedPaginationStrategy + + +class PostsVotesResponseBuilder(HttpResponseBuilder): + @classmethod + def posts_votes_response(cls) -> "PostsVotesResponseBuilder": + return cls(find_template("votes", __file__), FieldPath("votes"), CursorBasedPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/posts_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/posts_response_builder.py new file mode 100644 index 000000000000..47b5f24c244d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/posts_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + +from .pagination_strategies import CursorBasedPaginationStrategy + + +class PostsResponseBuilder(HttpResponseBuilder): + @classmethod + def posts_response(cls) -> "PostsResponseBuilder": + return cls(find_template("posts", __file__), FieldPath("posts"), CursorBasedPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/__init__.py new file mode 100644 index 000000000000..5f273e19b106 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/__init__.py @@ -0,0 +1,5 @@ +from .posts_records_builder import PostsRecordBuilder +from .ticket_forms_records_builder import TicketFormsRecordBuilder +from .post_comments_records_builder import PostsCommentsRecordBuilder +from .post_votes_records_builder import PostsVotesRecordBuilder +from .post_comment_votes_records_builder import PostCommentVotesRecordBuilder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comment_votes_records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comment_votes_records_builder.py new file mode 100644 index 000000000000..31d2f8ee218f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comment_votes_records_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .records_builder import ZendeskSupportRecordBuilder + + +class PostCommentVotesRecordBuilder(ZendeskSupportRecordBuilder): + @classmethod + def post_commetn_votes_record(cls) -> "PostCommentVotesRecordBuilder": + record_template = cls.extract_record("votes", __file__, NestedPath(["votes", 0])) + return cls(record_template, FieldPath("id"), FieldPath("updated_at")) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comments_records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comments_records_builder.py new file mode 100644 index 000000000000..860f6594477d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_comments_records_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .records_builder import ZendeskSupportRecordBuilder + + +class PostsCommentsRecordBuilder(ZendeskSupportRecordBuilder): + @classmethod + def posts_commetns_record(cls) -> "PostsCommentsRecordBuilder": + record_template = cls.extract_record("post_comments", __file__, NestedPath(["comments", 0])) + return cls(record_template, FieldPath("id"), FieldPath("updated_at")) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_votes_records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_votes_records_builder.py new file mode 100644 index 000000000000..4963bbe09466 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/post_votes_records_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .records_builder import ZendeskSupportRecordBuilder + + +class PostsVotesRecordBuilder(ZendeskSupportRecordBuilder): + @classmethod + def posts_votes_record(cls) -> "PostsVotesRecordBuilder": + record_template = cls.extract_record("votes", __file__, NestedPath(["votes", 0])) + return cls(record_template, FieldPath("id"), FieldPath("updated_at")) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/posts_records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/posts_records_builder.py new file mode 100644 index 000000000000..29aa291503b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/posts_records_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .records_builder import ZendeskSupportRecordBuilder + + +class PostsRecordBuilder(ZendeskSupportRecordBuilder): + @classmethod + def posts_record(cls) -> "PostsRecordBuilder": + record_template = cls.extract_record("posts", __file__, NestedPath(["posts", 0])) + return cls(record_template, FieldPath("id"), FieldPath("updated_at")) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/records_builder.py new file mode 100644 index 000000000000..49759695a905 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/records_builder.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, RecordBuilder, find_template + + +class ZendeskSupportRecordBuilder(RecordBuilder): + @staticmethod + def extract_record(resource: str, execution_folder: str, data_field: FieldPath): + return data_field.extract(find_template(resource=resource, execution_folder=execution_folder)) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/ticket_forms_records_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/ticket_forms_records_builder.py new file mode 100644 index 000000000000..8226040c45e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/ticket_forms_records_builder.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath + +from .records_builder import ZendeskSupportRecordBuilder + + +class TicketFormsRecordBuilder(ZendeskSupportRecordBuilder): + @classmethod + def ticket_forms_record(cls) -> "TicketFormsRecordBuilder": + record_template = cls.extract_record("ticket_forms", __file__, NestedPath(["ticket_forms", 0])) + return cls(record_template, FieldPath("id"), FieldPath("updated_at")) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_forms_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_forms_response_builder.py new file mode 100644 index 000000000000..3856e25e4a89 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_forms_response_builder.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template + +from .pagination_strategies import CursorBasedPaginationStrategy + + +class TicketFormsResponseBuilder(HttpResponseBuilder): + @classmethod + def ticket_forms_response(cls) -> "TicketFormsResponseBuilder": + return cls(find_template("ticket_forms", __file__), FieldPath("ticket_forms"), CursorBasedPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/403.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/403.json new file mode 100644 index 000000000000..0e580113d992 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/403.json @@ -0,0 +1,3 @@ +{ + "error": "the 403 error" +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/404.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/404.json new file mode 100644 index 000000000000..ef33442ae49c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/404.json @@ -0,0 +1,3 @@ +{ + "error": "the 404 error" +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/500.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/500.json new file mode 100644 index 000000000000..22b54757134e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/500.json @@ -0,0 +1,3 @@ +{ + "error": "the 500 error" +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/post_comments.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/post_comments.json new file mode 100644 index 000000000000..d543348c187c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/post_comments.json @@ -0,0 +1,19 @@ +{ + "meta": { + "has_more": false + }, + "links": { + "first": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271/comments?page%5Bsize%5D=100", + "last": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271/comments?page%5Bbefore%5D=bGFzdF9wYWdl&page%5Bsize%5D=100" + }, + "comments": [ + { + "author_id": 360786799676, + "post_id": 7253351904271, + "body": "My printer is on fire!", + "id": 35467, + "created_at": "2023-06-22T00:32:22Z", + "updated_at": "2023-06-22T00:32:22Z" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/posts.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/posts.json new file mode 100644 index 000000000000..c3cd94c90133 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/posts.json @@ -0,0 +1,36 @@ +{ + "meta": { + "has_more": false, + "after_cursor": "ZJWWk2QAAAAAaQ/KB9CYBgAA", + "before_cursor": "ZJWWk2QAAAAAaQ+0ds2YBgAA" + }, + "links": { + "first": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts?page%5Bsize%5D=100", + "last": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts?page%5Bbefore%5D=bGFzdF9wYWdl&page%5Bsize%5D=100" + }, + "posts": [ + { + "id": 7253351904271, + "title": "How do I get around the community?", + "details": "

      You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

      ", + "author_id": 360786799676, + "vote_sum": 0, + "vote_count": 0, + "comment_count": 0, + "follower_count": 0, + "topic_id": 7253351897871, + "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community-", + "created_at": "2023-06-22T00:32:21Z", + "updated_at": "2023-06-22T00:32:21Z", + "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community-.json", + "featured": false, + "pinned": false, + "closed": false, + "frozen": false, + "status": "none", + "non_author_editor_id": null, + "non_author_updated_at": null, + "content_tag_ids": [] + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/ticket_forms.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/ticket_forms.json new file mode 100644 index 000000000000..1e5b7092691d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/ticket_forms.json @@ -0,0 +1,29 @@ +{ + "next_page": null, + "previous_page": null, + "count": 1, + "ticket_forms": [ + { + "url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json", + "name": "Default Ticket Form", + "display_name": "Default Ticket Form", + "id": 360000084116, + "raw_name": "Default Ticket Form", + "raw_display_name": "Default Ticket Form", + "end_user_visible": true, + "position": 1, + "ticket_field_ids": [ + 360002833076, 360002833096, 360002833116, 360002833136, 360002833156, + 360002833176, 360002833196 + ], + "active": true, + "default": true, + "created_at": "2020-12-11T18:34:37Z", + "updated_at": "2020-12-11T18:34:37Z", + "in_all_brands": true, + "restricted_brand_ids": [], + "end_user_conditions": [], + "agent_conditions": [] + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/votes.json b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/votes.json new file mode 100644 index 000000000000..9f0811d119fd --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/votes.json @@ -0,0 +1,18 @@ +{ + "meta": { + "has_more": false + }, + "links": { + "first": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271/comments?page%5Bsize%5D=100", + "last": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271/comments?page%5Bbefore%5D=bGFzdF9wYWdl&page%5Bsize%5D=100" + }, + "votes": [ + { + "id": 35467, + "user_id": 888887, + "value": -1, + "created_at": "2023-06-22T00:32:22Z", + "updated_at": "2023-06-22T00:32:22Z" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index cdb9a4d84d53..6dde9b4f99e8 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -231,7 +231,7 @@ def test_str2unixtime(): def test_check_start_time_param(): expected = 1626936955 start_time = calendar.timegm(pendulum.parse(DATETIME_STR).utctimetuple()) - output = SourceZendeskIncrementalExportStream.check_start_time_param(start_time) + output = SourceZendeskIncrementalExportStream.validate_start_time(start_time) assert output == expected @@ -245,7 +245,7 @@ def test_check_start_time_param(): ids=["state present", "state is None"], ) def test_check_stream_state(stream_state, expected): - result = Tickets(**STREAM_ARGS).check_stream_state(stream_state) + result = Tickets(**STREAM_ARGS).get_stream_state_value(stream_state) assert result == expected @@ -474,7 +474,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte [ (Macros, None), (Posts, None), - (Organizations, None), + (Organizations, {}), (Groups, None), (TicketFields, None), ], @@ -703,7 +703,7 @@ def test_next_page_token(self, requests_mock, stream_cls, response, expected): ) def test_check_stream_state(self, stream_cls, expected): stream = stream_cls(**STREAM_ARGS) - result = stream.check_stream_state() + result = stream.get_stream_state_value() assert result == expected @pytest.mark.parametrize( @@ -712,7 +712,7 @@ def test_check_stream_state(self, stream_cls, expected): (GroupMemberships, {"page[size]": 100, "sort_by": "asc", "start_time": 1622505600}), (TicketForms, {"start_time": 1622505600}), (TicketMetricEvents, {"page[size]": 100, "start_time": 1622505600}), - (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}), + (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 200}), (SatisfactionRatings, {"page[size]": 100, "sort_by": "asc", "start_time": 1622505600}), (TicketMetrics, {"page[size]": 100, "start_time": 1622505600}), (OrganizationMemberships, {"page[size]": 100, "start_time": 1622505600}), @@ -750,7 +750,7 @@ class TestSourceZendeskIncrementalExportStream: def test_check_start_time_param(self, stream_cls): expected = int(dict(parse_qsl(urlparse(STREAM_URL).query)).get("start_time")) stream = stream_cls(**STREAM_ARGS) - result = stream.check_start_time_param(expected) + result = stream.validate_start_time(expected) assert result == expected @pytest.mark.parametrize( @@ -770,7 +770,7 @@ def test_next_page_token(self, requests_mock, stream_cls): requests_mock.get(STREAM_URL, json={stream_name: {}}) test_response = requests.get(STREAM_URL) output = stream.next_page_token(test_response) - assert output is None + assert output == {} @pytest.mark.parametrize( "stream_cls, expected", @@ -1101,3 +1101,13 @@ def test_read_non_json_error(requests_mock, caplog): ) read_full_refresh(stream) assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") + + +def test_read_ticket_audits_504_error(requests_mock, caplog): + requests_mock.get("https://subdomain.zendesk.com/api/v2/ticket_audits", status_code=504, text="upstream request timeout") + stream = TicketAudits(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + expected_message = ( + "Skipping stream `ticket_audits`. Timed out waiting for response: upstream request timeout..." + ) + read_full_refresh(stream) + assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") diff --git a/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc b/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc new file mode 100644 index 000000000000..753140399d72 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_zendesk_talk/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile deleted file mode 100644 index eb5a50eb68fa..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_zendesk_talk ./source_zendesk_talk -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.9 -LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/README.md b/airbyte-integrations/connectors/source-zendesk-talk/README.md index 344913b997a4..14b1c4f187d5 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/README.md +++ b/airbyte-integrations/connectors/source-zendesk-talk/README.md @@ -1,99 +1,91 @@ -# Zendesk Talk Source +# Zendesk-Talk source connector -This is the repository for the Zendesk Talk source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-talk). + +This is the repository for the Zendesk-Talk source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-talk). ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-talk) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_talk/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-talk) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_talk/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-talk test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-zendesk-talk spec +poetry run source-zendesk-talk check --config secrets/config.json +poetry run source-zendesk-talk discover --config secrets/config.json +poetry run source-zendesk-talk read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -### Locally running the connector docker image - +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: ```bash airbyte-ci connectors --name=source-zendesk-talk build ``` -An image will be built with the tag `airbyte/source-zendesk-talk:dev`. +An image will be available on your host with the tag `airbyte/source-zendesk-talk:dev`. -**Via `docker build`:** -```bash -docker build -t airbyte/source-zendesk-talk:dev . -``` -#### Run +### Running as a docker container Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-zendesk-talk:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-talk:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-talk:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-zendesk-talk:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-talk:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing +### Running our CI test suite You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash airbyte-ci connectors --name=source-zendesk-talk test ``` ### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. -### Publishing a new version of the connector +## Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-talk test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-talk.md`). +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-talk.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl index 056bf233dc49..2620572a5677 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl @@ -4,14 +4,14 @@ {"stream": "addresses", "data": {"id": 360000047915, "name": "Fake Zendesk 998", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "ADa89d87601b4f38b45ca172ba36bc4c36"}, "emitted_at": 1674159463525} {"stream": "addresses", "data": {"id": 360000049276, "name": "Fake Zendesk 997", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "AD1b03f6250ae793c562f9290a47404e5b"}, "emitted_at": 1674159463525} {"stream": "agents_activity", "data": {"name": "Team Airbyte", "agent_id": 360786799676, "via": "client", "avatar_url": "https://d3v-airbyte.zendesk.com/system/photos/7282824912911/Airbyte_logo_220x220_thumb.png", "forwarding_number": null, "average_talk_time": 0, "calls_accepted": 0, "calls_denied": 0, "calls_missed": 0, "online_time": 0, "available_time": 0, "total_call_duration": 0, "total_talk_time": 0, "total_wrap_up_time": 0, "away_time": 0, "call_status": null, "agent_state": "offline", "transfers_only_time": 0, "average_wrap_up_time": 0, "accepted_transfers": 0, "started_transfers": 0, "calls_put_on_hold": 0, "average_hold_time": 0, "total_hold_time": 0, "started_third_party_conferences": 0, "accepted_third_party_conferences": 0}, "emitted_at": 1688470692771} -{"stream": "calls", "data": {"id": 360088814475, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:15Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "failed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 18, "exceeded_queue_wait_time": null, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": null, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": null, "ivr_routed_to": null, "callback": null, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["silence"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472100} -{"stream": "calls", "data": {"id": 360120314196, "created_at": "2021-10-20T15:16:31Z", "updated_at": "2021-10-20T15:56:54Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 8, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} -{"stream": "calls", "data": {"id": 360121169675, "created_at": "2021-10-20T15:16:42Z", "updated_at": "2021-10-20T15:57:03Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 7, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} -{"stream": "calls", "data": {"id": 360121166995, "created_at": "2021-10-20T13:22:25Z", "updated_at": "2021-10-20T16:22:34Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 6, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360121168815, "created_at": "2021-10-20T14:40:05Z", "updated_at": "2021-10-20T17:40:25Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 17, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360120313656, "created_at": "2021-10-20T14:49:50Z", "updated_at": "2021-10-20T17:50:02Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 5, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472103} -{"stream": "calls", "data": {"id": 360120313676, "created_at": "2021-10-20T14:50:08Z", "updated_at": "2021-10-20T17:50:16Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 6, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472103} +{"stream": "calls", "data": {"id": 360088814475, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:15Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "failed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 18, "exceeded_queue_wait_time": null, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": null, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": null, "ivr_routed_to": null, "callback": null, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["silence"]}, "emitted_at": 1701476700438} +{"stream": "calls", "data": {"id": 360120314196, "created_at": "2021-10-20T15:16:31Z", "updated_at": "2021-10-20T15:56:54Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 8, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700440} +{"stream": "calls", "data": {"id": 360121169675, "created_at": "2021-10-20T15:16:42Z", "updated_at": "2021-10-20T15:57:03Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 7, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360121168815, "created_at": "2021-10-20T14:40:05Z", "updated_at": "2021-10-20T17:40:25Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 17, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} +{"stream": "calls", "data": {"id": 360120313656, "created_at": "2021-10-20T14:49:50Z", "updated_at": "2021-10-20T17:50:02Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 5, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} +{"stream": "calls", "data": {"id": 360120313696, "created_at": "2021-10-20T14:50:28Z", "updated_at": "2021-10-20T17:50:38Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 288, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 5, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} {"stream": "call_legs", "data": {"id": 360167085115, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:14Z", "agent_id": 360786799676, "user_id": 0, "duration": 18, "hold_time": 0, "wrap_up_time": 0, "available_via": "browser", "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": 1, "call_charge": "0.003", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["silence"], "call_id": 360088814475, "type": "agent"}, "emitted_at": 1674159475493} {"stream": "call_legs", "data": {"id": 360222253536, "created_at": "2021-10-20T13:22:27Z", "updated_at": "2021-10-20T14:02:47Z", "agent_id": 0, "user_id": null, "duration": 4, "hold_time": 0, "wrap_up_time": null, "available_via": null, "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": null, "call_charge": "0.0", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["none"], "call_id": 360121166995, "type": "customer"}, "emitted_at": 1674159475493} {"stream": "call_legs", "data": {"id": 360223248835, "created_at": "2021-10-20T14:34:28Z", "updated_at": "2021-10-20T14:40:20Z", "agent_id": 0, "user_id": null, "duration": 347, "hold_time": 0, "wrap_up_time": null, "available_via": null, "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": null, "call_charge": "0.0", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["information_not_available"], "call_id": 360120313416, "type": "customer"}, "emitted_at": 1674159475494} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/main.py b/airbyte-integrations/connectors/source-zendesk-talk/main.py index 88d4616c2155..679ec2c79a78 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/main.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_talk import SourceZendeskTalk +from source_zendesk_talk.run import run if __name__ == "__main__": - source = SourceZendeskTalk() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml index 98b742f21721..be1a86e383ae 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml @@ -1,28 +1,34 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - ${subdomain}.zendesk.com - zendesk.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: c8630570-086d-4a40-99ae-ea5b18673071 - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.12 dockerRepository: airbyte/source-zendesk-talk + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-talk githubIssueLabel: source-zendesk-talk icon: zendesk-talk.svg license: MIT name: Zendesk Talk + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zendesk-talk registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-talk + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/poetry.lock b/airbyte-integrations/connectors/source-zendesk-talk/poetry.lock new file mode 100644 index 000000000000..7941e3a9b3df --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/poetry.lock @@ -0,0 +1,1034 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.8" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.8.tar.gz", hash = "sha256:80cfad673302802e0f5d485879f1bd2f3679a4e3b12b2af42bd7bb37a3991a71"}, + {file = "airbyte_cdk-0.58.8-py3-none-any.whl", hash = "sha256:5b0b19745e96ba3f20683c48530d58a00be48361dfa34ec3c38cef8da03ba330"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "b9f1312ff855d2ea6c2f6c7a329923044ad6cd1b88c9c3de3b49736510b45be6" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-talk/pyproject.toml new file mode 100644 index 000000000000..80585d460f4a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.12" +name = "source-zendesk-talk" +description = "Source implementation for Zendesk Talk." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/zendesk-talk" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_zendesk_talk" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.58.8" + +[tool.poetry.scripts] +source-zendesk-talk = "source_zendesk_talk.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" +pytest-mock = "^3.6" +pytest = "^6.1" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt b/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt deleted file mode 100644 index 7b9114ed5867..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . diff --git a/airbyte-integrations/connectors/source-zendesk-talk/setup.py b/airbyte-integrations/connectors/source-zendesk-talk/setup.py deleted file mode 100644 index e0e910f6461b..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk"] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", -] - -setup( - name="source_zendesk_talk", - description="Source implementation for Zendesk Talk.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py new file mode 100644 index 000000000000..154690ce67d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_talk import SourceZendeskTalk + + +def run(): + source = SourceZendeskTalk() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zenefits/main.py b/airbyte-integrations/connectors/source-zenefits/main.py index 480734a07faa..7124907d176d 100644 --- a/airbyte-integrations/connectors/source-zenefits/main.py +++ b/airbyte-integrations/connectors/source-zenefits/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zenefits import SourceZenefits +from source_zenefits.run import run if __name__ == "__main__": - source = SourceZenefits() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zenefits/metadata.yaml b/airbyte-integrations/connectors/source-zenefits/metadata.yaml index 7392e0333eec..7fcc31df2d8b 100644 --- a/airbyte-integrations/connectors/source-zenefits/metadata.yaml +++ b/airbyte-integrations/connectors/source-zenefits/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.zenefits.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zenefits registries: cloud: enabled: false @@ -24,5 +28,5 @@ data: ql: 100 supportLevel: community tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zenefits/setup.py b/airbyte-integrations/connectors/source-zenefits/setup.py index 5cc7df205794..7d33a0d79c91 100644 --- a/airbyte-integrations/connectors/source-zenefits/setup.py +++ b/airbyte-integrations/connectors/source-zenefits/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-zenefits=source_zenefits.run:run", + ], + }, name="source_zenefits", description="Source implementation for Zenefits.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py b/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py new file mode 100644 index 000000000000..3fa66926fadf --- /dev/null +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zenefits import SourceZenefits + + +def run(): + source = SourceZenefits() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zenloop/main.py b/airbyte-integrations/connectors/source-zenloop/main.py index ec861621c9c7..dd3a6687740e 100644 --- a/airbyte-integrations/connectors/source-zenloop/main.py +++ b/airbyte-integrations/connectors/source-zenloop/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zenloop import SourceZenloop +from source_zenloop.run import run if __name__ == "__main__": - source = SourceZenloop() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zenloop/metadata.yaml b/airbyte-integrations/connectors/source-zenloop/metadata.yaml index 7c364bba5b83..33d3698900e9 100644 --- a/airbyte-integrations/connectors/source-zenloop/metadata.yaml +++ b/airbyte-integrations/connectors/source-zenloop/metadata.yaml @@ -11,6 +11,10 @@ data: icon: zenloop.svg license: MIT name: Zenloop + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zenloop registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-zenloop/setup.py b/airbyte-integrations/connectors/source-zenloop/setup.py index be53c38ee558..8bc3ceffdfcb 100644 --- a/airbyte-integrations/connectors/source-zenloop/setup.py +++ b/airbyte-integrations/connectors/source-zenloop/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zenloop=source_zenloop.run:run", + ], + }, name="source_zenloop", description="Source implementation for Zenloop.", author="Alexander Batoulis", author_email="alexander.batoulis@hometogo.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py new file mode 100644 index 000000000000..344453ff360c --- /dev/null +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zenloop import SourceZenloop + + +def run(): + source = SourceZenloop() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoho-crm/main.py b/airbyte-integrations/connectors/source-zoho-crm/main.py index 911fb44f52f2..2cf82bb23bf9 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/main.py +++ b/airbyte-integrations/connectors/source-zoho-crm/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zoho_crm import SourceZohoCrm +from source_zoho_crm.run import run if __name__ == "__main__": - source = SourceZohoCrm() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml b/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml index 915c75b1b4e3..275e22b752cd 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml +++ b/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml @@ -12,6 +12,10 @@ data: icon: zohocrm.svg license: MIT name: ZohoCRM + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-zoho-crm registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index 4627c0115e63..15425f380be4 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zoho-crm=source_zoho_crm.run:run", + ], + }, name="source_zoho_crm", description="Source implementation for Zoho Crm.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py new file mode 100644 index 000000000000..d915dc05f2b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zoho_crm import SourceZohoCrm + + +def run(): + source = SourceZohoCrm() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoom/main.py b/airbyte-integrations/connectors/source-zoom/main.py index da33508f6a8e..137aea5931e6 100644 --- a/airbyte-integrations/connectors/source-zoom/main.py +++ b/airbyte-integrations/connectors/source-zoom/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zoom import SourceZoom +from source_zoom.run import run if __name__ == "__main__": - source = SourceZoom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zoom/metadata.yaml b/airbyte-integrations/connectors/source-zoom/metadata.yaml index 9d5eb4f875ba..7f8e9be8b7c3 100644 --- a/airbyte-integrations/connectors/source-zoom/metadata.yaml +++ b/airbyte-integrations/connectors/source-zoom/metadata.yaml @@ -8,6 +8,11 @@ data: icon: zoom.svg license: MIT name: Zoom + remoteRegistries: + pypi: + # TODO: Enable once build problems are fixed + enabled: false + packageName: airbyte-source-zoom registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-zoom/setup.py b/airbyte-integrations/connectors/source-zoom/setup.py index edc76fde557e..6d4f526d5e35 100644 --- a/airbyte-integrations/connectors/source-zoom/setup.py +++ b/airbyte-integrations/connectors/source-zoom/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zoom=source_zoom.run:run", + ], + }, name="source_zoom", description="Source implementation for Zoom.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/run.py b/airbyte-integrations/connectors/source-zoom/source_zoom/run.py new file mode 100644 index 000000000000..e663e8441844 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zoom import SourceZoom + + +def run(): + source = SourceZoom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zuora/main.py b/airbyte-integrations/connectors/source-zuora/main.py index e65d47a77eda..404a72854a9e 100644 --- a/airbyte-integrations/connectors/source-zuora/main.py +++ b/airbyte-integrations/connectors/source-zuora/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zuora import SourceZuora +from source_zuora.run import run if __name__ == "__main__": - source = SourceZuora() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zuora/metadata.yaml b/airbyte-integrations/connectors/source-zuora/metadata.yaml index 4298aba664ae..2070c12d1591 100644 --- a/airbyte-integrations/connectors/source-zuora/metadata.yaml +++ b/airbyte-integrations/connectors/source-zuora/metadata.yaml @@ -12,11 +12,15 @@ data: icon: zuora.svg license: MIT name: Zuora - registries: - cloud: + remoteRegistries: + pypi: enabled: true + packageName: airbyte-source-zuora + registries: # Removed from registries due to LEGACY STATE + cloud: + enabled: false oss: - enabled: true + enabled: false releaseStage: alpha supportLevel: community tags: diff --git a/airbyte-integrations/connectors/source-zuora/setup.py b/airbyte-integrations/connectors/source-zuora/setup.py index 6fb6e922ce98..6cec429e1996 100644 --- a/airbyte-integrations/connectors/source-zuora/setup.py +++ b/airbyte-integrations/connectors/source-zuora/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zuora=source_zuora.run:run", + ], + }, name="source_zuora", description="Airbyte source-connector for Zuora.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zuora/source_zuora/run.py b/airbyte-integrations/connectors/source-zuora/source_zuora/run.py new file mode 100644 index 000000000000..58495d4d67e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zuora/source_zuora/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zuora import SourceZuora + + +def run(): + source = SourceZuora() + launch(source, sys.argv[1:]) diff --git a/airbyte-lib/.gitattributes b/airbyte-lib/.gitattributes new file mode 100644 index 000000000000..7af38cfbe107 --- /dev/null +++ b/airbyte-lib/.gitattributes @@ -0,0 +1,2 @@ +# Hide diffs in auto-generated files +docs/generated/**/* linguist-generated=true diff --git a/airbyte-lib/.gitignore b/airbyte-lib/.gitignore new file mode 100644 index 000000000000..fccb6b3edd80 --- /dev/null +++ b/airbyte-lib/.gitignore @@ -0,0 +1,2 @@ +.venv* +.env diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md new file mode 100644 index 000000000000..868a5483dc63 --- /dev/null +++ b/airbyte-lib/README.md @@ -0,0 +1,109 @@ +# airbyte-lib + +airbyte-lib is a library that allows to run Airbyte syncs embedded into any Python application, without the need to run Airbyte server. + +## Development + +- Make sure [Poetry is installed](https://python-poetry.org/docs/#). +- Run `poetry install` +- For examples, check out the `examples` folder. They can be run via `poetry run python examples/` +- Unit tests and type checks can be run via `poetry run pytest` + +## Release + +- In your PR: + - Bump the version in `pyproject.toml` + - Add a changelog entry to the table below +- Once the PR is merged, go to Github and trigger the `Publish AirbyteLib Manually` workflow. This will publish the new version to PyPI. + +## Secrets Management + +AirbyteLib can auto-import secrets from the following sources: + +1. Environment variables. +2. Variables defined in a local `.env` ("Dotenv") file. +3. [Google Colab secrets](https://medium.com/@parthdasawant/how-to-use-secrets-in-google-colab-450c38e3ec75). +4. Manual entry via [`getpass`](https://docs.python.org/3.9/library/getpass.html). + +_Note: Additional secret store options may be supported in the future. [More info here.](https://github.com/airbytehq/airbyte-lib-private-beta/discussions/5)_ + +### Retrieving Secrets + +```python +from airbyte_lib import get_secret, SecretSource + +source = get_connection("source-github") +source.set_config( + "credentials": { + "personal_access_token": get_secret("GITHUB_PERSONAL_ACCESS_TOKEN"), + } +) +``` + +The `get_secret()` function accepts an optional `source` argument of enum type `SecretSource`. If omitted or set to `SecretSource.ANY`, AirbyteLib will search all available secrets sources. If `source` is set to a specific source, then only that source will be checked. If a list of `SecretSource` entries is passed, then the sources will be checked using the provided ordering. + +By default, AirbyteLib will prompt the user for any requested secrets that are not provided via other secret managers. You can disable this prompt by passing `prompt=False` to `get_secret()`. + +### Versioning + +Versioning follows [Semantic Versioning](https://semver.org/). For new features, bump the minor version. For bug fixes, bump the patch version. For pre-releases, append `dev.N` to the version. For example, `0.1.0dev.1` is the first pre-release of the `0.1.0` version. + +## Documentation + +Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). To generate the documentation, run `poetry run generate-docs`. The documentation will be generated in the `docs/generate` folder. This needs to be done manually when changing the public interface of the library. + +A unit test validates the documentation is up to date. + +## Connector compatibility + +To make a connector compatible with airbyte-lib, the following requirements must be met: +* The connector must be a Python package, with a `pyproject.toml` or a `setup.py` file. +* In the package, there must be a `run.py` file that contains a `run` method. This method should read arguments from the command line, and run the connector with them, outputting messages to stdout. +* The `pyproject.toml` or `setup.py` file must specify a command line entry point for the `run` method called `source-`. This is usually done by adding a `console_scripts` section to the `pyproject.toml` file, or a `entry_points` section to the `setup.py` file. For example: + +```toml +[tool.poetry.scripts] +source-my-connector = "my_connector.run:run" +``` + +```python +setup( + ... + entry_points={ + 'console_scripts': [ + 'source-my-connector = my_connector.run:run', + ], + }, + ... +) +``` + +To publish a connector to PyPI, specify the `pypi` section in the `metadata.yaml` file. For example: + +```yaml +data: + # ... + remoteRegistries: + pypi: + enabled: true + packageName: "airbyte-source-my-connector" +``` + +## Validating source connectors + +To validate a source connector for compliance, the `airbyte-lib-validate-source` script can be used. It can be used like this: + +```bash +airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.json +``` + +The script will install the python package in the provided directory, and run the connector against the provided config. The config should be a valid JSON file, with the same structure as the one that would be provided to the connector in Airbyte. The script will exit with a non-zero exit code if the connector fails to run. + +For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required. + +## Changelog + +| Version | PR | Description | +| ----------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 0.1.0 | [#35184](https://github.com/airbytehq/airbyte/pull/35184) | Beta Release 0.1.0 | +| 0.1.0dev.2 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Initial publish - add publish workflow | diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py new file mode 100644 index 000000000000..aeeea8506c32 --- /dev/null +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -0,0 +1,26 @@ +"""AirbyteLib brings Airbyte ELT to every Python developer.""" +from __future__ import annotations + +from airbyte_lib._factories.cache_factories import get_default_cache, new_local_cache +from airbyte_lib._factories.connector_factories import get_source +from airbyte_lib.caches import DuckDBCache, DuckDBCacheConfig +from airbyte_lib.datasets import CachedDataset +from airbyte_lib.registry import get_available_connectors +from airbyte_lib.results import ReadResult +from airbyte_lib.secrets import SecretSource, get_secret +from airbyte_lib.source import Source + + +__all__ = [ + "CachedDataset", + "DuckDBCache", + "DuckDBCacheConfig", + "get_available_connectors", + "get_source", + "get_default_cache", + "get_secret", + "new_local_cache", + "ReadResult", + "SecretSource", + "Source", +] diff --git a/airbyte-lib/airbyte_lib/_executor.py b/airbyte-lib/airbyte_lib/_executor.py new file mode 100644 index 000000000000..cedc774a9a7f --- /dev/null +++ b/airbyte-lib/airbyte_lib/_executor.py @@ -0,0 +1,457 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import shlex +import subprocess +import sys +from abc import ABC, abstractmethod +from contextlib import contextmanager, suppress +from pathlib import Path +from shutil import rmtree +from typing import IO, TYPE_CHECKING, Any, NoReturn, cast + +from rich import print + +from airbyte_lib import exceptions as exc +from airbyte_lib.registry import ConnectorMetadata +from airbyte_lib.telemetry import SourceTelemetryInfo, SourceType + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + +_LATEST_VERSION = "latest" + + +class Executor(ABC): + def __init__( + self, + *, + name: str | None = None, + metadata: ConnectorMetadata | None = None, + target_version: str | None = None, + ) -> None: + """Initialize a connector executor. + + The 'name' param is required if 'metadata' is None. + """ + if not name and not metadata: + raise exc.AirbyteLibInternalError(message="Either name or metadata must be provided.") + + self.name: str = name or cast(ConnectorMetadata, metadata).name # metadata is not None here + self.metadata: ConnectorMetadata | None = metadata + self.enforce_version: bool = target_version is not None + + self.reported_version: str | None = None + self.target_version: str | None = None + if target_version: + if metadata and target_version == _LATEST_VERSION: + self.target_version = metadata.latest_available_version + else: + self.target_version = target_version + + @abstractmethod + def execute(self, args: list[str]) -> Iterator[str]: + pass + + @abstractmethod + def ensure_installation(self, *, auto_fix: bool = True) -> None: + _ = auto_fix + pass + + @abstractmethod + def install(self) -> None: + pass + + @abstractmethod + def get_telemetry_info(self) -> SourceTelemetryInfo: + pass + + @abstractmethod + def uninstall(self) -> None: + pass + + +@contextmanager +def _stream_from_subprocess(args: list[str]) -> Generator[Iterable[str], None, None]: + process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + def _stream_from_file(file: IO[str]) -> Generator[str, Any, None]: + while True: + line = file.readline() + if not line: + break + yield line + + if process.stdout is None: + raise exc.AirbyteSubprocessError( + message="Subprocess did not return a stdout stream.", + context={ + "args": args, + "returncode": process.returncode, + }, + ) + try: + yield _stream_from_file(process.stdout) + finally: + # Close the stdout stream + if process.stdout: + process.stdout.close() + + # Terminate the process if it is still running + if process.poll() is None: # Check if the process is still running + process.terminate() + try: + # Wait for a short period to allow process to terminate gracefully + process.wait(timeout=10) + except subprocess.TimeoutExpired: + # If the process does not terminate within the timeout, force kill it + process.kill() + + # Now, the process is either terminated or killed. Check the exit code. + exit_code = process.wait() + + # If the exit code is not 0 or -15 (SIGTERM), raise an exception + if exit_code not in (0, -15): + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=exit_code, + ) + + +class VenvExecutor(Executor): + def __init__( + self, + name: str | None = None, + *, + metadata: ConnectorMetadata | None = None, + target_version: str | None = None, + pip_url: str | None = None, + install_root: Path | None = None, + ) -> None: + """Initialize a connector executor that runs a connector in a virtual environment. + + Args: + name: The name of the connector. + metadata: (Optional.) The metadata of the connector. + target_version: (Optional.) The version of the connector to install. + pip_url: (Optional.) The pip URL of the connector to install. + install_root: (Optional.) The root directory where the virtual environment will be + created. If not provided, the current working directory will be used. + """ + super().__init__(name=name, metadata=metadata, target_version=target_version) + + if not pip_url and metadata and not metadata.pypi_package_name: + raise exc.AirbyteConnectorNotPyPiPublishedError( + connector_name=self.name, + context={ + "metadata": metadata, + }, + ) + + self.pip_url = pip_url or ( + metadata.pypi_package_name + if metadata and metadata.pypi_package_name + else f"airbyte-{self.name}" + ) + self.install_root = install_root or Path.cwd() + + def _get_venv_name(self) -> str: + return f".venv-{self.name}" + + def _get_venv_path(self) -> Path: + return self.install_root / self._get_venv_name() + + def _get_connector_path(self) -> Path: + return self._get_venv_path() / "bin" / self.name + + def _run_subprocess_and_raise_on_failure(self, args: list[str]) -> None: + result = subprocess.run( + args, + check=False, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=result.returncode, + log_text=result.stderr.decode("utf-8"), + ) + + def uninstall(self) -> None: + if self._get_venv_path().exists(): + rmtree(str(self._get_venv_path())) + + self.reported_version = None # Reset the reported version from the previous installation + + @property + def docs_url(self) -> str: + """Get the URL to the connector's documentation.""" + # TODO: Refactor installation so that this can just live in the Source class. + return "https://docs.airbyte.com/integrations/sources/" + self.name.lower().replace( + "source-", "" + ) + + def install(self) -> None: + """Install the connector in a virtual environment. + + After installation, the installed version will be stored in self.reported_version. + """ + self._run_subprocess_and_raise_on_failure( + [sys.executable, "-m", "venv", str(self._get_venv_path())] + ) + + pip_path = str(self._get_venv_path() / "bin" / "pip") + print( + f"Installing '{self.name}' into virtual environment '{self._get_venv_path()!s}'.\n" + f"Running 'pip install {self.pip_url}'...\n" + ) + try: + self._run_subprocess_and_raise_on_failure( + args=[pip_path, "install", *shlex.split(self.pip_url)] + ) + except exc.AirbyteSubprocessFailedError as ex: + # If the installation failed, remove the virtual environment + # Otherwise, the connector will be considered as installed and the user may not be able + # to retry the installation. + with suppress(exc.AirbyteSubprocessFailedError): + self.uninstall() + + raise exc.AirbyteConnectorInstallationError from ex + + # Assuming the installation succeeded, store the installed version + self.reported_version = self._get_installed_version(raise_on_error=False, recheck=True) + print( + f"Connector '{self.name}' installed successfully!\n" + f"For more information, see the {self.name} documentation:\n" + f"{self.docs_url}#reference\n" + ) + + def _get_installed_version( + self, + *, + raise_on_error: bool = False, + recheck: bool = False, + ) -> str | None: + """Detect the version of the connector installed. + + Returns the version string if it can be detected, otherwise None. + + If raise_on_error is True, raise an exception if the version cannot be detected. + + If recheck if False and the version has already been detected, return the cached value. + + In the venv, we run the following: + > python -c "from importlib.metadata import version; print(version(''))" + """ + if not recheck and self.reported_version: + return self.reported_version + + connector_name = self.name + if not self.interpreter_path.exists(): + # No point in trying to detect the version if the interpreter does not exist + if raise_on_error: + raise exc.AirbyteLibInternalError( + message="Connector's virtual environment interpreter could not be found.", + context={ + "interpreter_path": self.interpreter_path, + }, + ) + return None + + try: + package_name = ( + self.metadata.pypi_package_name + if self.metadata and self.metadata.pypi_package_name + else f"airbyte-{connector_name}" + ) + return subprocess.check_output( + [ + self.interpreter_path, + "-c", + f"from importlib.metadata import version; print(version('{package_name}'))", + ], + universal_newlines=True, + stderr=subprocess.PIPE, # Don't print to stderr + ).strip() + except Exception: + if raise_on_error: + raise + + return None + + @property + def interpreter_path(self) -> Path: + return self._get_venv_path() / "bin" / "python" + + def ensure_installation( + self, + *, + auto_fix: bool = True, + ) -> None: + """Ensure that the connector is installed in a virtual environment. + + If not yet installed and if install_if_missing is True, then install. + + Optionally, verify that the installed version matches the target version. + + Note: Version verification is not supported for connectors installed from a + local path. + """ + # Store the installed version (or None if not installed) + if not self.reported_version: + self.reported_version = self._get_installed_version() + + original_installed_version = self.reported_version + + reinstalled = False + venv_name = f".venv-{self.name}" + if not self._get_venv_path().exists(): + if not auto_fix: + raise exc.AirbyteConnectorInstallationError( + message="Virtual environment does not exist.", + connector_name=self.name, + context={ + "venv_path": self._get_venv_path(), + }, + ) + + # If the venv path does not exist, install. + self.install() + reinstalled = True + + elif not self._get_connector_path().exists(): + if not auto_fix: + raise exc.AirbyteConnectorInstallationError( + message="Could not locate connector executable within the virtual environment.", + connector_name=self.name, + context={ + "connector_path": self._get_connector_path(), + }, + ) + + # If the connector path does not exist, uninstall and re-install. + # This is sometimes caused by a failed or partial installation. + print( + "Connector executable not found within the virtual environment " + f"at {self._get_connector_path()!s}.\nReinstalling..." + ) + self.uninstall() + self.install() + reinstalled = True + + # By now, everything should be installed. Raise an exception if not. + + connector_path = self._get_connector_path() + if not connector_path.exists(): + raise exc.AirbyteConnectorInstallationError( + message="Connector's executable could not be found within the virtual environment.", + connector_name=self.name, + context={ + "connector_path": self._get_connector_path(), + }, + ) from FileNotFoundError(connector_path) + + if self.enforce_version: + version_after_reinstall: str | None = None + if self.reported_version != self.target_version: + if auto_fix and not reinstalled: + # If we haven't already reinstalled above, reinstall now. + self.install() + reinstalled = True + + if reinstalled: + version_after_reinstall = self.reported_version + + # Check the version again + if self.reported_version != self.target_version: + raise exc.AirbyteConnectorInstallationError( + message="Connector's reported version does not match the target version.", + connector_name=self.name, + context={ + "venv_name": venv_name, + "target_version": self.target_version, + "original_installed_version": original_installed_version, + "version_after_reinstall": version_after_reinstall, + }, + ) + + def execute(self, args: list[str]) -> Iterator[str]: + connector_path = self._get_connector_path() + + with _stream_from_subprocess([str(connector_path), *args]) as stream: + yield from stream + + def get_telemetry_info(self) -> SourceTelemetryInfo: + return SourceTelemetryInfo( + name=self.name, + type=SourceType.VENV, + version=self.reported_version, + ) + + +class PathExecutor(Executor): + def __init__( + self, + name: str | None = None, + *, + path: Path, + target_version: str | None = None, + ) -> None: + """Initialize a connector executor that runs a connector from a local path. + + If path is simply the name of the connector, it will be expected to exist in the current + PATH or in the current working directory. + """ + self.path: Path = path + name = name or path.name + super().__init__(name=name, target_version=target_version) + + def ensure_installation( + self, + *, + auto_fix: bool = True, + ) -> None: + """Ensure that the connector executable can be found. + + The auto_fix parameter is ignored for this executor type. + """ + _ = auto_fix + try: + self.execute(["spec"]) + except Exception as e: + # TODO: Improve error handling. We should try to distinguish between + # a connector that is not installed and a connector that is not + # working properly. + raise exc.AirbyteConnectorExecutableNotFoundError( + connector_name=self.name, + ) from e + + def install(self) -> NoReturn: + raise exc.AirbyteConnectorInstallationError( + message="Connector cannot be installed because it is not managed by airbyte-lib.", + connector_name=self.name, + ) + + def uninstall(self) -> NoReturn: + raise exc.AirbyteConnectorInstallationError( + message="Connector cannot be uninstalled because it is not managed by airbyte-lib.", + connector_name=self.name, + ) + + def execute(self, args: list[str]) -> Iterator[str]: + with _stream_from_subprocess([str(self.path), *args]) as stream: + yield from stream + + def get_telemetry_info(self) -> SourceTelemetryInfo: + return SourceTelemetryInfo( + str(self.name), + SourceType.LOCAL_INSTALL, + version=self.reported_version, + ) diff --git a/airbyte-lib/airbyte_lib/_factories/__init__.py b/airbyte-lib/airbyte_lib/_factories/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/airbyte_lib/_factories/cache_factories.py b/airbyte-lib/airbyte_lib/_factories/cache_factories.py new file mode 100644 index 000000000000..82ad3241920c --- /dev/null +++ b/airbyte-lib/airbyte_lib/_factories/cache_factories.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from pathlib import Path + +import ulid + +from airbyte_lib import exceptions as exc +from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig + + +def get_default_cache() -> DuckDBCache: + """Get a local cache for storing data, using the default database path. + + Cache files are stored in the `.cache` directory, relative to the current + working directory. + """ + config = DuckDBCacheConfig( + db_path="./.cache/default_cache_db.duckdb", + ) + return DuckDBCache(config=config) + + +def new_local_cache( + cache_name: str | None = None, + cache_dir: str | Path | None = None, + *, + cleanup: bool = True, +) -> DuckDBCache: + """Get a local cache for storing data, using a name string to seed the path. + + Args: + cache_name: Name to use for the cache. Defaults to None. + cache_dir: Root directory to store the cache in. Defaults to None. + cleanup: Whether to clean up temporary files. Defaults to True. + + Cache files are stored in the `.cache` directory, relative to the current + working directory. + """ + if cache_name: + if " " in cache_name: + raise exc.AirbyteLibInputError( + message="Cache name cannot contain spaces.", + input_value=cache_name, + ) + + if not cache_name.replace("_", "").isalnum(): + raise exc.AirbyteLibInputError( + message="Cache name can only contain alphanumeric characters and underscores.", + input_value=cache_name, + ) + + cache_name = cache_name or str(ulid.ULID()) + cache_dir = cache_dir or Path(f"./.cache/{cache_name}") + if not isinstance(cache_dir, Path): + cache_dir = Path(cache_dir) + + config = DuckDBCacheConfig( + db_path=cache_dir / f"db_{cache_name}.duckdb", + cache_dir=cache_dir, + cleanup=cleanup, + ) + return DuckDBCache(config=config) diff --git a/airbyte-lib/airbyte_lib/_factories/connector_factories.py b/airbyte-lib/airbyte_lib/_factories/connector_factories.py new file mode 100644 index 000000000000..5d0c516ec7d1 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_factories/connector_factories.py @@ -0,0 +1,120 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import shutil +import warnings +from pathlib import Path +from typing import Any + +from airbyte_lib import exceptions as exc +from airbyte_lib._executor import PathExecutor, VenvExecutor +from airbyte_lib.registry import ConnectorMetadata, get_connector_metadata +from airbyte_lib.source import Source + + +def get_connector( + name: str, + config: dict[str, Any] | None = None, + *, + version: str | None = None, + pip_url: str | None = None, + local_executable: Path | str | None = None, + install_if_missing: bool = True, +) -> Source: + """Deprecated. Use get_source instead.""" + warnings.warn( + "The `get_connector()` function is deprecated and will be removed in a future version." + "Please use `get_source()` instead.", + DeprecationWarning, + stacklevel=2, + ) + return get_source( + name=name, + config=config, + version=version, + pip_url=pip_url, + local_executable=local_executable, + install_if_missing=install_if_missing, + ) + + +def get_source( + name: str, + config: dict[str, Any] | None = None, + *, + version: str | None = None, + pip_url: str | None = None, + local_executable: Path | str | None = None, + install_if_missing: bool = True, +) -> Source: + """Get a connector by name and version. + + Args: + name: connector name + config: connector config - if not provided, you need to set it later via the set_config + method. + version: connector version - if not provided, the currently installed version will be used. + If no version is installed, the latest available version will be used. The version can + also be set to "latest" to force the use of the latest available version. + pip_url: connector pip URL - if not provided, the pip url will be inferred from the + connector name. + local_executable: If set, the connector will be assumed to already be installed and will be + executed using this path or executable name. Otherwise, the connector will be installed + automatically in a virtual environment. + install_if_missing: Whether to install the connector if it is not available locally. This + parameter is ignored when local_executable is set. + """ + if local_executable: + if pip_url: + raise exc.AirbyteLibInputError( + message="Param 'pip_url' is not supported when 'local_executable' is set." + ) + if version: + raise exc.AirbyteLibInputError( + message="Param 'version' is not supported when 'local_executable' is set." + ) + + if isinstance(local_executable, str): + if "/" in local_executable or "\\" in local_executable: + # Assume this is a path + local_executable = Path(local_executable).absolute() + else: + which_executable = shutil.which(local_executable) + if which_executable is None: + raise FileNotFoundError(local_executable) + local_executable = Path(which_executable).absolute() + + print(f"Using local `{name}` executable: {local_executable!s}") + return Source( + name=name, + config=config, + executor=PathExecutor( + name=name, + path=local_executable, + ), + ) + + # else: we are installing a connector in a virtual environment: + + metadata: ConnectorMetadata | None = None + try: + metadata = get_connector_metadata(name) + except exc.AirbyteConnectorNotRegisteredError: + if not pip_url: + # We don't have a pip url or registry entry, so we can't install the connector + raise + + executor = VenvExecutor( + name=name, + metadata=metadata, + target_version=version, + pip_url=pip_url, + ) + if install_if_missing: + executor.ensure_installation() + + return Source( + executor=executor, + name=name, + config=config, + ) diff --git a/airbyte-lib/airbyte_lib/_file_writers/__init__.py b/airbyte-lib/airbyte_lib/_file_writers/__init__.py new file mode 100644 index 000000000000..aae8c474ca97 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/__init__.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from .base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase +from .parquet import ParquetWriter, ParquetWriterConfig + + +__all__ = [ + "FileWriterBatchHandle", + "FileWriterBase", + "FileWriterConfigBase", + "ParquetWriter", + "ParquetWriterConfig", +] diff --git a/airbyte-lib/airbyte_lib/_file_writers/base.py b/airbyte-lib/airbyte_lib/_file_writers/base.py new file mode 100644 index 000000000000..e037c567e7c8 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/base.py @@ -0,0 +1,126 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define abstract base class for File Writers, which write and read from file storage.""" + +from __future__ import annotations + +import abc +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, cast, final + +from overrides import overrides + +from airbyte_lib._processors import BatchHandle, RecordProcessor +from airbyte_lib.config import CacheConfigBase + + +if TYPE_CHECKING: + import pyarrow as pa + + from airbyte_protocol.models import ( + AirbyteStateMessage, + ) + + +DEFAULT_BATCH_SIZE = 10000 + + +# The batch handle for file writers is a list of Path objects. +@dataclass +class FileWriterBatchHandle(BatchHandle): + """The file writer batch handle is a list of Path objects.""" + + files: list[Path] = field(default_factory=list) + + +class FileWriterConfigBase(CacheConfigBase): + """Configuration for the Snowflake cache.""" + + cache_dir: Path = Path("./.cache/files/") + """The directory to store cache files in.""" + cleanup: bool = True + """Whether to clean up temporary files after processing a batch.""" + + +class FileWriterBase(RecordProcessor, abc.ABC): + """A generic base implementation for a file-based cache.""" + + config_class = FileWriterConfigBase + config: FileWriterConfigBase + + @abc.abstractmethod + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return a list of paths to one or more cache files. + """ + ... + + @final + def write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table, + ) -> FileWriterBatchHandle: + """Write a batch of records to the cache. + + This method is final because it should not be overridden. + + Subclasses should override `_write_batch` instead. + """ + return self._write_batch(stream_name, batch_id, record_batch) + + @overrides + def _cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For file writers, this means deleting the files created and declared in the batch. + + This method is a no-op if the `cleanup` config option is set to False. + """ + if self.config.cleanup: + batch_handle = cast(FileWriterBatchHandle, batch_handle) + _ = stream_name, batch_id + for file_path in batch_handle.files: + file_path.unlink() + + @final + def cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For file writers, this means deleting the files created and declared in the batch. + + This method is final because it should not be overridden. + + Subclasses should override `_cleanup_batch` instead. + """ + self._cleanup_batch(stream_name, batch_id, batch_handle) + + @overrides + def _finalize_state_messages( + self, + stream_name: str, + state_messages: list[AirbyteStateMessage], + ) -> None: + """ + State messages are not used in file writers, so this method is a no-op. + """ + pass diff --git a/airbyte-lib/airbyte_lib/_file_writers/parquet.py b/airbyte-lib/airbyte_lib/_file_writers/parquet.py new file mode 100644 index 000000000000..bc7fbe9cd704 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/parquet.py @@ -0,0 +1,91 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Parquet cache implementation.""" +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import pyarrow as pa +import ulid +from overrides import overrides +from pyarrow import parquet + +from airbyte_lib import exceptions as exc +from airbyte_lib._file_writers.base import ( + FileWriterBase, + FileWriterBatchHandle, + FileWriterConfigBase, +) +from airbyte_lib._util.text_util import lower_case_set + + +class ParquetWriterConfig(FileWriterConfigBase): + """Configuration for the Snowflake cache.""" + + # Inherits `cache_dir` from base class + + +class ParquetWriter(FileWriterBase): + """A Parquet cache implementation.""" + + config_class = ParquetWriterConfig + + def get_new_cache_file_path( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> Path: + """Return a new cache file path for the given stream.""" + batch_id = batch_id or str(ulid.ULID()) + config: ParquetWriterConfig = cast(ParquetWriterConfig, self.config) + target_dir = Path(config.cache_dir) + target_dir.mkdir(parents=True, exist_ok=True) + return target_dir / f"{stream_name}_{batch_id}.parquet" + + def _get_missing_columns( + self, + stream_name: str, + record_batch: pa.Table, + ) -> list[str]: + """Return a list of columns that are missing in the batch. + + The comparison is based on a case-insensitive comparison of the column names. + """ + if not self._catalog_manager: + raise exc.AirbyteLibInternalError(message="Catalog manager should exist but does not.") + stream = self._catalog_manager.get_stream_config(stream_name) + stream_property_names = stream.stream.json_schema["properties"].keys() + return [ + col + for col in stream_property_names + if col.lower() not in lower_case_set(record_batch.schema.names) + ] + + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return the path to the cache file. + """ + _ = batch_id # unused + output_file_path = self.get_new_cache_file_path(stream_name) + + missing_columns = self._get_missing_columns(stream_name, record_batch) + if missing_columns: + # We need to append columns with the missing column name(s) and a null type + null_array = cast(pa.Array, pa.array([None] * len(record_batch), type=pa.null())) + for col in missing_columns: + record_batch = record_batch.append_column(col, null_array) + + with parquet.ParquetWriter(output_file_path, schema=record_batch.schema) as writer: + writer.write_table(record_batch) + + batch_handle = FileWriterBatchHandle() + batch_handle.files.append(output_file_path) + return batch_handle diff --git a/airbyte-lib/airbyte_lib/_processors.py b/airbyte-lib/airbyte_lib/_processors.py new file mode 100644 index 000000000000..e879d22214e0 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_processors.py @@ -0,0 +1,396 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define abstract base class for Processors, including Caches and File writers. + +Processors can all take input from STDIN or a stream of Airbyte messages. + +Caches will pass their input to the File Writer. They share a common base class so certain +abstractions like "write" and "finalize" can be handled in either layer, or both. +""" + +from __future__ import annotations + +import abc +import contextlib +import io +import sys +from collections import defaultdict +from typing import TYPE_CHECKING, Any, cast, final + +import pyarrow as pa +import ulid + +from airbyte_protocol.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStreamState, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + Type, +) + +from airbyte_lib import exceptions as exc +from airbyte_lib._util import protocol_util +from airbyte_lib.progress import progress +from airbyte_lib.strategies import WriteStrategy + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + from airbyte_lib.caches._catalog_manager import CatalogManager + from airbyte_lib.config import CacheConfigBase + + +DEFAULT_BATCH_SIZE = 10_000 +DEBUG_MODE = False # Set to True to enable additional debug logging. + + +class BatchHandle: + pass + + +class AirbyteMessageParsingError(Exception): + """Raised when an Airbyte message is invalid or cannot be parsed.""" + + +class RecordProcessor(abc.ABC): + """Abstract base class for classes which can process input records.""" + + config_class: type[CacheConfigBase] + skip_finalize_step: bool = False + _expected_streams: set[str] + + def __init__( + self, + config: CacheConfigBase | dict | None, + *, + catalog_manager: CatalogManager | None = None, + ) -> None: + if isinstance(config, dict): + config = self.config_class(**config) + + self.config = config or self.config_class() + if not isinstance(self.config, self.config_class): + err_msg = ( + f"Expected config class of type '{self.config_class.__name__}'. " + f"Instead found '{type(self.config).__name__}'." + ) + raise TypeError(err_msg) + + self.source_catalog: ConfiguredAirbyteCatalog | None = None + self._source_name: str | None = None + + self._pending_batches: dict[str, dict[str, Any]] = defaultdict(lambda: {}, {}) + self._finalized_batches: dict[str, dict[str, Any]] = defaultdict(lambda: {}, {}) + + self._pending_state_messages: dict[str, list[AirbyteStateMessage]] = defaultdict(list, {}) + self._finalized_state_messages: dict[ + str, + list[AirbyteStateMessage], + ] = defaultdict(list, {}) + + self._catalog_manager: CatalogManager | None = catalog_manager + self._setup() + + def register_source( + self, + source_name: str, + incoming_source_catalog: ConfiguredAirbyteCatalog, + stream_names: set[str], + ) -> None: + """Register the source name and catalog.""" + if not self._catalog_manager: + raise exc.AirbyteLibInternalError( + message="Catalog manager should exist but does not.", + ) + self._catalog_manager.register_source( + source_name, + incoming_source_catalog=incoming_source_catalog, + incoming_stream_names=stream_names, + ) + self._expected_streams = stream_names + + @property + def _streams_with_data(self) -> set[str]: + """Return a list of known streams.""" + return self._pending_batches.keys() | self._finalized_batches.keys() + + @final + def process_stdin( + self, + write_strategy: WriteStrategy = WriteStrategy.AUTO, + *, + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + """Process the input stream from stdin. + + Return a list of summaries for testing. + """ + input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8") + self.process_input_stream( + input_stream, write_strategy=write_strategy, max_batch_size=max_batch_size + ) + + @final + def _airbyte_messages_from_buffer( + self, + buffer: io.TextIOBase, + ) -> Iterator[AirbyteMessage]: + """Yield messages from a buffer.""" + yield from (AirbyteMessage.parse_raw(line) for line in buffer) + + @final + def process_input_stream( + self, + input_stream: io.TextIOBase, + write_strategy: WriteStrategy = WriteStrategy.AUTO, + *, + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + """Parse the input stream and process data in batches. + + Return a list of summaries for testing. + """ + messages = self._airbyte_messages_from_buffer(input_stream) + self.process_airbyte_messages( + messages, + write_strategy=write_strategy, + max_batch_size=max_batch_size, + ) + + @final + def process_airbyte_messages( + self, + messages: Iterable[AirbyteMessage], + write_strategy: WriteStrategy, + *, + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + """Process a stream of Airbyte messages.""" + if not isinstance(write_strategy, WriteStrategy): + raise exc.AirbyteInternalError( + message="Invalid `write_strategy` argument. Expected instance of WriteStrategy.", + context={"write_strategy": write_strategy}, + ) + + stream_batches: dict[str, list[dict]] = defaultdict(list, {}) + + # Process messages, writing to batches as we go + for message in messages: + if message.type is Type.RECORD: + record_msg = cast(AirbyteRecordMessage, message.record) + stream_name = record_msg.stream + stream_batch = stream_batches[stream_name] + stream_batch.append(protocol_util.airbyte_record_message_to_dict(record_msg)) + + if len(stream_batch) >= max_batch_size: + record_batch = pa.Table.from_pylist(stream_batch) + self._process_batch(stream_name, record_batch) + progress.log_batch_written(stream_name, len(stream_batch)) + stream_batch.clear() + + elif message.type is Type.STATE: + state_msg = cast(AirbyteStateMessage, message.state) + if state_msg.type in [AirbyteStateType.GLOBAL, AirbyteStateType.LEGACY]: + self._pending_state_messages[f"_{state_msg.type}"].append(state_msg) + else: + stream_state = cast(AirbyteStreamState, state_msg.stream) + stream_name = stream_state.stream_descriptor.name + self._pending_state_messages[stream_name].append(state_msg) + + else: + # Ignore unexpected or unhandled message types: + # Type.LOG, Type.TRACE, Type.CONTROL, etc. + pass + + # Add empty streams to the dictionary, so we create a destination table for it + for stream_name in self._expected_streams: + if stream_name not in stream_batches: + if DEBUG_MODE: + print(f"Stream {stream_name} has no data") + stream_batches[stream_name] = [] + + # We are at the end of the stream. Process whatever else is queued. + for stream_name, stream_batch in stream_batches.items(): + record_batch = pa.Table.from_pylist(stream_batch) + self._process_batch(stream_name, record_batch) + progress.log_batch_written(stream_name, len(stream_batch)) + + # Finalize any pending batches + for stream_name in list(self._pending_batches.keys()): + self._finalize_batches(stream_name, write_strategy=write_strategy) + progress.log_stream_finalized(stream_name) + + @final + def _process_batch( + self, + stream_name: str, + record_batch: pa.Table, + ) -> tuple[str, Any, Exception | None]: + """Process a single batch. + + Returns a tuple of the batch ID, batch handle, and an exception if one occurred. + """ + batch_id = self._new_batch_id() + batch_handle = self._write_batch( + stream_name, + batch_id, + record_batch, + ) or self._get_batch_handle(stream_name, batch_id) + + if self.skip_finalize_step: + self._finalized_batches[stream_name][batch_id] = batch_handle + else: + self._pending_batches[stream_name][batch_id] = batch_handle + + return batch_id, batch_handle, None + + @abc.abstractmethod + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table, + ) -> BatchHandle: + """Process a single batch. + + Returns a batch handle, such as a path or any other custom reference. + """ + + def _cleanup_batch( # noqa: B027 # Intentionally empty, not abstract + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + This method is called after the given batch has been finalized. + + For instance, file writers can override this method to delete the files created. Caches, + similarly, can override this method to delete any other temporary artifacts. + """ + pass + + def _new_batch_id(self) -> str: + """Return a new batch handle.""" + return str(ulid.ULID()) + + def _get_batch_handle( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> str: + """Return a new batch handle. + + By default this is a concatenation of the stream name and batch ID. + However, any Python object can be returned, such as a Path object. + """ + batch_id = batch_id or self._new_batch_id() + return f"{stream_name}_{batch_id}" + + def _finalize_batches( + self, + stream_name: str, + write_strategy: WriteStrategy, + ) -> dict[str, BatchHandle]: + """Finalize all uncommitted batches. + + Returns a mapping of batch IDs to batch handles, for processed batches. + + This is a generic implementation, which can be overridden. + """ + _ = write_strategy # Unused + with self._finalizing_batches(stream_name) as batches_to_finalize: + if batches_to_finalize and not self.skip_finalize_step: + raise NotImplementedError( + "Caches need to be finalized but no _finalize_batch() method " + f"exists for class {self.__class__.__name__}", + ) + + return batches_to_finalize + + @abc.abstractmethod + def _finalize_state_messages( + self, + stream_name: str, + state_messages: list[AirbyteStateMessage], + ) -> None: + """Handle state messages. + Might be a no-op if the processor doesn't handle incremental state.""" + pass + + @final + @contextlib.contextmanager + def _finalizing_batches( + self, + stream_name: str, + ) -> Generator[dict[str, BatchHandle], str, None]: + """Context manager to use for finalizing batches, if applicable. + + Returns a mapping of batch IDs to batch handles, for those processed batches. + """ + batches_to_finalize = self._pending_batches[stream_name].copy() + state_messages_to_finalize = self._pending_state_messages[stream_name].copy() + self._pending_batches[stream_name].clear() + self._pending_state_messages[stream_name].clear() + + progress.log_batches_finalizing(stream_name, len(batches_to_finalize)) + yield batches_to_finalize + self._finalize_state_messages(stream_name, state_messages_to_finalize) + progress.log_batches_finalized(stream_name, len(batches_to_finalize)) + + self._finalized_batches[stream_name].update(batches_to_finalize) + self._finalized_state_messages[stream_name] += state_messages_to_finalize + + for batch_id, batch_handle in batches_to_finalize.items(): + self._cleanup_batch(stream_name, batch_id, batch_handle) + + def _setup(self) -> None: # noqa: B027 # Intentionally empty, not abstract + """Create the database. + + By default this is a no-op but subclasses can override this method to prepare + any necessary resources. + """ + + def _teardown(self) -> None: + """Teardown the processor resources. + + By default, the base implementation simply calls _cleanup_batch() for all pending batches. + """ + for stream_name, pending_batches in self._pending_batches.items(): + for batch_id, batch_handle in pending_batches.items(): + self._cleanup_batch( + stream_name=stream_name, + batch_id=batch_id, + batch_handle=batch_handle, + ) + + @final + def __del__(self) -> None: + """Teardown temporary resources when instance is unloaded from memory.""" + self._teardown() + + @final + def _get_stream_config( + self, + stream_name: str, + ) -> ConfiguredAirbyteStream: + """Return the column definitions for the given stream.""" + if not self._catalog_manager: + raise exc.AirbyteLibInternalError( + message="Catalog manager should exist but does not.", + ) + + return self._catalog_manager.get_stream_config(stream_name) + + @final + def _get_stream_json_schema( + self, + stream_name: str, + ) -> dict[str, Any]: + """Return the column definitions for the given stream.""" + return self._get_stream_config(stream_name).stream.json_schema diff --git a/airbyte-lib/airbyte_lib/_util/__init__.py b/airbyte-lib/airbyte_lib/_util/__init__.py new file mode 100644 index 000000000000..1073e4feb232 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_util/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Internal utility functions for dealing with pip. + +Note: This module is for internal use only and it should not be depended upon for production use. +It is subject to change without notice. +""" +from __future__ import annotations + +from airbyte_lib._util.pip_util import connector_pip_url, github_pip_url + + +__all__ = [ + "connector_pip_url", + "github_pip_url", +] diff --git a/airbyte-lib/airbyte_lib/_util/pip_util.py b/airbyte-lib/airbyte_lib/_util/pip_util.py new file mode 100644 index 000000000000..b965c52f9392 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_util/pip_util.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Internal utility functions for dealing with pip.""" + +from __future__ import annotations + + +def github_pip_url( + owner: str = "airbytehq", + repo: str = "airbyte", + *, + package_name: str, + branch_or_ref: str | None = None, + subdirectory: str | None = None, +) -> str: + """Return the pip URL for a GitHub repository. + + Results will look like: + - `git+airbytehq/airbyte.git#egg=airbyte-lib&subdirectory=airbyte-lib' + - `git+airbytehq/airbyte.git@master#egg=airbyte-lib&subdirectory=airbyte-lib' + - `git+airbytehq/airbyte.git@my-branch#egg=source-github + &subdirectory=airbyte-integrations/connectors/source-github' + """ + result = f"git+https://github.com/{owner}/{repo}.git" + + if branch_or_ref: + result += f"@{branch_or_ref}" + + next_delimiter = "#" + if package_name: + result += f"{next_delimiter}egg={package_name}" + next_delimiter = "&" + + if subdirectory: + result += f"{next_delimiter}subdirectory={subdirectory}" + + return result + + +def connector_pip_url( + connector_name: str, + /, + branch: str, + *, + owner: str | None = None, +) -> str: + """Return a pip URL for a connector in the main `airbytehq/airbyte` git repo.""" + owner = owner or "airbytehq" + if not connector_name.startswith("source-") and not connector_name.startswith("destination-"): + connector_name = "source-" + connector_name + + return github_pip_url( + owner=owner, + repo="airbyte", + branch_or_ref=branch, + package_name=connector_name, + subdirectory=f"airbyte-integrations/connectors/{connector_name}", + ) diff --git a/airbyte-lib/airbyte_lib/_util/protocol_util.py b/airbyte-lib/airbyte_lib/_util/protocol_util.py new file mode 100644 index 000000000000..2ddaa1346e30 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_util/protocol_util.py @@ -0,0 +1,77 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Internal utility functions, especially for dealing with Airbyte Protocol.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast + +from airbyte_protocol.models import ( + AirbyteMessage, + AirbyteRecordMessage, + ConfiguredAirbyteCatalog, + Type, +) + +from airbyte_lib import exceptions as exc + + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + +def airbyte_messages_to_record_dicts( + messages: Iterable[AirbyteMessage], +) -> Iterator[dict[str, Any]]: + """Convert an AirbyteMessage to a dictionary.""" + yield from ( + cast(dict[str, Any], airbyte_message_to_record_dict(message)) + for message in messages + if message is not None and message.type == Type.RECORD + ) + + +def airbyte_message_to_record_dict(message: AirbyteMessage) -> dict[str, Any] | None: + """Convert an AirbyteMessage to a dictionary. + + Return None if the message is not a record message. + """ + if message.type != Type.RECORD: + return None + + return airbyte_record_message_to_dict(message.record) + + +def airbyte_record_message_to_dict( + record_message: AirbyteRecordMessage, +) -> dict[str, Any]: + """Convert an AirbyteMessage to a dictionary. + + Return None if the message is not a record message. + """ + result = record_message.data + + # TODO: Add the metadata columns (this breaks tests) + # result["_airbyte_extracted_at"] = datetime.datetime.fromtimestamp( + # record_message.emitted_at + # ) + + return result # noqa: RET504 # unnecessary assignment and then return (see TODO above) + + +def get_primary_keys_from_stream( + stream_name: str, + configured_catalog: ConfiguredAirbyteCatalog, +) -> set[str]: + """Get the primary keys from a stream in the configured catalog.""" + stream = next( + (stream for stream in configured_catalog.streams if stream.stream.name == stream_name), + None, + ) + if stream is None: + raise exc.AirbyteStreamNotFoundError( + stream_name=stream_name, + connector_name=configured_catalog.connection.configuration["name"], + available_streams=[stream.stream.name for stream in configured_catalog.streams], + ) + + return set(stream.stream.source_defined_primary_key or []) diff --git a/airbyte-lib/airbyte_lib/_util/text_util.py b/airbyte-lib/airbyte_lib/_util/text_util.py new file mode 100644 index 000000000000..d5f890993868 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_util/text_util.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Internal utility functions for dealing with text.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def lower_case_set(str_iter: Iterable[str]) -> set[str]: + """Converts a list of strings to a set of lower case strings.""" + return {s.lower() for s in str_iter} diff --git a/airbyte-lib/airbyte_lib/caches/__init__.py b/airbyte-lib/airbyte_lib/caches/__init__.py new file mode 100644 index 000000000000..3cb5c31cf119 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/__init__.py @@ -0,0 +1,19 @@ +"""Base module for all caches.""" +from __future__ import annotations + +from airbyte_lib.caches.base import SQLCacheBase +from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig +from airbyte_lib.caches.postgres import PostgresCache, PostgresCacheConfig +from airbyte_lib.caches.snowflake import SnowflakeCacheConfig, SnowflakeSQLCache + + +# We export these classes for easy access: `airbyte_lib.caches...` +__all__ = [ + "DuckDBCache", + "DuckDBCacheConfig", + "PostgresCache", + "PostgresCacheConfig", + "SQLCacheBase", + "SnowflakeCacheConfig", + "SnowflakeSQLCache", +] diff --git a/airbyte-lib/airbyte_lib/caches/_catalog_manager.py b/airbyte-lib/airbyte_lib/caches/_catalog_manager.py new file mode 100644 index 000000000000..3eb94e148f63 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/_catalog_manager.py @@ -0,0 +1,285 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A SQL Cache implementation.""" +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Callable + +from sqlalchemy import Column, DateTime, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import Session +from sqlalchemy.sql import func + +from airbyte_protocol.models import ( + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) + +from airbyte_lib import exceptions as exc + + +if TYPE_CHECKING: + from sqlalchemy.engine import Engine + +STREAMS_TABLE_NAME = "_airbytelib_streams" +STATE_TABLE_NAME = "_airbytelib_state" + +GLOBAL_STATE_STREAM_NAMES = ["_GLOBAL", "_LEGACY"] + +Base = declarative_base() + + +class CachedStream(Base): # type: ignore[valid-type,misc] + __tablename__ = STREAMS_TABLE_NAME + + stream_name = Column(String) + source_name = Column(String) + table_name = Column(String, primary_key=True) + catalog_metadata = Column(String) + + +class StreamState(Base): # type: ignore[valid-type,misc] + __tablename__ = STATE_TABLE_NAME + + source_name = Column(String) + stream_name = Column(String) + table_name = Column(String, primary_key=True) + state_json = Column(String) + last_updated = Column(DateTime(timezone=True), onupdate=func.now(), default=func.now()) + + +class CatalogManager: + """ + A class to manage the stream catalog of data synced to a cache: + * What streams exist and to what tables they map + * The JSON schema for each stream + * The state of each stream if available + """ + + def __init__( + self, + engine: Engine, + table_name_resolver: Callable[[str], str], + ) -> None: + self._engine: Engine = engine + self._table_name_resolver = table_name_resolver + self._source_catalog: ConfiguredAirbyteCatalog | None = None + self._load_catalog_from_internal_table() + assert self._source_catalog is not None + + @property + def source_catalog(self) -> ConfiguredAirbyteCatalog: + """Return the source catalog. + + Raises: + AirbyteLibInternalError: If the source catalog is not set. + """ + if not self._source_catalog: + raise exc.AirbyteLibInternalError( + message="Source catalog should be initialized but is not.", + ) + + return self._source_catalog + + def _ensure_internal_tables(self) -> None: + engine = self._engine + Base.metadata.create_all(engine) + + def save_state( + self, + source_name: str, + state: AirbyteStateMessage, + stream_name: str, + ) -> None: + self._ensure_internal_tables() + engine = self._engine + with Session(engine) as session: + session.query(StreamState).filter( + StreamState.table_name == self._table_name_resolver(stream_name) + ).delete() + session.commit() + session.add( + StreamState( + source_name=source_name, + stream_name=stream_name, + table_name=self._table_name_resolver(stream_name), + state_json=state.json(), + ) + ) + session.commit() + + def get_state( + self, + source_name: str, + streams: list[str], + ) -> list[dict] | None: + self._ensure_internal_tables() + engine = self._engine + with Session(engine) as session: + states = ( + session.query(StreamState) + .filter( + StreamState.source_name == source_name, + StreamState.stream_name.in_([*streams, *GLOBAL_STATE_STREAM_NAMES]), + ) + .all() + ) + if not states: + return None + # Only return the states if the table name matches what the current cache + # would generate. Otherwise consider it part of a different cache. + states = [ + state + for state in states + if state.table_name == self._table_name_resolver(state.stream_name) + ] + return [json.loads(state.state_json) for state in states] + + def register_source( + self, + source_name: str, + incoming_source_catalog: ConfiguredAirbyteCatalog, + incoming_stream_names: set[str], + ) -> None: + """Register a source and its streams in the cache.""" + self._update_catalog( + incoming_source_catalog=incoming_source_catalog, + incoming_stream_names=incoming_stream_names, + ) + self._save_catalog_to_internal_table( + source_name=source_name, + incoming_source_catalog=incoming_source_catalog, + incoming_stream_names=incoming_stream_names, + ) + + def _update_catalog( + self, + incoming_source_catalog: ConfiguredAirbyteCatalog, + incoming_stream_names: set[str], + ) -> None: + if not self._source_catalog: + self._source_catalog = ConfiguredAirbyteCatalog( + streams=[ + stream + for stream in incoming_source_catalog.streams + if stream.stream.name in incoming_stream_names + ], + ) + assert len(self._source_catalog.streams) == len(incoming_stream_names) + return + + # Keep existing streams untouched if not incoming + unchanged_streams: list[ConfiguredAirbyteStream] = [ + stream + for stream in self._source_catalog.streams + if stream.stream.name not in incoming_stream_names + ] + new_streams: list[ConfiguredAirbyteStream] = [ + stream + for stream in incoming_source_catalog.streams + if stream.stream.name in incoming_stream_names + ] + self._source_catalog = ConfiguredAirbyteCatalog(streams=unchanged_streams + new_streams) + + def _save_catalog_to_internal_table( + self, + source_name: str, + incoming_source_catalog: ConfiguredAirbyteCatalog, + incoming_stream_names: set[str], + ) -> None: + self._ensure_internal_tables() + engine = self._engine + with Session(engine) as session: + # Delete and replace existing stream entries from the catalog cache + table_name_entries_to_delete = [ + self._table_name_resolver(incoming_stream_name) + for incoming_stream_name in incoming_stream_names + ] + result = ( + session.query(CachedStream) + .filter(CachedStream.table_name.in_(table_name_entries_to_delete)) + .delete() + ) + _ = result + session.commit() + insert_streams = [ + CachedStream( + source_name=source_name, + stream_name=stream.stream.name, + table_name=self._table_name_resolver(stream.stream.name), + catalog_metadata=json.dumps(stream.stream.json_schema), + ) + for stream in incoming_source_catalog.streams + ] + session.add_all(insert_streams) + session.commit() + + def get_stream_config( + self, + stream_name: str, + ) -> ConfiguredAirbyteStream: + """Return the column definitions for the given stream.""" + if not self.source_catalog: + raise exc.AirbyteLibInternalError( + message="Cannot get stream JSON schema without a catalog.", + ) + + matching_streams: list[ConfiguredAirbyteStream] = [ + stream for stream in self.source_catalog.streams if stream.stream.name == stream_name + ] + if not matching_streams: + raise exc.AirbyteStreamNotFoundError( + stream_name=stream_name, + context={ + "available_streams": [ + stream.stream.name for stream in self.source_catalog.streams + ], + }, + ) + + if len(matching_streams) > 1: + raise exc.AirbyteLibInternalError( + message="Multiple streams found with same name.", + context={ + "stream_name": stream_name, + }, + ) + + return matching_streams[0] + + def _load_catalog_from_internal_table(self) -> None: + self._ensure_internal_tables() + engine = self._engine + with Session(engine) as session: + # load all the streams + streams: list[CachedStream] = session.query(CachedStream).all() + if not streams: + # no streams means the cache is pristine + if not self._source_catalog: + self._source_catalog = ConfiguredAirbyteCatalog(streams=[]) + + return + + # load the catalog + self._source_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name=stream.stream_name, + json_schema=json.loads(stream.catalog_metadata), + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + for stream in streams + # only load the streams where the table name matches what + # the current cache would generate + if stream.table_name == self._table_name_resolver(stream.stream_name) + ] + ) diff --git a/airbyte-lib/airbyte_lib/caches/base.py b/airbyte-lib/airbyte_lib/caches/base.py new file mode 100644 index 000000000000..b5ee35e680ac --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/base.py @@ -0,0 +1,986 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A SQL Cache implementation.""" +from __future__ import annotations + +import abc +import enum +from contextlib import contextmanager +from functools import cached_property +from typing import TYPE_CHECKING, cast, final + +import pandas as pd +import pyarrow as pa +import sqlalchemy +import ulid +from overrides import overrides +from sqlalchemy import ( + Column, + Table, + and_, + create_engine, + insert, + null, + select, + text, + update, +) +from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.elements import TextClause + +from airbyte_lib import exceptions as exc +from airbyte_lib._file_writers.base import FileWriterBase, FileWriterBatchHandle +from airbyte_lib._processors import BatchHandle, RecordProcessor +from airbyte_lib._util.text_util import lower_case_set +from airbyte_lib.caches._catalog_manager import CatalogManager +from airbyte_lib.config import CacheConfigBase +from airbyte_lib.datasets._sql import CachedDataset +from airbyte_lib.strategies import WriteStrategy +from airbyte_lib.types import SQLTypeConverter + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterator + from pathlib import Path + + from sqlalchemy.engine import Connection, Engine + from sqlalchemy.engine.cursor import CursorResult + from sqlalchemy.engine.reflection import Inspector + from sqlalchemy.sql.base import Executable + + from airbyte_protocol.models import ( + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ) + + from airbyte_lib.datasets._base import DatasetBase + from airbyte_lib.telemetry import CacheTelemetryInfo + + +DEBUG_MODE = False # Set to True to enable additional debug logging. + + +class RecordDedupeMode(enum.Enum): + APPEND = "append" + REPLACE = "replace" + + +class SQLRuntimeError(Exception): + """Raised when an SQL operation fails.""" + + +class SQLCacheConfigBase(CacheConfigBase): + """Same as a regular config except it exposes the 'get_sql_alchemy_url()' method.""" + + schema_name: str = "airbyte_raw" + + table_prefix: str | None = None + """ A prefix to add to all table names. + If 'None', a prefix will be created based on the source name. + """ + + table_suffix: str = "" + """A suffix to add to all table names.""" + + @abc.abstractmethod + def get_sql_alchemy_url(self) -> str: + """Returns a SQL Alchemy URL.""" + ... + + @abc.abstractmethod + def get_database_name(self) -> str: + """Return the name of the database.""" + ... + + +class GenericSQLCacheConfig(SQLCacheConfigBase): + """Allows configuring 'sql_alchemy_url' directly.""" + + sql_alchemy_url: str + + @overrides + def get_sql_alchemy_url(self) -> str: + """Returns a SQL Alchemy URL.""" + return self.sql_alchemy_url + + +class SQLCacheBase(RecordProcessor): + """A base class to be used for SQL Caches. + + Optionally we can use a file cache to store the data in parquet files. + """ + + type_converter_class: type[SQLTypeConverter] = SQLTypeConverter + config_class: type[SQLCacheConfigBase] + file_writer_class: type[FileWriterBase] + + supports_merge_insert = False + use_singleton_connection = False # If true, the same connection is used for all operations. + + # Constructor: + + @final # We don't want subclasses to have to override the constructor. + def __init__( + self, + config: SQLCacheConfigBase | None = None, + file_writer: FileWriterBase | None = None, + ) -> None: + self.config: SQLCacheConfigBase + self._engine: Engine | None = None + self._connection_to_reuse: Connection | None = None + super().__init__(config) + self._ensure_schema_exists() + self._catalog_manager = CatalogManager( + engine=self.get_sql_engine(), + table_name_resolver=lambda stream_name: self.get_sql_table_name(stream_name), + ) + self.file_writer = file_writer or self.file_writer_class( + config, catalog_manager=self._catalog_manager + ) + self.type_converter = self.type_converter_class() + self._cached_table_definitions: dict[str, sqlalchemy.Table] = {} + + def __getitem__(self, stream: str) -> DatasetBase: + return self.streams[stream] + + def __contains__(self, stream: str) -> bool: + return stream in self._streams_with_data + + def __iter__(self) -> Iterator[str]: + return iter(self._streams_with_data) + + # Public interface: + + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return self.config.get_sql_alchemy_url() + + @final + @cached_property + def database_name(self) -> str: + """Return the name of the database.""" + return self.config.get_database_name() + + @final + def get_sql_engine(self) -> Engine: + """Return a new SQL engine to use.""" + if self._engine: + return self._engine + + sql_alchemy_url = self.get_sql_alchemy_url() + + execution_options = {"schema_translate_map": {None: self.config.schema_name}} + if self.use_singleton_connection: + if self._connection_to_reuse is None: + # This temporary bootstrap engine will be created once and is needed to + # create the long-lived connection object. + bootstrap_engine = create_engine( + sql_alchemy_url, + ) + self._connection_to_reuse = bootstrap_engine.connect() + + self._engine = create_engine( + sql_alchemy_url, + creator=lambda: self._connection_to_reuse, + poolclass=StaticPool, + echo=DEBUG_MODE, + execution_options=execution_options, + # isolation_level="AUTOCOMMIT", + ) + else: + # Regular engine creation for new connections + self._engine = create_engine( + sql_alchemy_url, + echo=DEBUG_MODE, + execution_options=execution_options, + # isolation_level="AUTOCOMMIT", + ) + + return self._engine + + def _init_connection_settings(self, connection: Connection) -> None: + """This is called automatically whenever a new connection is created. + + By default this is a no-op. Subclasses can use this to set connection settings, such as + timezone, case-sensitivity settings, and other session-level variables. + """ + pass + + @contextmanager + def get_sql_connection(self) -> Generator[sqlalchemy.engine.Connection, None, None]: + """A context manager which returns a new SQL connection for running queries. + + If the connection needs to close, it will be closed automatically. + """ + if self.use_singleton_connection and self._connection_to_reuse is not None: + connection = self._connection_to_reuse + self._init_connection_settings(connection) + yield connection + + else: + with self.get_sql_engine().begin() as connection: + self._init_connection_settings(connection) + yield connection + + if not self.use_singleton_connection: + connection.close() + del connection + + def get_sql_table_name( + self, + stream_name: str, + ) -> str: + """Return the name of the SQL table for the given stream.""" + table_prefix = self.config.table_prefix or "" + + # TODO: Add default prefix based on the source name. + + return self._normalize_table_name( + f"{table_prefix}{stream_name}{self.config.table_suffix}", + ) + + @final + def get_sql_table( + self, + stream_name: str, + ) -> sqlalchemy.Table: + """Return the main table object for the stream.""" + return self._get_table_by_name(self.get_sql_table_name(stream_name)) + + def _get_table_by_name( + self, + table_name: str, + *, + force_refresh: bool = False, + ) -> sqlalchemy.Table: + """Return a table object from a table name. + + To prevent unnecessary round-trips to the database, the table is cached after the first + query. To ignore the cache and force a refresh, set 'force_refresh' to True. + """ + if force_refresh or table_name not in self._cached_table_definitions: + self._cached_table_definitions[table_name] = sqlalchemy.Table( + table_name, + sqlalchemy.MetaData(schema=self.config.schema_name), + autoload_with=self.get_sql_engine(), + ) + + return self._cached_table_definitions[table_name] + + @final + @property + def streams( + self, + ) -> dict[str, CachedDataset]: + """Return a temporary table name.""" + result = {} + for stream_name in self._streams_with_data: + result[stream_name] = CachedDataset(self, stream_name) + + return result + + # Read methods: + + def get_records( + self, + stream_name: str, + ) -> CachedDataset: + """Uses SQLAlchemy to select all rows from the table.""" + return CachedDataset(self, stream_name) + + def get_pandas_dataframe( + self, + stream_name: str, + ) -> pd.DataFrame: + """Return a Pandas data frame with the stream's data.""" + table_name = self.get_sql_table_name(stream_name) + engine = self.get_sql_engine() + return pd.read_sql_table(table_name, engine) + + # Protected members (non-public interface): + + def _ensure_schema_exists( + self, + ) -> None: + """Return a new (unique) temporary table name.""" + schema_name = self.config.schema_name + if schema_name in self._get_schemas_list(): + return + + sql = f"CREATE SCHEMA IF NOT EXISTS {schema_name}" + + try: + self._execute_sql(sql) + except Exception as ex: + # Ignore schema exists errors. + if "already exists" not in str(ex): + raise + + if DEBUG_MODE: + found_schemas = self._get_schemas_list() + assert ( + schema_name in found_schemas + ), f"Schema {schema_name} was not created. Found: {found_schemas}" + + def _quote_identifier(self, identifier: str) -> str: + """Return the given identifier, quoted.""" + return f'"{identifier}"' + + @final + def _get_temp_table_name( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> str: + """Return a new (unique) temporary table name.""" + batch_id = batch_id or str(ulid.ULID()) + return self._normalize_table_name(f"{stream_name}_{batch_id}") + + def _fully_qualified( + self, + table_name: str, + ) -> str: + """Return the fully qualified name of the given table.""" + return f"{self.config.schema_name}.{self._quote_identifier(table_name)}" + + @final + def _create_table_for_loading( + self, + /, + stream_name: str, + batch_id: str, + ) -> str: + """Create a new table for loading data.""" + temp_table_name = self._get_temp_table_name(stream_name, batch_id) + column_definition_str = ",\n ".join( + f"{self._quote_identifier(column_name)} {sql_type}" + for column_name, sql_type in self._get_sql_column_definitions(stream_name).items() + ) + self._create_table(temp_table_name, column_definition_str) + + return temp_table_name + + def _get_tables_list( + self, + ) -> list[str]: + """Return a list of all tables in the database.""" + with self.get_sql_connection() as conn: + inspector: Inspector = sqlalchemy.inspect(conn) + return inspector.get_table_names(schema=self.config.schema_name) + + def _get_schemas_list( + self, + database_name: str | None = None, + ) -> list[str]: + """Return a list of all tables in the database.""" + inspector: Inspector = sqlalchemy.inspect(self.get_sql_engine()) + database_name = database_name or self.database_name + found_schemas = inspector.get_schema_names() + return [ + found_schema.split(".")[-1].strip('"') + for found_schema in found_schemas + if "." not in found_schema + or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) + ] + + def _ensure_final_table_exists( + self, + stream_name: str, + *, + create_if_missing: bool = True, + ) -> str: + """Create the final table if it doesn't already exist. + + Return the table name. + """ + table_name = self.get_sql_table_name(stream_name) + did_exist = self._table_exists(table_name) + if not did_exist and create_if_missing: + column_definition_str = ",\n ".join( + f"{self._quote_identifier(column_name)} {sql_type}" + for column_name, sql_type in self._get_sql_column_definitions( + stream_name, + ).items() + ) + self._create_table(table_name, column_definition_str) + + return table_name + + def _ensure_compatible_table_schema( + self, + stream_name: str, + *, + raise_on_error: bool = False, + ) -> bool: + """Return true if the given table is compatible with the stream's schema. + + If raise_on_error is true, raise an exception if the table is not compatible. + + TODO: Expand this to check for column types and sizes, and to add missing columns. + + Returns true if the table is compatible, false if it is not. + """ + json_schema = self._get_stream_json_schema(stream_name) + stream_column_names: list[str] = json_schema["properties"].keys() + table_column_names: list[str] = self.get_sql_table(stream_name).columns.keys() + + lower_case_table_column_names = lower_case_set(table_column_names) + missing_columns = [ + stream_col + for stream_col in stream_column_names + if stream_col.lower() not in lower_case_table_column_names + ] + if missing_columns: + if raise_on_error: + raise exc.AirbyteLibCacheTableValidationError( + violation="Cache table is missing expected columns.", + context={ + "stream_column_names": stream_column_names, + "table_column_names": table_column_names, + "missing_columns": missing_columns, + }, + ) + return False # Some columns are missing. + + return True # All columns exist. + + @final + def _create_table( + self, + table_name: str, + column_definition_str: str, + primary_keys: list[str] | None = None, + ) -> None: + if DEBUG_MODE: + assert table_name not in self._get_tables_list(), f"Table {table_name} already exists." + + if primary_keys: + pk_str = ", ".join(primary_keys) + column_definition_str += f",\n PRIMARY KEY ({pk_str})" + + cmd = f""" + CREATE TABLE {self._fully_qualified(table_name)} ( + {column_definition_str} + ) + """ + _ = self._execute_sql(cmd) + if DEBUG_MODE: + tables_list = self._get_tables_list() + assert ( + table_name in tables_list + ), f"Table {table_name} was not created. Found: {tables_list}" + + def _normalize_column_name( + self, + raw_name: str, + ) -> str: + return raw_name.lower().replace(" ", "_").replace("-", "_") + + def _normalize_table_name( + self, + raw_name: str, + ) -> str: + return raw_name.lower().replace(" ", "_").replace("-", "_") + + @final + def _get_sql_column_definitions( + self, + stream_name: str, + ) -> dict[str, sqlalchemy.types.TypeEngine]: + """Return the column definitions for the given stream.""" + columns: dict[str, sqlalchemy.types.TypeEngine] = {} + properties = self._get_stream_json_schema(stream_name)["properties"] + for property_name, json_schema_property_def in properties.items(): + clean_prop_name = self._normalize_column_name(property_name) + columns[clean_prop_name] = self.type_converter.to_sql_type( + json_schema_property_def, + ) + + # TODO: Add the metadata columns (this breaks tests) + # columns["_airbyte_extracted_at"] = sqlalchemy.TIMESTAMP() + # columns["_airbyte_loaded_at"] = sqlalchemy.TIMESTAMP() + return columns + + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return the path to the cache file. + """ + return self.file_writer.write_batch(stream_name, batch_id, record_batch) + + def _cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For SQL caches, we only need to call the cleanup operation on the file writer. + + Subclasses should call super() if they override this method. + """ + self.file_writer.cleanup_batch(stream_name, batch_id, batch_handle) + + @final + @overrides + def _finalize_batches( + self, + stream_name: str, + write_strategy: WriteStrategy, + ) -> dict[str, BatchHandle]: + """Finalize all uncommitted batches. + + This is a generic 'final' implementation, which should not be overridden. + + Returns a mapping of batch IDs to batch handles, for those processed batches. + + TODO: Add a dedupe step here to remove duplicates from the temp table. + Some sources will send us duplicate records within the same stream, + although this is a fairly rare edge case we can ignore in V1. + """ + with self._finalizing_batches(stream_name) as batches_to_finalize: + if not batches_to_finalize: + return {} + + files: list[Path] = [] + # Get a list of all files to finalize from all pending batches. + for batch_handle in batches_to_finalize.values(): + batch_handle = cast(FileWriterBatchHandle, batch_handle) + files += batch_handle.files + # Use the max batch ID as the batch ID for table names. + max_batch_id = max(batches_to_finalize.keys()) + + # Make sure the target schema and target table exist. + self._ensure_schema_exists() + final_table_name = self._ensure_final_table_exists( + stream_name, + create_if_missing=True, + ) + self._ensure_compatible_table_schema( + stream_name=stream_name, + raise_on_error=True, + ) + + temp_table_name = self._write_files_to_new_table( + files=files, + stream_name=stream_name, + batch_id=max_batch_id, + ) + try: + self._write_temp_table_to_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + write_strategy=write_strategy, + ) + finally: + self._drop_temp_table(temp_table_name, if_exists=True) + + # Return the batch handles as measure of work completed. + return batches_to_finalize + + @overrides + def _finalize_state_messages( + self, + stream_name: str, + state_messages: list[AirbyteStateMessage], + ) -> None: + """Handle state messages by passing them to the catalog manager.""" + if not self._catalog_manager: + raise exc.AirbyteLibInternalError( + message="Catalog manager should exist but does not.", + ) + if state_messages and self._source_name: + self._catalog_manager.save_state( + source_name=self._source_name, + stream_name=stream_name, + state=state_messages[-1], + ) + + def get_state(self) -> list[dict]: + """Return the current state of the source.""" + if not self._source_name: + return [] + if not self._catalog_manager: + raise exc.AirbyteLibInternalError( + message="Catalog manager should exist but does not.", + ) + return ( + self._catalog_manager.get_state(self._source_name, list(self._streams_with_data)) or [] + ) + + def _execute_sql(self, sql: str | TextClause | Executable) -> CursorResult: + """Execute the given SQL statement.""" + if isinstance(sql, str): + sql = text(sql) + if isinstance(sql, TextClause): + sql = sql.execution_options( + autocommit=True, + ) + + with self.get_sql_connection() as conn: + try: + result = conn.execute(sql) + except ( + sqlalchemy.exc.ProgrammingError, + sqlalchemy.exc.SQLAlchemyError, + ) as ex: + msg = f"Error when executing SQL:\n{sql}\n{type(ex).__name__}{ex!s}" + raise SQLRuntimeError(msg) from None # from ex + + return result + + def _drop_temp_table( + self, + table_name: str, + *, + if_exists: bool = True, + ) -> None: + """Drop the given table.""" + exists_str = "IF EXISTS" if if_exists else "" + self._execute_sql(f"DROP TABLE {exists_str} {self._fully_qualified(table_name)}") + + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write a file(s) to a new table. + + This is a generic implementation, which can be overridden by subclasses + to improve performance. + """ + temp_table_name = self._create_table_for_loading(stream_name, batch_id) + for file_path in files: + with pa.parquet.ParquetFile(file_path) as pf: + record_batch = pf.read() + dataframe = record_batch.to_pandas() + + # Pandas will auto-create the table if it doesn't exist, which we don't want. + if not self._table_exists(temp_table_name): + raise exc.AirbyteLibInternalError( + message="Table does not exist after creation.", + context={ + "temp_table_name": temp_table_name, + }, + ) + + dataframe.to_sql( + temp_table_name, + self.get_sql_alchemy_url(), + schema=self.config.schema_name, + if_exists="append", + index=False, + dtype=self._get_sql_column_definitions(stream_name), + ) + return temp_table_name + + @final + def _write_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + write_strategy: WriteStrategy, + ) -> None: + """Write the temp table into the final table using the provided write strategy.""" + has_pks: bool = bool(self._get_primary_keys(stream_name)) + has_incremental_key: bool = bool(self._get_incremental_key(stream_name)) + if write_strategy == WriteStrategy.MERGE and not has_pks: + raise exc.AirbyteLibInputError( + message="Cannot use merge strategy on a stream with no primary keys.", + context={ + "stream_name": stream_name, + }, + ) + + if write_strategy == WriteStrategy.AUTO: + if has_pks: + write_strategy = WriteStrategy.MERGE + elif has_incremental_key: + write_strategy = WriteStrategy.APPEND + else: + write_strategy = WriteStrategy.REPLACE + + if write_strategy == WriteStrategy.REPLACE: + self._swap_temp_table_with_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + ) + return + + if write_strategy == WriteStrategy.APPEND: + self._append_temp_table_to_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + ) + return + + if write_strategy == WriteStrategy.MERGE: + if not self.supports_merge_insert: + # Fallback to emulated merge if the database does not support merge natively. + self._emulated_merge_temp_table_to_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + ) + return + + self._merge_temp_table_to_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + ) + return + + raise exc.AirbyteLibInternalError( + message="Write strategy is not supported.", + context={ + "write_strategy": write_strategy, + }, + ) + + def _append_temp_table_to_final_table( + self, + temp_table_name: str, + final_table_name: str, + stream_name: str, + ) -> None: + nl = "\n" + columns = [self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)] + self._execute_sql( + f""" + INSERT INTO {self._fully_qualified(final_table_name)} ( + {f',{nl} '.join(columns)} + ) + SELECT + {f',{nl} '.join(columns)} + FROM {self._fully_qualified(temp_table_name)} + """, + ) + + def _get_primary_keys( + self, + stream_name: str, + ) -> list[str]: + pks = self._get_stream_config(stream_name).primary_key + if not pks: + return [] + + joined_pks = [".".join(pk) for pk in pks] + for pk in joined_pks: + if "." in pk: + msg = "Nested primary keys are not yet supported. Found: {pk}" + raise NotImplementedError(msg) + + return joined_pks + + def _get_incremental_key( + self, + stream_name: str, + ) -> str | None: + return self._get_stream_config(stream_name).cursor_field + + def _swap_temp_table_with_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the main one. + + This implementation requires MERGE support in the SQL DB. + Databases that do not support this syntax can override this method. + """ + if final_table_name is None: + raise exc.AirbyteLibInternalError(message="Arg 'final_table_name' cannot be None.") + if temp_table_name is None: + raise exc.AirbyteLibInternalError(message="Arg 'temp_table_name' cannot be None.") + + _ = stream_name + deletion_name = f"{final_table_name}_deleteme" + commands = "\n".join( + [ + f"ALTER TABLE {final_table_name} RENAME TO {deletion_name};", + f"ALTER TABLE {temp_table_name} RENAME TO {final_table_name};", + f"DROP TABLE {deletion_name};", + ] + ) + self._execute_sql(commands) + + def _merge_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the main one. + + This implementation requires MERGE support in the SQL DB. + Databases that do not support this syntax can override this method. + """ + nl = "\n" + columns = {self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)} + pk_columns = {self._quote_identifier(c) for c in self._get_primary_keys(stream_name)} + non_pk_columns = columns - pk_columns + join_clause = "{nl} AND ".join(f"tmp.{pk_col} = final.{pk_col}" for pk_col in pk_columns) + set_clause = "{nl} ".join(f"{col} = tmp.{col}" for col in non_pk_columns) + self._execute_sql( + f""" + MERGE INTO {self._fully_qualified(final_table_name)} final + USING ( + SELECT * + FROM {self._fully_qualified(temp_table_name)} + ) AS tmp + ON {join_clause} + WHEN MATCHED THEN UPDATE + SET + {set_clause} + WHEN NOT MATCHED THEN INSERT + ( + {f',{nl} '.join(columns)} + ) + VALUES ( + tmp.{f',{nl} tmp.'.join(columns)} + ); + """, + ) + + def _get_column_by_name(self, table: str | Table, column_name: str) -> Column: + """Return the column object for the given column name. + + This method is case-insensitive. + """ + if isinstance(table, str): + table = self._get_table_by_name(table) + try: + # Try to get the column in a case-insensitive manner + return next(col for col in table.c if col.name.lower() == column_name.lower()) + except StopIteration: + raise exc.AirbyteLibInternalError( + message="Could not find matching column.", + context={ + "table": table, + "column_name": column_name, + }, + ) from None + + def _emulated_merge_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Emulate the merge operation using a series of SQL commands. + + This is a fallback implementation for databases that do not support MERGE. + """ + final_table = self._get_table_by_name(final_table_name) + temp_table = self._get_table_by_name(temp_table_name) + pk_columns = self._get_primary_keys(stream_name) + + columns_to_update: set[str] = self._get_sql_column_definitions( + stream_name=stream_name + ).keys() - set(pk_columns) + + # Create a dictionary mapping columns in users_final to users_stage for updating + update_values = { + self._get_column_by_name(final_table, column): ( + self._get_column_by_name(temp_table, column) + ) + for column in columns_to_update + } + + # Craft the WHERE clause for composite primary keys + join_conditions = [ + self._get_column_by_name(final_table, pk_column) + == self._get_column_by_name(temp_table, pk_column) + for pk_column in pk_columns + ] + join_clause = and_(*join_conditions) + + # Craft the UPDATE statement + update_stmt = update(final_table).values(update_values).where(join_clause) + + # Define a join between temp_table and final_table + joined_table = temp_table.outerjoin(final_table, join_clause) + + # Define a condition that checks for records in temp_table that do not have a corresponding + # record in final_table + where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null() + + # Select records from temp_table that are not in final_table + select_new_records_stmt = ( + select([temp_table]).select_from(joined_table).where(where_not_exists_clause) + ) + + # Craft the INSERT statement using the select statement + insert_new_records_stmt = insert(final_table).from_select( + names=[column.name for column in temp_table.columns], select=select_new_records_stmt + ) + + if DEBUG_MODE: + print(str(update_stmt)) + print(str(insert_new_records_stmt)) + + with self.get_sql_connection() as conn: + conn.execute(update_stmt) + conn.execute(insert_new_records_stmt) + + @final + def _table_exists( + self, + table_name: str, + ) -> bool: + """Return true if the given table exists.""" + return table_name in self._get_tables_list() + + @overrides + def register_source( + self, + source_name: str, + incoming_source_catalog: ConfiguredAirbyteCatalog, + stream_names: set[str], + ) -> None: + """Register the source with the cache. + + We use stream_names to determine which streams will receive data, and + we only register the stream if is expected to receive data. + + This method is called by the source when it is initialized. + """ + self._source_name = source_name + self._ensure_schema_exists() + super().register_source( + source_name, + incoming_source_catalog, + stream_names=stream_names, + ) + + @property + @overrides + def _streams_with_data(self) -> set[str]: + """Return a list of known streams.""" + if not self._catalog_manager: + raise exc.AirbyteLibInternalError( + message="Cannot get streams with data without a catalog.", + ) + return { + stream.stream.name + for stream in self._catalog_manager.source_catalog.streams + if self._table_exists(self.get_sql_table_name(stream.stream.name)) + } + + @abc.abstractmethod + def get_telemetry_info(self) -> CacheTelemetryInfo: + pass diff --git a/airbyte-lib/airbyte_lib/caches/duckdb.py b/airbyte-lib/airbyte_lib/caches/duckdb.py new file mode 100644 index 000000000000..07c7fbdaf125 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/duckdb.py @@ -0,0 +1,205 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A DuckDB implementation of the cache.""" + +from __future__ import annotations + +import warnings +from pathlib import Path +from textwrap import dedent, indent +from typing import cast + +from overrides import overrides + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.telemetry import CacheTelemetryInfo + + +# Suppress warnings from DuckDB about reflection on indices. +# https://github.com/Mause/duckdb_engine/issues/905 +warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", +) + + +class DuckDBCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the DuckDB cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + db_path: Path | str + """Normally db_path is a Path object. + + There are some cases, such as when connecting to MotherDuck, where it could be a string that + is not also a path, such as "md:" to connect the user's default MotherDuck DB. + """ + schema_name: str = "main" + """The name of the schema to write to. Defaults to "main".""" + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + # return f"duckdb:///{self.db_path}?schema={self.schema_name}" + return f"duckdb:///{self.db_path!s}" + + def get_database_name(self) -> str: + """Return the name of the database.""" + if self.db_path == ":memory:": + return "memory" + + # Return the file name without the extension + return str(self.db_path).split("/")[-1].split(".")[0] + + +class DuckDBCacheBase(SQLCacheBase): + """A DuckDB implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + """ + + config_class = DuckDBCacheConfig + supports_merge_insert = False + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("duckdb") + + @overrides + def _setup(self) -> None: + """Create the database parent folder if it doesn't yet exist.""" + config = cast(DuckDBCacheConfig, self.config) + + if config.db_path == ":memory:": + return + + Path(config.db_path).parent.mkdir(parents=True, exist_ok=True) + + +class DuckDBCache(DuckDBCacheBase): + """A DuckDB implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + """ + + file_writer_class = ParquetWriter + + # TODO: Delete or rewrite this method after DuckDB adds support for primary key inspection. + # @overrides + # def _merge_temp_table_to_final_table( + # self, + # stream_name: str, + # temp_table_name: str, + # final_table_name: str, + # ) -> None: + # """Merge the temp table into the main one. + + # This implementation requires MERGE support in the SQL DB. + # Databases that do not support this syntax can override this method. + # """ + # if not self._get_primary_keys(stream_name): + # raise exc.AirbyteLibInternalError( + # message="Primary keys not found. Cannot run merge updates without primary keys.", + # context={ + # "stream_name": stream_name, + # }, + # ) + + # _ = stream_name + # final_table = self._fully_qualified(final_table_name) + # staging_table = self._fully_qualified(temp_table_name) + # self._execute_sql( + # # https://duckdb.org/docs/sql/statements/insert.html + # # NOTE: This depends on primary keys being set properly in the final table. + # f""" + # INSERT OR REPLACE INTO {final_table} BY NAME + # (SELECT * FROM {staging_table}) + # """ + # ) + + @overrides + def _ensure_compatible_table_schema( + self, + stream_name: str, + *, + raise_on_error: bool = True, + ) -> bool: + """Return true if the given table is compatible with the stream's schema. + + In addition to the base implementation, this also checks primary keys. + """ + # call super + if not super()._ensure_compatible_table_schema( + stream_name=stream_name, + raise_on_error=raise_on_error, + ): + return False + + # TODO: Add validation for primary keys after DuckDB adds support for primary key + # inspection: https://github.com/Mause/duckdb_engine/issues/594 + # This is a problem because DuckDB implicitly joins on primary keys during MERGE. + # pk_cols = self._get_primary_keys(stream_name) + # table = self.get_sql_table(table_name) + # table_pk_cols = table.primary_key.columns.keys() + # if set(pk_cols) != set(table_pk_cols): + # if raise_on_error: + # raise exc.AirbyteLibCacheTableValidationError( + # violation="Primary keys do not match.", + # context={ + # "stream_name": stream_name, + # "table_name": table_name, + # "expected": pk_cols, + # "found": table_pk_cols, + # }, + # ) + # return False + + return True + + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write a file(s) to a new table. + + We use DuckDB's `read_parquet` function to efficiently read the files and insert + them into the table in a single operation. + + Note: This implementation is fragile in regards to column ordering. However, since + we are inserting into a temp table we have just created, there should be no + drift between the table schema and the file schema. + """ + temp_table_name = self._create_table_for_loading( + stream_name=stream_name, + batch_id=batch_id, + ) + columns_list = [ + self._quote_identifier(c) + for c in list(self._get_sql_column_definitions(stream_name).keys()) + ] + columns_list_str = indent("\n, ".join(columns_list), " ") + files_list = ", ".join([f"'{f!s}'" for f in files]) + insert_statement = dedent( + f""" + INSERT INTO {self.config.schema_name}.{temp_table_name} + ( + {columns_list_str} + ) + SELECT + {columns_list_str} + FROM read_parquet( + [{files_list}], + union_by_name = true + ) + """ + ) + self._execute_sql(insert_statement) + return temp_table_name diff --git a/airbyte-lib/airbyte_lib/caches/postgres.py b/airbyte-lib/airbyte_lib/caches/postgres.py new file mode 100644 index 000000000000..324d29c2d58e --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/postgres.py @@ -0,0 +1,55 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Postgres implementation of the cache.""" + +from __future__ import annotations + +from overrides import overrides + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.telemetry import CacheTelemetryInfo + + +class PostgresCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the Postgres cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + host: str + port: int + username: str + password: str + database: str + + # Already defined in base class: `schema_name` + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + + def get_database_name(self) -> str: + """Return the name of the database.""" + return self.database + + +class PostgresCache(SQLCacheBase): + """A Postgres implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + + TOOD: Add optimized bulk load path for Postgres. Could use an alternate file writer + or another import method. (Relatively low priority, since for now it works fine as-is.) + """ + + config_class = PostgresCacheConfig + file_writer_class = ParquetWriter + supports_merge_insert = False # TODO: Add native implementation for merge insert + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("postgres") diff --git a/airbyte-lib/airbyte_lib/caches/snowflake.py b/airbyte-lib/airbyte_lib/caches/snowflake.py new file mode 100644 index 000000000000..2a59f723af06 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/snowflake.py @@ -0,0 +1,164 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Snowflake implementation of the cache.""" + +from __future__ import annotations + +from textwrap import dedent, indent +from typing import TYPE_CHECKING + +import sqlalchemy +from overrides import overrides +from snowflake.sqlalchemy import URL, VARIANT + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import ( + RecordDedupeMode, + SQLCacheBase, + SQLCacheConfigBase, +) +from airbyte_lib.telemetry import CacheTelemetryInfo +from airbyte_lib.types import SQLTypeConverter + + +if TYPE_CHECKING: + from pathlib import Path + + from sqlalchemy.engine import Connection + + +class SnowflakeCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the Snowflake cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + account: str + username: str + password: str + warehouse: str + database: str + role: str + + dedupe_mode = RecordDedupeMode.APPEND + + # Already defined in base class: + # schema_name: str + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return str( + URL( + account=self.account, + user=self.username, + password=self.password, + database=self.database, + warehouse=self.warehouse, + schema=self.schema_name, + role=self.role, + ) + ) + + def get_database_name(self) -> str: + """Return the name of the database.""" + return self.database + + +class SnowflakeTypeConverter(SQLTypeConverter): + """A class to convert types for Snowflake.""" + + @overrides + def to_sql_type( + self, + json_schema_property_def: dict[str, str | dict | list], + ) -> sqlalchemy.types.TypeEngine: + """Convert a value to a SQL type. + + We first call the parent class method to get the type. Then if the type JSON, we + replace it with VARIANT. + """ + sql_type = super().to_sql_type(json_schema_property_def) + if isinstance(sql_type, sqlalchemy.types.JSON): + return VARIANT() + + return sql_type + + +class SnowflakeSQLCache(SQLCacheBase): + """A Snowflake implementation of the cache. + + Parquet is used for local file storage before bulk loading. + """ + + config_class = SnowflakeCacheConfig + file_writer_class = ParquetWriter + type_converter_class = SnowflakeTypeConverter + + @overrides + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write files to a new table.""" + temp_table_name = self._create_table_for_loading( + stream_name=stream_name, + batch_id=batch_id, + ) + internal_sf_stage_name = f"@%{temp_table_name}" + put_files_statements = "\n".join( + [ + f"PUT 'file://{file_path.absolute()!s}' {internal_sf_stage_name};" + for file_path in files + ] + ) + self._execute_sql(put_files_statements) + + columns_list = [ + self._quote_identifier(c) + for c in list(self._get_sql_column_definitions(stream_name).keys()) + ] + files_list = ", ".join([f"'{f.name}'" for f in files]) + columns_list_str: str = indent("\n, ".join(columns_list), " " * 12) + variant_cols_str: str = ("\n" + " " * 21 + ", ").join([f"$1:{col}" for col in columns_list]) + copy_statement = dedent( + f""" + COPY INTO {temp_table_name} + ( + {columns_list_str} + ) + FROM ( + SELECT {variant_cols_str} + FROM {internal_sf_stage_name} + ) + FILES = ( {files_list} ) + FILE_FORMAT = ( TYPE = PARQUET ) + ; + """ + ) + self._execute_sql(copy_statement) + return temp_table_name + + @overrides + def _init_connection_settings(self, connection: Connection) -> None: + """We set Snowflake-specific settings for the session. + + This sets QUOTED_IDENTIFIERS_IGNORE_CASE setting to True, which is necessary because + Snowflake otherwise will treat quoted table and column references as case-sensitive. + More info: https://docs.snowflake.com/en/sql-reference/identifiers-syntax + + This also sets MULTI_STATEMENT_COUNT to 0, which allows multi-statement commands. + """ + connection.execute( + """ + ALTER SESSION SET + QUOTED_IDENTIFIERS_IGNORE_CASE = TRUE + MULTI_STATEMENT_COUNT = 0 + """ + ) + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("snowflake") diff --git a/airbyte-lib/airbyte_lib/config.py b/airbyte-lib/airbyte_lib/config.py new file mode 100644 index 000000000000..4fd4e60367a9 --- /dev/null +++ b/airbyte-lib/airbyte_lib/config.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define base Config interface, used by Caches and also File Writers (Processors).""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class CacheConfigBase( + BaseModel +): # TODO: meta=EnforceOverrides (Pydantic doesn't like it currently.) + pass diff --git a/airbyte-lib/airbyte_lib/datasets/__init__.py b/airbyte-lib/airbyte_lib/datasets/__init__.py new file mode 100644 index 000000000000..bfd4f02ce319 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/__init__.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from airbyte_lib.datasets._base import DatasetBase +from airbyte_lib.datasets._lazy import LazyDataset +from airbyte_lib.datasets._map import DatasetMap +from airbyte_lib.datasets._sql import CachedDataset, SQLDataset + + +__all__ = [ + "CachedDataset", + "DatasetBase", + "DatasetMap", + "LazyDataset", + "SQLDataset", +] diff --git a/airbyte-lib/airbyte_lib/datasets/_base.py b/airbyte-lib/airbyte_lib/datasets/_base.py new file mode 100644 index 000000000000..f0fdfab52b91 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_base.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Iterator, Mapping +from typing import Any, cast + +from pandas import DataFrame + + +class DatasetBase(ABC): + """Base implementation for all datasets.""" + + @abstractmethod + def __iter__(self) -> Iterator[Mapping[str, Any]]: + """Return the iterator of records.""" + raise NotImplementedError + + def to_pandas(self) -> DataFrame: + """Return a pandas DataFrame representation of the dataset. + + The base implementation simply passes the record iterator to Panda's DataFrame constructor. + """ + # Technically, we return an iterator of Mapping objects. However, pandas + # expects an iterator of dict objects. This cast is safe because we know + # duck typing is correct for this use case. + return DataFrame(cast(Iterator[dict[str, Any]], self)) diff --git a/airbyte-lib/airbyte_lib/datasets/_lazy.py b/airbyte-lib/airbyte_lib/datasets/_lazy.py new file mode 100644 index 000000000000..83d67cec0043 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_lazy.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from overrides import overrides + +from airbyte_lib.datasets import DatasetBase + + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + + +class LazyDataset(DatasetBase): + """A dataset that is loaded incrementally from a source or a SQL query.""" + + def __init__( + self, + iterator: Iterator[Mapping[str, Any]], + ) -> None: + self._iterator: Iterator[Mapping[str, Any]] = iterator + super().__init__() + + @overrides + def __iter__(self) -> Iterator[Mapping[str, Any]]: + return self._iterator + + def __next__(self) -> Mapping[str, Any]: + return next(self._iterator) diff --git a/airbyte-lib/airbyte_lib/datasets/_map.py b/airbyte-lib/airbyte_lib/datasets/_map.py new file mode 100644 index 000000000000..42eaed88f0e3 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_map.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A generic interface for a set of streams. + +TODO: This is a work in progress. It is not yet used by any other code. +TODO: Implement before release, or delete. +""" +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from airbyte_lib.datasets._base import DatasetBase + + +class DatasetMap(Mapping): + """A generic interface for a set of streams or datasets.""" + + def __init__(self) -> None: + self._datasets: dict[str, DatasetBase] = {} + + def __getitem__(self, key: str) -> DatasetBase: + return self._datasets[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._datasets) + + def __len__(self) -> int: + return len(self._datasets) diff --git a/airbyte-lib/airbyte_lib/datasets/_sql.py b/airbyte-lib/airbyte_lib/datasets/_sql.py new file mode 100644 index 000000000000..7dfb22482146 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_sql.py @@ -0,0 +1,136 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, cast + +from overrides import overrides +from sqlalchemy import and_, func, select, text + +from airbyte_lib.datasets._base import DatasetBase + + +if TYPE_CHECKING: + from collections.abc import Iterator + + from pandas import DataFrame + from sqlalchemy import Selectable, Table + from sqlalchemy.sql import ClauseElement + + from airbyte_lib.caches import SQLCacheBase + + +class SQLDataset(DatasetBase): + """A dataset that is loaded incrementally from a SQL query. + + The CachedDataset class is a subclass of this class, which simply passes a SELECT over the full + table as the query statement. + """ + + def __init__( + self, + cache: SQLCacheBase, + stream_name: str, + query_statement: Selectable, + ) -> None: + self._length: int | None = None + self._cache: SQLCacheBase = cache + self._stream_name: str = stream_name + self._query_statement: Selectable = query_statement + super().__init__() + + @property + def stream_name(self) -> str: + return self._stream_name + + def __iter__(self) -> Iterator[Mapping[str, Any]]: + with self._cache.get_sql_connection() as conn: + for row in conn.execute(self._query_statement): + # Access to private member required because SQLAlchemy doesn't expose a public API. + # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.engine.row.RowMapping.html + yield cast(Mapping[str, Any], row._mapping) # noqa: SLF001 + + def __len__(self) -> int: + """Return the number of records in the dataset. + + This method caches the length of the dataset after the first call. + """ + if self._length is None: + count_query = select([func.count()]).select_from(self._query_statement.alias()) + with self._cache.get_sql_connection() as conn: + self._length = conn.execute(count_query).scalar() + + return self._length + + def to_pandas(self) -> DataFrame: + return self._cache.get_pandas_dataframe(self._stream_name) + + def with_filter(self, *filter_expressions: ClauseElement | str) -> SQLDataset: + """Filter the dataset by a set of column values. + + Filters can be specified as either a string or a SQLAlchemy expression. + + Filters are lazily applied to the dataset, so they can be chained together. For example: + + dataset.with_filter("id > 5").with_filter("id < 10") + + is equivalent to: + + dataset.with_filter("id > 5", "id < 10") + """ + # Convert all strings to TextClause objects. + filters: list[ClauseElement] = [ + text(expression) if isinstance(expression, str) else expression + for expression in filter_expressions + ] + filtered_select = self._query_statement.where(and_(*filters)) + return SQLDataset( + cache=self._cache, + stream_name=self._stream_name, + query_statement=filtered_select, + ) + + +class CachedDataset(SQLDataset): + """A dataset backed by a SQL table cache. + + Because this dataset includes all records from the underlying table, we also expose the + underlying table as a SQLAlchemy Table object. + """ + + def __init__(self, cache: SQLCacheBase, stream_name: str) -> None: + self._sql_table: Table = cache.get_sql_table(stream_name) + super().__init__( + cache=cache, + stream_name=stream_name, + query_statement=self._sql_table.select(), + ) + + @overrides + def to_pandas(self) -> DataFrame: + return self._cache.get_pandas_dataframe(self._stream_name) + + def to_sql_table(self) -> Table: + return self._sql_table + + def __eq__(self, value: object) -> bool: + """Return True if the value is a CachedDataset with the same cache and stream name. + + In the case of CachedDataset objects, we can simply compare the cache and stream name. + + Note that this equality check is only supported on CachedDataset objects and not for + the base SQLDataset implementation. This is because of the complexity and computational + cost of comparing two arbitrary SQL queries that could be bound to different variables, + as well as the chance that two queries can be syntactically equivalent without being + text-wise equivalent. + """ + if not isinstance(value, SQLDataset): + return False + + if self._cache is not value._cache: + return False + + if self._stream_name != value._stream_name: + return False + + return True diff --git a/airbyte-lib/airbyte_lib/exceptions.py b/airbyte-lib/airbyte_lib/exceptions.py new file mode 100644 index 000000000000..04322c080580 --- /dev/null +++ b/airbyte-lib/airbyte_lib/exceptions.py @@ -0,0 +1,277 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""All exceptions used in the Airbyte Lib. + +This design is modeled after structlog's exceptions, in that we bias towards auto-generated +property prints rather than sentence-like string concatenation. + +E.g. Instead of this: +> Subprocess failed with exit code '1' + +We do this: +> Subprocess failed. (exit_code=1) + +The benefit of this approach is that we can easily support structured logging, and we can +easily add new properties to exceptions without having to update all the places where they +are raised. We can also support any arbitrary number of properties in exceptions, without spending +time on building sentence-like string constructions with optional inputs. + + +In addition, the following principles are applied for exception class design: + +- All exceptions inherit from a common base class. +- All exceptions have a message attribute. +- The first line of the docstring is used as the default message. +- The default message can be overridden by explicitly setting the message attribute. +- Exceptions may optionally have a guidance attribute. +- Exceptions may optionally have a help_url attribute. +- Rendering is automatically handled by the base class. +- Any helpful context not defined by the exception class can be passed in the `context` dict arg. +- Within reason, avoid sending PII to the exception constructor. +- Exceptions are dataclasses, so they can be instantiated with keyword arguments. +- Use the 'from' syntax to chain exceptions when it is helpful to do so. + E.g. `raise AirbyteConnectorNotFoundError(...) from FileNotFoundError(connector_path)` +- Any exception that adds a new property should also be decorated as `@dataclass`. +""" +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import indent +from typing import Any + + +NEW_ISSUE_URL = "https://github.com/airbytehq/airbyte/issues/new/choose" +DOCS_URL = "https://docs.airbyte.io/" + + +# Base error class + + +@dataclass +class AirbyteError(Exception): + """Base class for exceptions in Airbyte.""" + + guidance: str | None = None + help_url: str | None = None + log_text: str | list[str] | None = None + context: dict[str, Any] | None = None + message: str | None = None + + def get_message(self) -> str: + """Return the best description for the exception. + + We resolve the following in order: + 1. The message sent to the exception constructor (if provided). + 2. The first line of the class's docstring. + """ + if self.message: + return self.message + + return self.__doc__.split("\n")[0] if self.__doc__ else "" + + def __str__(self) -> str: + special_properties = ["message", "guidance", "help_url", "log_text", "context"] + display_properties = { + k: v + for k, v in self.__dict__.items() + if k not in special_properties and not k.startswith("_") and v is not None + } + display_properties.update(self.context or {}) + context_str = "\n ".join( + f"{str(k).replace('_', ' ').title()}: {v!r}" for k, v in display_properties.items() + ) + exception_str = f"{self.__class__.__name__}: {self.get_message()}\n" + if context_str: + exception_str += " " + context_str + + if self.log_text: + if isinstance(self.log_text, list): + self.log_text = "\n".join(self.log_text) + + exception_str += f"\nLog output: \n {indent(self.log_text, ' ')}" + + if self.guidance: + exception_str += f"\nSuggestion: {self.guidance}" + + if self.help_url: + exception_str += f"\nMore info: {self.help_url}" + + return exception_str + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + properties_str = ", ".join( + f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_") + ) + return f"{class_name}({properties_str})" + + +# AirbyteLib Internal Errors (these are probably bugs) + + +@dataclass +class AirbyteLibInternalError(AirbyteError): + """An internal error occurred in Airbyte Lib.""" + + guidance = "Please consider reporting this error to the Airbyte team." + help_url = NEW_ISSUE_URL + + +# AirbyteLib Input Errors (replaces ValueError for user input) + + +@dataclass +class AirbyteLibInputError(AirbyteError, ValueError): + """The input provided to AirbyteLib did not match expected validation rules. + + This inherits from ValueError so that it can be used as a drop-in replacement for + ValueError in the Airbyte Lib API. + """ + + # TODO: Consider adding a help_url that links to the auto-generated API reference. + + guidance = "Please check the provided value and try again." + input_value: str | None = None + + +@dataclass +class AirbyteLibNoStreamsSelectedError(AirbyteLibInputError): + """No streams were selected for the source.""" + + guidance = ( + "Please call `select_streams()` to select at least one stream from the list provided. " + "You can also call `select_all_streams()` to select all available streams for this source." + ) + connector_name: str | None = None + available_streams: list[str] | None = None + + +# AirbyteLib Cache Errors + + +class AirbyteLibCacheError(AirbyteError): + """Error occurred while accessing the cache.""" + + +@dataclass +class AirbyteLibCacheTableValidationError(AirbyteLibCacheError): + """Cache table validation failed.""" + + violation: str | None = None + + +@dataclass +class AirbyteConnectorConfigurationMissingError(AirbyteLibCacheError): + """Connector is missing configuration.""" + + connector_name: str | None = None + + +# Subprocess Errors + + +@dataclass +class AirbyteSubprocessError(AirbyteError): + """Error when running subprocess.""" + + run_args: list[str] | None = None + + +@dataclass +class AirbyteSubprocessFailedError(AirbyteSubprocessError): + """Subprocess failed.""" + + exit_code: int | None = None + + +# Connector Registry Errors + + +class AirbyteConnectorRegistryError(AirbyteError): + """Error when accessing the connector registry.""" + + +@dataclass +class AirbyteConnectorNotRegisteredError(AirbyteConnectorRegistryError): + """Connector not found in registry.""" + + connector_name: str | None = None + guidance = "Please double check the connector name." + + +@dataclass +class AirbyteConnectorNotPyPiPublishedError(AirbyteConnectorRegistryError): + """Connector found, but not published to PyPI.""" + + connector_name: str | None = None + guidance = "This likely means that the connector is not ready for use with airbyte-lib." + + +# Connector Errors + + +@dataclass +class AirbyteConnectorError(AirbyteError): + """Error when running the connector.""" + + connector_name: str | None = None + + +class AirbyteConnectorExecutableNotFoundError(AirbyteConnectorError): + """Connector executable not found.""" + + +class AirbyteConnectorInstallationError(AirbyteConnectorError): + """Error when installing the connector.""" + + +class AirbyteConnectorReadError(AirbyteConnectorError): + """Error when reading from the connector.""" + + +class AirbyteNoDataFromConnectorError(AirbyteConnectorError): + """No data was provided from the connector.""" + + +class AirbyteConnectorMissingCatalogError(AirbyteConnectorError): + """Connector did not return a catalog.""" + + +class AirbyteConnectorMissingSpecError(AirbyteConnectorError): + """Connector did not return a spec.""" + + +class AirbyteConnectorCheckFailedError(AirbyteConnectorError): + """Connector check failed.""" + + guidance = ( + "Please double-check your config or review the connector's logs for more information." + ) + + +@dataclass +class AirbyteConnectorFailedError(AirbyteConnectorError): + """Connector failed.""" + + exit_code: int | None = None + + +@dataclass +class AirbyteStreamNotFoundError(AirbyteConnectorError): + """Connector stream not found.""" + + stream_name: str | None = None + available_streams: list[str] | None = None + + +@dataclass +class AirbyteLibSecretNotFoundError(AirbyteError): + """Secret not found.""" + + guidance = "Please ensure that the secret is set." + help_url = ( + "https://docs.airbyte.com/using-airbyte/airbyte-lib/getting-started#secrets-management" + ) + + secret_name: str | None = None + sources: list[str] | None = None diff --git a/airbyte-lib/airbyte_lib/progress.py b/airbyte-lib/airbyte_lib/progress.py new file mode 100644 index 000000000000..5d9de1cfc053 --- /dev/null +++ b/airbyte-lib/airbyte_lib/progress.py @@ -0,0 +1,381 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A simple progress bar for the command line and IPython notebooks.""" +from __future__ import annotations + +import datetime +import math +import sys +import time +from contextlib import suppress +from enum import Enum, auto +from typing import cast + +from rich.errors import LiveError +from rich.live import Live as RichLive +from rich.markdown import Markdown as RichMarkdown + + +DEFAULT_REFRESHES_PER_SECOND = 2 +IS_REPL = hasattr(sys, "ps1") # True if we're in a Python REPL, in which case we can use Rich. + +try: + IS_NOTEBOOK = True + from IPython import display as ipy_display + +except ImportError: + ipy_display = None + IS_NOTEBOOK = False + + +class ProgressStyle(Enum): + """An enum of progress bar styles.""" + + AUTO = auto() + """Automatically select the best style for the environment.""" + + RICH = auto() + """A Rich progress bar.""" + + IPYTHON = auto() + """Use IPython display methods.""" + + PLAIN = auto() + """A plain text progress print.""" + + NONE = auto() + """Skip progress prints.""" + + +MAX_UPDATE_FREQUENCY = 1000 +"""The max number of records to read before updating the progress bar.""" + + +def _to_time_str(timestamp: float) -> str: + """Convert a timestamp float to a local time string. + + For now, we'll just use UTC to avoid breaking tests. In the future, we should + return a local time string. + """ + datetime_obj = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + # TODO: Uncomment this line when we can get tests to properly account for local timezones. + # For now, we'll just use UTC to avoid breaking tests. + # datetime_obj = datetime_obj.astimezone() + return datetime_obj.strftime("%H:%M:%S") + + +def _get_elapsed_time_str(seconds: int) -> str: + """Return duration as a string. + + Seconds are included until 10 minutes is exceeded. + Minutes are always included after 1 minute elapsed. + Hours are always included after 1 hour elapsed. + """ + if seconds <= 60: # noqa: PLR2004 # Magic numbers OK here. + return f"{seconds} seconds" + + if seconds < 60 * 10: + minutes = seconds // 60 + seconds = seconds % 60 + return f"{minutes}min {seconds}s" + + if seconds < 60 * 60: + minutes = seconds // 60 + seconds = seconds % 60 + return f"{minutes}min" + + hours = seconds // (60 * 60) + minutes = (seconds % (60 * 60)) // 60 + return f"{hours}hr {minutes}min" + + +class ReadProgress: + """A simple progress bar for the command line and IPython notebooks.""" + + def __init__( + self, + style: ProgressStyle = ProgressStyle.AUTO, + ) -> None: + """Initialize the progress tracker.""" + # Streams expected (for progress bar) + self.num_streams_expected = 0 + + # Reads + self.read_start_time = time.time() + self.read_end_time: float | None = None + self.total_records_read = 0 + + # Writes + self.total_records_written = 0 + self.total_batches_written = 0 + self.written_stream_names: set[str] = set() + + # Finalization + self.finalize_start_time: float | None = None + self.finalize_end_time: float | None = None + self.total_records_finalized = 0 + self.total_batches_finalized = 0 + self.finalized_stream_names: set[str] = set() + + self.last_update_time: float | None = None + + self._rich_view: RichLive | None = None + self.style: ProgressStyle = style + if self.style == ProgressStyle.AUTO: + self.style = ProgressStyle.PLAIN + if IS_NOTEBOOK: + self.style = ProgressStyle.IPYTHON + + elif IS_REPL: + self.style = ProgressStyle.PLAIN + + else: + # Test for Rich availability: + self._rich_view = RichLive() + try: + self._rich_view.start() + self._rich_view.stop() + self._rich_view = None + self.style = ProgressStyle.RICH + except LiveError: + # Rich live view not available. Using plain text progress. + self._rich_view = None + self.style = ProgressStyle.PLAIN + + def _start(self) -> None: + """Start the progress bar.""" + if self.style == ProgressStyle.RICH and not self._rich_view: + self._rich_view = RichLive( + auto_refresh=True, + refresh_per_second=DEFAULT_REFRESHES_PER_SECOND, + ) + self._rich_view.start() + + def _stop(self) -> None: + """Stop the progress bar.""" + if self._rich_view: + with suppress(Exception): + self._rich_view.stop() + self._rich_view = None + + def __del__(self) -> None: + """Close the Rich view.""" + self._stop() + + def log_success(self) -> None: + """Log success and stop tracking progress.""" + if self.finalize_end_time is None: + # If we haven't already finalized, do so now. + + self.finalize_end_time = time.time() + + self.update_display(force_refresh=True) + self._stop() + + def reset(self, num_streams_expected: int) -> None: + """Reset the progress tracker.""" + # Streams expected (for progress bar) + self.num_streams_expected = num_streams_expected + + # Reads + self.read_start_time = time.time() + self.read_end_time = None + self.total_records_read = 0 + + # Writes + self.total_records_written = 0 + self.total_batches_written = 0 + self.written_stream_names = set() + + # Finalization + self.finalize_start_time = None + self.finalize_end_time = None + self.total_records_finalized = 0 + self.total_batches_finalized = 0 + self.finalized_stream_names = set() + + self._start() + + @property + def elapsed_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.finalize_end_time: + return int(self.finalize_end_time - self.read_start_time) + + return int(time.time() - self.read_start_time) + + @property + def elapsed_time_string(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_seconds) + + @property + def elapsed_seconds_since_last_update(self) -> float | None: + """Return the number of seconds elapsed since the last update.""" + if self.last_update_time is None: + return None + + return time.time() - self.last_update_time + + @property + def elapsed_read_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.read_end_time is None: + return int(time.time() - self.read_start_time) + + return int(self.read_end_time - self.read_start_time) + + @property + def elapsed_read_time_string(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_read_seconds) + + @property + def elapsed_finalization_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.finalize_start_time is None: + return 0 + if self.finalize_end_time is None: + return int(time.time() - self.finalize_start_time) + return int(self.finalize_end_time - self.finalize_start_time) + + @property + def elapsed_finalization_time_str(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_finalization_seconds) + + def log_records_read(self, new_total_count: int) -> None: + """Load a number of records read.""" + self.total_records_read = new_total_count + + # This is some math to make updates adaptive to the scale of records read. + # We want to update the display more often when the count is low, and less + # often when the count is high. + updated_period = min( + MAX_UPDATE_FREQUENCY, 10 ** math.floor(math.log10(max(self.total_records_read, 1)) / 4) + ) + if self.total_records_read % updated_period != 0: + return + + self.update_display() + + def log_batch_written(self, stream_name: str, batch_size: int) -> None: + """Log that a batch has been written. + + Args: + stream_name: The name of the stream. + batch_size: The number of records in the batch. + """ + self.total_records_written += batch_size + self.total_batches_written += 1 + self.written_stream_names.add(stream_name) + self.update_display() + + def log_batches_finalizing(self, stream_name: str, num_batches: int) -> None: + """Log that batch are ready to be finalized. + + In our current implementation, we ignore the stream name and number of batches. + We just use this as a signal that we're finished reading and have begun to + finalize any accumulated batches. + """ + _ = stream_name, num_batches # unused for now + if self.finalize_start_time is None: + self.read_end_time = time.time() + self.finalize_start_time = self.read_end_time + + self.update_display(force_refresh=True) + + def log_batches_finalized(self, stream_name: str, num_batches: int) -> None: + """Log that a batch has been finalized.""" + _ = stream_name # unused for now + self.total_batches_finalized += num_batches + self.update_display(force_refresh=True) + + def log_stream_finalized(self, stream_name: str) -> None: + """Log that a stream has been finalized.""" + self.finalized_stream_names.add(stream_name) + self.update_display(force_refresh=True) + if len(self.finalized_stream_names) == self.num_streams_expected: + self.log_success() + + def update_display(self, *, force_refresh: bool = False) -> None: + """Update the display.""" + # Don't update more than twice per second unless force_refresh is True. + if ( + not force_refresh + and self.last_update_time # if not set, then we definitely need to update + and cast(float, self.elapsed_seconds_since_last_update) < 0.5 # noqa: PLR2004 + ): + return + + status_message = self._get_status_message() + + if self.style == ProgressStyle.IPYTHON: + # We're in a notebook so use the IPython display. + ipy_display.clear_output(wait=True) + ipy_display.display(ipy_display.Markdown(status_message)) + + elif self.style == ProgressStyle.RICH and self._rich_view is not None: + self._rich_view.update(RichMarkdown(status_message)) + + elif self.style == ProgressStyle.PLAIN: + # TODO: Add a plain text progress print option that isn't too noisy. + pass + + elif self.style == ProgressStyle.NONE: + pass + + self.last_update_time = time.time() + + def _get_status_message(self) -> str: + """Compile and return a status message.""" + # Format start time as a friendly string in local timezone: + start_time_str = _to_time_str(self.read_start_time) + records_per_second: float = 0.0 + if self.elapsed_read_seconds > 0: + records_per_second = round(self.total_records_read / self.elapsed_read_seconds, 1) + status_message = ( + f"## Read Progress\n\n" + f"Started reading at {start_time_str}.\n\n" + f"Read **{self.total_records_read:,}** records " + f"over **{self.elapsed_read_time_string}** " + f"({records_per_second:,} records / second).\n\n" + ) + if self.total_records_written > 0: + status_message += ( + f"Wrote **{self.total_records_written:,}** records " + f"over {self.total_batches_written:,} batches.\n\n" + ) + if self.read_end_time is not None: + read_end_time_str = _to_time_str(self.read_end_time) + status_message += f"Finished reading at {read_end_time_str}.\n\n" + if self.finalize_start_time is not None: + finalize_start_time_str = _to_time_str(self.finalize_start_time) + status_message += f"Started finalizing streams at {finalize_start_time_str}.\n\n" + status_message += ( + f"Finalized **{self.total_batches_finalized}** batches " + f"over {self.elapsed_finalization_time_str}.\n\n" + ) + if self.finalized_stream_names: + status_message += ( + f"Completed {len(self.finalized_stream_names)} " + + (f"out of {self.num_streams_expected} " if self.num_streams_expected else "") + + "streams:\n\n" + ) + for stream_name in self.finalized_stream_names: + status_message += f" - {stream_name}\n" + + status_message += "\n\n" + + if self.finalize_end_time is not None: + completion_time_str = _to_time_str(self.finalize_end_time) + status_message += ( + f"Completed writing at {completion_time_str}. " + f"Total time elapsed: {self.elapsed_time_string}\n\n" + ) + status_message += "\n------------------------------------------------\n" + + return status_message + + +progress = ReadProgress() diff --git a/airbyte-lib/airbyte_lib/py.typed b/airbyte-lib/airbyte_lib/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py new file mode 100644 index 000000000000..bab4a87f034f --- /dev/null +++ b/airbyte-lib/airbyte_lib/registry.py @@ -0,0 +1,117 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import os +from copy import copy +from dataclasses import dataclass +from pathlib import Path + +import requests + +from airbyte_lib import exceptions as exc +from airbyte_lib.version import get_version + + +__cache: dict[str, ConnectorMetadata] | None = None + + +REGISTRY_ENV_VAR = "AIRBYTE_LOCAL_REGISTRY" +REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + + +@dataclass +class ConnectorMetadata: + name: str + latest_available_version: str + pypi_package_name: str | None + + +def _get_registry_url() -> str: + if REGISTRY_ENV_VAR in os.environ: + return str(os.environ.get(REGISTRY_ENV_VAR)) + + return REGISTRY_URL + + +def _registry_entry_to_connector_metadata(entry: dict) -> ConnectorMetadata: + name = entry["dockerRepository"].replace("airbyte/", "") + remote_registries: dict = entry.get("remoteRegistries", {}) + pypi_registry: dict = remote_registries.get("pypi", {}) + pypi_package_name: str = pypi_registry.get("packageName", None) + pypi_enabled: bool = pypi_registry.get("enabled", False) + return ConnectorMetadata( + name=name, + latest_available_version=entry["dockerImageTag"], + pypi_package_name=pypi_package_name if pypi_enabled else None, + ) + + +def _get_registry_cache(*, force_refresh: bool = False) -> dict[str, ConnectorMetadata]: + """Return the registry cache.""" + global __cache + if __cache and not force_refresh: + return __cache + + registry_url = _get_registry_url() + if registry_url.startswith("http"): + response = requests.get( + registry_url, headers={"User-Agent": f"airbyte-lib-{get_version()}"} + ) + response.raise_for_status() + data = response.json() + else: + # Assume local file + with Path(registry_url).open() as f: + data = json.load(f) + + new_cache: dict[str, ConnectorMetadata] = {} + + for connector in data["sources"]: + connector_metadata = _registry_entry_to_connector_metadata(connector) + new_cache[connector_metadata.name] = connector_metadata + + if len(new_cache) == 0: + raise exc.AirbyteLibInternalError( + message="Connector registry is empty.", + context={ + "registry_url": _get_registry_url(), + }, + ) + + __cache = new_cache + return __cache + + +def get_connector_metadata(name: str) -> ConnectorMetadata: + """Check the cache for the connector. + + If the cache is empty, populate by calling update_cache. + """ + cache = copy(_get_registry_cache()) + if not cache: + raise exc.AirbyteLibInternalError( + message="Connector registry could not be loaded.", + context={ + "registry_url": _get_registry_url(), + }, + ) + if name not in cache: + raise exc.AirbyteConnectorNotRegisteredError( + connector_name=name, + context={ + "registry_url": _get_registry_url(), + "available_connectors": get_available_connectors(), + }, + ) + return cache[name] + + +def get_available_connectors() -> list[str]: + """Return a list of all available connectors. + + Connectors will be returned in alphabetical order, with the standard prefix "source-". + """ + return sorted( + conn.name for conn in _get_registry_cache().values() if conn.pypi_package_name is not None + ) diff --git a/airbyte-lib/airbyte_lib/results.py b/airbyte-lib/airbyte_lib/results.py new file mode 100644 index 000000000000..5c5021fc8afc --- /dev/null +++ b/airbyte-lib/airbyte_lib/results.py @@ -0,0 +1,56 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING + +from airbyte_lib.datasets import CachedDataset + + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.engine import Engine + + from airbyte_lib.caches import SQLCacheBase + + +class ReadResult(Mapping[str, CachedDataset]): + def __init__( + self, processed_records: int, cache: SQLCacheBase, processed_streams: list[str] + ) -> None: + self.processed_records = processed_records + self._cache = cache + self._processed_streams = processed_streams + + def __getitem__(self, stream: str) -> CachedDataset: + if stream not in self._processed_streams: + raise KeyError(stream) + + return CachedDataset(self._cache, stream) + + def __contains__(self, stream: object) -> bool: + if not isinstance(stream, str): + return False + + return stream in self._processed_streams + + def __iter__(self) -> Iterator[str]: + return self._processed_streams.__iter__() + + def __len__(self) -> int: + return len(self._processed_streams) + + def get_sql_engine(self) -> Engine: + return self._cache.get_sql_engine() + + @property + def streams(self) -> Mapping[str, CachedDataset]: + return { + stream_name: CachedDataset(self._cache, stream_name) + for stream_name in self._processed_streams + } + + @property + def cache(self) -> SQLCacheBase: + return self._cache diff --git a/airbyte-lib/airbyte_lib/secrets.py b/airbyte-lib/airbyte_lib/secrets.py new file mode 100644 index 000000000000..6aea9f163d2f --- /dev/null +++ b/airbyte-lib/airbyte_lib/secrets.py @@ -0,0 +1,128 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Secrets management for AirbyteLib.""" +from __future__ import annotations + +import contextlib +import os +from enum import Enum, auto +from getpass import getpass +from typing import TYPE_CHECKING + +from dotenv import dotenv_values + +from airbyte_lib import exceptions as exc + + +if TYPE_CHECKING: + from collections.abc import Callable + + +try: + from google.colab import userdata as colab_userdata +except ImportError: + colab_userdata = None + + +class SecretSource(Enum): + ENV = auto() + DOTENV = auto() + GOOGLE_COLAB = auto() + ANY = auto() + + PROMPT = auto() + + +def _get_secret_from_env( + secret_name: str, +) -> str | None: + if secret_name not in os.environ: + return None + + return os.environ[secret_name] + + +def _get_secret_from_dotenv( + secret_name: str, +) -> str | None: + try: + dotenv_vars: dict[str, str | None] = dotenv_values() + except Exception: + # Can't locate or parse a .env file + return None + + if secret_name not in dotenv_vars: + # Secret not found + return None + + return dotenv_vars[secret_name] + + +def _get_secret_from_colab( + secret_name: str, +) -> str | None: + if colab_userdata is None: + # The module doesn't exist. We probably aren't in Colab. + return None + + try: + return colab_userdata.get(secret_name) + except Exception: + # Secret name not found. Continue. + return None + + +def _get_secret_from_prompt( + secret_name: str, +) -> str | None: + with contextlib.suppress(Exception): + return getpass(f"Enter the value for secret '{secret_name}': ") + + return None + + +_SOURCE_FUNCTIONS: dict[SecretSource, Callable] = { + SecretSource.ENV: _get_secret_from_env, + SecretSource.DOTENV: _get_secret_from_dotenv, + SecretSource.GOOGLE_COLAB: _get_secret_from_colab, + SecretSource.PROMPT: _get_secret_from_prompt, +} + + +def get_secret( + secret_name: str, + source: SecretSource | list[SecretSource] = SecretSource.ANY, + *, + prompt: bool = True, +) -> str: + """Get a secret from the environment. + + The optional `source` argument of enum type `SecretSource` or list of `SecretSource` options. + If left blank, the `source` arg will be `SecretSource.ANY`. If `source` is set to a specific + source, then only that source will be checked. If a list of `SecretSource` entries is passed, + then the sources will be checked using the provided ordering. + + If `prompt` to `True` or if SecretSource.PROMPT is declared in the `source` arg, then the + user will be prompted to enter the secret if it is not found in any of the other sources. + """ + sources = [source] if not isinstance(source, list) else source + all_sources = set(_SOURCE_FUNCTIONS.keys()) - {SecretSource.PROMPT} + if SecretSource.ANY in sources: + sources += [s for s in all_sources if s not in sources] + sources.remove(SecretSource.ANY) + + if prompt or SecretSource.PROMPT in sources: + if SecretSource.PROMPT in sources: + sources.remove(SecretSource.PROMPT) + + sources.append(SecretSource.PROMPT) # Always check prompt last + + for source in sources: + fn = _SOURCE_FUNCTIONS[source] # Get the matching function for this source + val = fn(secret_name) + if val: + return val + + raise exc.AirbyteLibSecretNotFoundError( + secret_name=secret_name, + sources=[str(s) for s in sources], + ) diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py new file mode 100644 index 000000000000..47beb0ee0aa0 --- /dev/null +++ b/airbyte-lib/airbyte_lib/source.py @@ -0,0 +1,582 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import tempfile +import warnings +from contextlib import contextmanager, suppress +from typing import TYPE_CHECKING, Any + +import jsonschema +import pendulum +import yaml +from rich import print + +from airbyte_protocol.models import ( + AirbyteCatalog, + AirbyteMessage, + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + Status, + SyncMode, + TraceType, + Type, +) + +from airbyte_lib import exceptions as exc +from airbyte_lib._factories.cache_factories import get_default_cache +from airbyte_lib._util import protocol_util +from airbyte_lib._util.text_util import lower_case_set # Internal utility functions +from airbyte_lib.datasets._lazy import LazyDataset +from airbyte_lib.progress import progress +from airbyte_lib.results import ReadResult +from airbyte_lib.strategies import WriteStrategy +from airbyte_lib.telemetry import ( + CacheTelemetryInfo, + SyncState, + send_telemetry, + streaming_cache_info, +) + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + from airbyte_lib._executor import Executor + from airbyte_lib.caches import SQLCacheBase + + +@contextmanager +def as_temp_files(files_contents: list[Any]) -> Generator[list[str], Any, None]: + """Write the given contents to temporary files and yield the file paths as strings.""" + temp_files: list[Any] = [] + try: + for content in files_contents: + temp_file = tempfile.NamedTemporaryFile(mode="w+t", delete=True) + temp_file.write( + json.dumps(content) if isinstance(content, dict) else content, + ) + temp_file.flush() + temp_files.append(temp_file) + yield [file.name for file in temp_files] + finally: + for temp_file in temp_files: + with suppress(Exception): + temp_file.close() + + +class Source: + """A class representing a source that can be called.""" + + def __init__( + self, + executor: Executor, + name: str, + config: dict[str, Any] | None = None, + streams: list[str] | None = None, + *, + validate: bool = False, + ) -> None: + """Initialize the source. + + If config is provided, it will be validated against the spec if validate is True. + """ + self.executor = executor + self.name = name + self._processed_records = 0 + self._config_dict: dict[str, Any] | None = None + self._last_log_messages: list[str] = [] + self._discovered_catalog: AirbyteCatalog | None = None + self._spec: ConnectorSpecification | None = None + self._selected_stream_names: list[str] = [] + if config is not None: + self.set_config(config, validate=validate) + if streams is not None: + self.set_streams(streams) + + def set_streams(self, streams: list[str]) -> None: + """Deprecated. See select_streams().""" + warnings.warn( + "The 'set_streams' method is deprecated and will be removed in a future version. " + "Please use the 'select_streams' method instead.", + DeprecationWarning, + stacklevel=2, + ) + self.select_streams(streams) + + def select_all_streams(self) -> None: + """Select all streams. + + This is a more streamlined equivalent to: + > source.select_streams(source.get_available_streams()). + """ + self._selected_stream_names = self.get_available_streams() + + def select_streams(self, streams: list[str]) -> None: + """Select the stream names that should be read from the connector. + + Currently, if this is not set, all streams will be read. + """ + available_streams = self.get_available_streams() + for stream in streams: + if stream not in available_streams: + raise exc.AirbyteStreamNotFoundError( + stream_name=stream, + connector_name=self.name, + available_streams=available_streams, + ) + self._selected_stream_names = streams + + def get_selected_streams(self) -> list[str]: + """Get the selected streams. + + If no streams are selected, return an empty list. + """ + return self._selected_stream_names + + def set_config( + self, + config: dict[str, Any], + *, + validate: bool = False, + ) -> None: + """Set the config for the connector. + + If validate is True, raise an exception if the config fails validation. + + If validate is False, validation will be deferred until check() or validate_config() + is called. + """ + if validate: + self.validate_config(config) + + self._config_dict = config + + def get_config(self) -> dict[str, Any]: + """Get the config for the connector.""" + return self._config + + @property + def _config(self) -> dict[str, Any]: + if self._config_dict is None: + raise exc.AirbyteConnectorConfigurationMissingError( + guidance="Provide via get_source() or set_config()" + ) + return self._config_dict + + def _discover(self) -> AirbyteCatalog: + """Call discover on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with discover --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self._config]) as [config_file]: + for msg in self._execute(["discover", "--config", config_file]): + if msg.type == Type.CATALOG and msg.catalog: + return msg.catalog + raise exc.AirbyteConnectorMissingCatalogError( + log_text=self._last_log_messages, + ) + + def validate_config(self, config: dict[str, Any] | None = None) -> None: + """Validate the config against the spec. + + If config is not provided, the already-set config will be validated. + """ + spec = self._get_spec(force_refresh=False) + config = self._config if config is None else config + jsonschema.validate(config, spec.connectionSpecification) + + def get_available_streams(self) -> list[str]: + """Get the available streams from the spec.""" + return [s.name for s in self.discovered_catalog.streams] + + def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: + """Call spec on the connector. + + This involves the following steps: + * execute the connector with spec + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + if force_refresh or self._spec is None: + for msg in self._execute(["spec"]): + if msg.type == Type.SPEC and msg.spec: + self._spec = msg.spec + break + + if self._spec: + return self._spec + + raise exc.AirbyteConnectorMissingSpecError( + log_text=self._last_log_messages, + ) + + @property + def _yaml_spec(self) -> str: + """Get the spec as a yaml string. + + For now, the primary use case is for writing and debugging a valid config for a source. + + This is private for now because we probably want better polish before exposing this + as a stable interface. This will also get easier when we have docs links with this info + for each connector. + """ + spec_obj: ConnectorSpecification = self._get_spec() + spec_dict = spec_obj.dict(exclude_unset=True) + # convert to a yaml string + return yaml.dump(spec_dict) + + @property + def docs_url(self) -> str: + """Get the URL to the connector's documentation.""" + # TODO: Replace with docs URL from metadata when available + return "https://docs.airbyte.com/integrations/sources/" + self.name.lower().replace( + "source-", "" + ) + + @property + def discovered_catalog(self) -> AirbyteCatalog: + """Get the raw catalog for the given streams. + + If the catalog is not yet known, we call discover to get it. + """ + if self._discovered_catalog is None: + self._discovered_catalog = self._discover() + + return self._discovered_catalog + + @property + def configured_catalog(self) -> ConfiguredAirbyteCatalog: + """Get the configured catalog for the given streams. + + If the raw catalog is not yet known, we call discover to get it. + + If no specific streams are selected, we return a catalog that syncs all available streams. + + TODO: We should consider disabling by default the streams that the connector would + disable by default. (For instance, streams that require a premium license are sometimes + disabled by default within the connector.) + """ + # Ensure discovered catalog is cached before we start + _ = self.discovered_catalog + + # Filter for selected streams if set, otherwise use all available streams: + streams_filter: list[str] = self._selected_stream_names or self.get_available_streams() + + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=stream, + destination_sync_mode=DestinationSyncMode.overwrite, + primary_key=stream.source_defined_primary_key, + # TODO: The below assumes all sources can coalesce from incremental sync to + # full_table as needed. CDK supports this, so it might be safe: + sync_mode=SyncMode.incremental, + ) + for stream in self.discovered_catalog.streams + if stream.name in streams_filter + ], + ) + + def get_records(self, stream: str) -> LazyDataset: + """Read a stream from the connector. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs the given stream in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the first AirbyteRecordMessages that come along. + * Make sure the subprocess is killed when the function returns. + """ + catalog = self._discover() + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in catalog.streams + if s.name == stream + ], + ) + if len(configured_catalog.streams) == 0: + raise exc.AirbyteLibInputError( + message="Requested stream does not exist.", + context={ + "stream": stream, + "available_streams": self.get_available_streams(), + "connector_name": self.name, + }, + ) from KeyError(stream) + + configured_stream = configured_catalog.streams[0] + all_properties = set(configured_stream.stream.json_schema["properties"].keys()) + + def _with_missing_columns(records: Iterable[dict[str, Any]]) -> Iterator[dict[str, Any]]: + """Add missing columns to the record with null values.""" + for record in records: + existing_properties_lower = lower_case_set(record.keys()) + appended_dict = { + prop: None + for prop in all_properties + if prop.lower() not in existing_properties_lower + } + yield {**record, **appended_dict} + + iterator: Iterator[dict[str, Any]] = _with_missing_columns( + protocol_util.airbyte_messages_to_record_dicts( + self._read_with_catalog( + streaming_cache_info, + configured_catalog, + ), + ) + ) + return LazyDataset(iterator) + + def check(self) -> None: + """Call check on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with check --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self._config]) as [config_file]: + try: + for msg in self._execute(["check", "--config", config_file]): + if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: + if msg.connectionStatus.status != Status.FAILED: + print(f"Connection check succeeded for `{self.name}`.") + return + + raise exc.AirbyteConnectorCheckFailedError( + help_url=self.docs_url, + context={ + "failure_reason": msg.connectionStatus.message, + }, + ) + raise exc.AirbyteConnectorCheckFailedError(log_text=self._last_log_messages) + except exc.AirbyteConnectorReadError as ex: + raise exc.AirbyteConnectorCheckFailedError( + message="The connector failed to check the connection.", + log_text=ex.log_text, + ) from ex + + def install(self) -> None: + """Install the connector if it is not yet installed.""" + self.executor.install() + print("For configuration instructions, see: \n" f"{self.docs_url}#reference\n") + + def uninstall(self) -> None: + """Uninstall the connector if it is installed. + + This only works if the use_local_install flag wasn't used and installation is managed by + airbyte-lib. + """ + self.executor.uninstall() + + def _read( + self, + cache_info: CacheTelemetryInfo, + state: list[AirbyteStateMessage] | None = None, + ) -> Iterable[AirbyteMessage]: + """ + Call read on the connector. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs all streams in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteMessage that come along. + """ + # Ensure discovered and configured catalog properties are cached before we start reading + _ = self.discovered_catalog + _ = self.configured_catalog + yield from self._read_with_catalog( + cache_info, + catalog=self.configured_catalog, + state=state, + ) + + def _read_with_catalog( + self, + cache_info: CacheTelemetryInfo, + catalog: ConfiguredAirbyteCatalog, + state: list[AirbyteStateMessage] | None = None, + ) -> Iterator[AirbyteMessage]: + """Call read on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteRecordMessages that come along. + * Send out telemetry on the performed sync (with information about which source was used and + the type of the cache) + """ + source_tracking_information = self.executor.get_telemetry_info() + send_telemetry(source_tracking_information, cache_info, SyncState.STARTED) + sync_failed = False + self._processed_records = 0 # Reset the counter before we start + try: + with as_temp_files( + [self._config, catalog.json(), json.dumps(state) if state else "[]"] + ) as [ + config_file, + catalog_file, + state_file, + ]: + yield from self._execute( + [ + "read", + "--config", + config_file, + "--catalog", + catalog_file, + "--state", + state_file, + ], + ) + except Exception: + send_telemetry( + source_tracking_information, cache_info, SyncState.FAILED, self._processed_records + ) + sync_failed = True + raise + finally: + if not sync_failed: + send_telemetry( + source_tracking_information, + cache_info, + SyncState.SUCCEEDED, + self._processed_records, + ) + + def _add_to_logs(self, message: str) -> None: + self._last_log_messages.append(message) + self._last_log_messages = self._last_log_messages[-10:] + + def _execute(self, args: list[str]) -> Iterator[AirbyteMessage]: + """Execute the connector with the given arguments. + + This involves the following steps: + * Locate the right venv. It is called ".venv-" + * Spawn a subprocess with .venv-/bin/ + * Read the output line by line of the subprocess and serialize them AirbyteMessage objects. + Drop if not valid. + """ + # Fail early if the connector is not installed. + self.executor.ensure_installation(auto_fix=False) + + try: + self._last_log_messages = [] + for line in self.executor.execute(args): + try: + message = AirbyteMessage.parse_raw(line) + if message.type is Type.RECORD: + self._processed_records += 1 + if message.type == Type.LOG: + self._add_to_logs(message.log.message) + if message.type == Type.TRACE and message.trace.type == TraceType.ERROR: + self._add_to_logs(message.trace.error.message) + yield message + except Exception: + self._add_to_logs(line) + except Exception as e: + raise exc.AirbyteConnectorReadError( + log_text=self._last_log_messages, + ) from e + + def _tally_records( + self, + messages: Iterable[AirbyteMessage], + ) -> Generator[AirbyteMessage, Any, None]: + """This method simply tallies the number of records processed and yields the messages.""" + self._processed_records = 0 # Reset the counter before we start + progress.reset(len(self._selected_stream_names or [])) + + for message in messages: + yield message + progress.log_records_read(self._processed_records) + + def read( + self, + cache: SQLCacheBase | None = None, + *, + write_strategy: str | WriteStrategy = WriteStrategy.AUTO, + force_full_refresh: bool = False, + ) -> ReadResult: + """Read from the connector and write to the cache. + + Args: + cache: The cache to write to. If None, a default cache will be used. + write_strategy: The strategy to use when writing to the cache. If a string, it must be + one of "append", "upsert", "replace", or "auto". If a WriteStrategy, it must be one + of WriteStrategy.APPEND, WriteStrategy.UPSERT, WriteStrategy.REPLACE, or + WriteStrategy.AUTO. + force_full_refresh: If True, the source will operate in full refresh mode. Otherwise, + streams will be read in incremental mode if supported by the connector. This option + must be True when using the "replace" strategy. + """ + if write_strategy == WriteStrategy.REPLACE and not force_full_refresh: + raise exc.AirbyteLibInputError( + message="The replace strategy requires full refresh mode.", + context={ + "write_strategy": write_strategy, + "force_full_refresh": force_full_refresh, + }, + ) + if cache is None: + cache = get_default_cache() + + if isinstance(write_strategy, str): + try: + write_strategy = WriteStrategy(write_strategy) + except ValueError: + raise exc.AirbyteLibInputError( + message="Invalid strategy", + context={ + "write_strategy": write_strategy, + "available_strategies": [s.value for s in WriteStrategy], + }, + ) from None + + if not self._selected_stream_names: + raise exc.AirbyteLibNoStreamsSelectedError( + connector_name=self.name, + available_streams=self.get_available_streams(), + ) + + cache.register_source( + source_name=self.name, + incoming_source_catalog=self.configured_catalog, + stream_names=set(self._selected_stream_names), + ) + state = cache.get_state() if not force_full_refresh else None + print(f"Started `{self.name}` read operation at {pendulum.now().format('HH:mm:ss')}...") + cache.process_airbyte_messages( + self._tally_records( + self._read( + cache.get_telemetry_info(), + state=state, + ), + ), + write_strategy=write_strategy, + ) + print(f"Completed `{self.name}` read operation at {pendulum.now().format('HH:mm:ss')}.") + + return ReadResult( + processed_records=self._processed_records, + cache=cache, + processed_streams=[stream.stream.name for stream in self.configured_catalog.streams], + ) diff --git a/airbyte-lib/airbyte_lib/strategies.py b/airbyte-lib/airbyte_lib/strategies.py new file mode 100644 index 000000000000..4d0b75a06590 --- /dev/null +++ b/airbyte-lib/airbyte_lib/strategies.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Read and write strategies for AirbyteLib.""" +from __future__ import annotations + +from enum import Enum + + +class WriteStrategy(str, Enum): + """Read strategies for AirbyteLib.""" + + MERGE = "merge" + """Merge new records with existing records. + + This requires a primary key to be set on the stream. + If no primary key is set, this will raise an exception. + + To apply this strategy in cases where some destination streams don't have a primary key, + please use the `auto` strategy instead. + """ + + APPEND = "append" + """Append new records to existing records.""" + + REPLACE = "replace" + """Replace existing records with new records.""" + + AUTO = "auto" + """Automatically determine the best strategy to use. + + This will use the following logic: + - If there's a primary key, use merge. + - Else, if there's an incremental key, use append. + - Else, use full replace (table swap). + """ diff --git a/airbyte-lib/airbyte_lib/telemetry.py b/airbyte-lib/airbyte_lib/telemetry.py new file mode 100644 index 000000000000..4e929d6d9d32 --- /dev/null +++ b/airbyte-lib/airbyte_lib/telemetry.py @@ -0,0 +1,78 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import datetime +import os +from contextlib import suppress +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any + +import requests + +from airbyte_lib.version import get_version + + +TRACKING_KEY = os.environ.get("AIRBYTE_TRACKING_KEY", "") or "cukeSffc0G6gFQehKDhhzSurDzVSZ2OP" + + +class SourceType(str, Enum): + VENV = "venv" + LOCAL_INSTALL = "local_install" + + +@dataclass +class CacheTelemetryInfo: + type: str + + +streaming_cache_info = CacheTelemetryInfo("streaming") + + +class SyncState(str, Enum): + STARTED = "started" + FAILED = "failed" + SUCCEEDED = "succeeded" + + +@dataclass +class SourceTelemetryInfo: + name: str + type: SourceType + version: str | None + + +def send_telemetry( + source_info: SourceTelemetryInfo, + cache_info: CacheTelemetryInfo, + state: SyncState, + number_of_records: int | None = None, +) -> None: + # If DO_NOT_TRACK is set, we don't send any telemetry + if os.environ.get("DO_NOT_TRACK"): + return + + current_time: str = datetime.datetime.utcnow().isoformat() # noqa: DTZ003 # prefer now() over utcnow() + payload: dict[str, Any] = { + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": asdict(source_info), + "state": state, + "cache": asdict(cache_info), + # explicitly set to 0.0.0.0 to avoid leaking IP addresses + "ip": "0.0.0.0", + "flags": { + "CI": bool(os.environ.get("CI")), + }, + }, + "timestamp": current_time, + } + if number_of_records is not None: + payload["properties"]["number_of_records"] = number_of_records + + # Suppress exceptions if host is unreachable or network is unavailable + with suppress(Exception): + # Do not handle the response, we don't want to block the execution + _ = requests.post("https://api.segment.io/v1/track", auth=(TRACKING_KEY, ""), json=payload) diff --git a/airbyte-lib/airbyte_lib/types.py b/airbyte-lib/airbyte_lib/types.py new file mode 100644 index 000000000000..a95dbf59d68e --- /dev/null +++ b/airbyte-lib/airbyte_lib/types.py @@ -0,0 +1,127 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Type conversion methods for SQL Caches.""" +from __future__ import annotations + +from typing import cast + +import sqlalchemy +from rich import print + + +# Compare to documentation here: https://docs.airbyte.com/understanding-airbyte/supported-data-types +CONVERSION_MAP = { + "string": sqlalchemy.types.VARCHAR, + "integer": sqlalchemy.types.BIGINT, + "number": sqlalchemy.types.DECIMAL, + "boolean": sqlalchemy.types.BOOLEAN, + "date": sqlalchemy.types.DATE, + "timestamp_with_timezone": sqlalchemy.types.TIMESTAMP, + "timestamp_without_timezone": sqlalchemy.types.TIMESTAMP, + "time_with_timezone": sqlalchemy.types.TIME, + "time_without_timezone": sqlalchemy.types.TIME, + # Technically 'object' and 'array' as JSON Schema types, not airbyte types. + # We include them here for completeness. + "object": sqlalchemy.types.JSON, + "array": sqlalchemy.types.JSON, +} + + +class SQLTypeConversionError(Exception): + """An exception to be raised when a type conversion fails.""" + + +def _get_airbyte_type( # noqa: PLR0911 # Too many return statements + json_schema_property_def: dict[str, str | dict | list], +) -> tuple[str, str | None]: + """Get the airbyte type and subtype from a JSON schema property definition. + + Subtype is only used for array types. Otherwise, subtype will return None. + """ + airbyte_type = cast(str, json_schema_property_def.get("airbyte_type", None)) + if airbyte_type: + return airbyte_type, None + + json_schema_type = json_schema_property_def.get("type", None) + json_schema_format = json_schema_property_def.get("format", None) + + # if json_schema_type is an array of two strings with one of them being null, pick the other one + # this strategy is often used by connectors to indicate a field might not be set all the time + if isinstance(json_schema_type, list): + non_null_types = [t for t in json_schema_type if t != "null"] + if len(non_null_types) == 1: + json_schema_type = non_null_types[0] + + if json_schema_type == "string": + if json_schema_format == "date": + return "date", None + + if json_schema_format == "date-time": + return "timestamp_with_timezone", None + + if json_schema_format == "time": + return "time_without_timezone", None + + if json_schema_type in ["string", "number", "boolean", "integer"]: + return cast(str, json_schema_type), None + + if json_schema_type == "object": + return "object", None + + if json_schema_type == "array": + items_def = json_schema_property_def.get("items", None) + if isinstance(items_def, dict): + subtype, _ = _get_airbyte_type(items_def) + return "array", subtype + + return "array", None + + err_msg = f"Could not determine airbyte type from JSON schema type: {json_schema_property_def}" + raise SQLTypeConversionError(err_msg) + + +class SQLTypeConverter: + """A base class to perform type conversions.""" + + def __init__( + self, + conversion_map: dict | None = None, + ) -> None: + self.conversion_map = conversion_map or CONVERSION_MAP + + @staticmethod + def get_failover_type() -> sqlalchemy.types.TypeEngine: + """Get the 'last resort' type to use if no other type is found.""" + return sqlalchemy.types.VARCHAR() + + def to_sql_type( + self, + json_schema_property_def: dict[str, str | dict | list], + ) -> sqlalchemy.types.TypeEngine: + """Convert a value to a SQL type.""" + try: + airbyte_type, _ = _get_airbyte_type(json_schema_property_def) + return self.conversion_map[airbyte_type]() + except SQLTypeConversionError: + print(f"Could not determine airbyte type from JSON schema: {json_schema_property_def}") + except KeyError: + print(f"Could not find SQL type for airbyte type: {airbyte_type}") + + json_schema_type = json_schema_property_def.get("type", None) + json_schema_format = json_schema_property_def.get("format", None) + + if json_schema_type == "string" and json_schema_format == "date": + return sqlalchemy.types.DATE() + + if json_schema_type == "string" and json_schema_format == "date-time": + return sqlalchemy.types.TIMESTAMP() + + if json_schema_type == "array": + # TODO: Implement array type conversion. + return self.get_failover_type() + + if json_schema_type == "object": + # TODO: Implement object type handling. + return self.get_failover_type() + + return self.get_failover_type() diff --git a/airbyte-lib/airbyte_lib/validate.py b/airbyte-lib/airbyte_lib/validate.py new file mode 100644 index 000000000000..243556cd4020 --- /dev/null +++ b/airbyte-lib/airbyte_lib/validate.py @@ -0,0 +1,159 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Defines the `airbyte-lib-validate-source` CLI. + +This tool checks if connectors are compatible with airbyte-lib. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml +from rich import print + +import airbyte_lib as ab +from airbyte_lib import exceptions as exc + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate a connector") + parser.add_argument( + "--connector-dir", + type=str, + required=True, + help="Path to the connector directory", + ) + parser.add_argument( + "--validate-install-only", + action="store_true", + help="Only validate that the connector can be installed and config can be validated.", + ) + parser.add_argument( + "--sample-config", + type=str, + required=False, + help="Path to the sample config.json file. Required without --validate-install-only.", + ) + return parser.parse_args() + + +def _run_subprocess_and_raise_on_failure(args: list[str]) -> None: + result = subprocess.run( + args, + check=False, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=result.returncode, + log_text=result.stderr.decode("utf-8"), + ) + + +def full_tests(connector_name: str, sample_config: str) -> None: + print("Creating source and validating spec and version...") + source = ab.get_source( + # TODO: FIXME: noqa: SIM115, PTH123 + connector_name, + config=json.load(open(sample_config)), # noqa: SIM115, PTH123, + install_if_missing=False, + ) + + print("Running check...") + source.check() + + print("Fetching streams...") + streams = source.get_available_streams() + + # try to peek all streams - if one works, stop, if none works, throw exception + for stream in streams: + try: + print(f"Trying to read from stream {stream}...") + record = next(source.get_records(stream)) + assert record, "No record returned" + break + except exc.AirbyteError as e: + print(f"Could not read from stream {stream}: {e}") + except Exception as e: + print(f"Unhandled error occurred when trying to read from {stream}: {e}") + else: + raise exc.AirbyteNoDataFromConnectorError( + context={"selected_streams": streams}, + ) + + +def install_only_test(connector_name: str) -> None: + print("Creating source and validating spec is returned successfully...") + source = ab.get_source(connector_name) + source._get_spec(force_refresh=True) # noqa: SLF001 + + +def run() -> None: + """Handle CLI entrypoint for the `airbyte-lib-validate-source` command. + + It's called like this: + > airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.json + + It performs a basic smoke test to make sure the connector in question is airbyte-lib compliant: + * Can be installed into a venv + * Can be called via cli entrypoint + * Answers according to the Airbyte protocol when called with spec, check, discover and read. + """ + # parse args + args = _parse_args() + connector_dir = args.connector_dir + sample_config = args.sample_config + validate_install_only = args.validate_install_only + validate(connector_dir, sample_config, validate_install_only=validate_install_only) + + +def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None: + # read metadata.yaml + metadata_path = Path(connector_dir) / "metadata.yaml" + with Path(metadata_path).open() as stream: + metadata = yaml.safe_load(stream)["data"] + + # TODO: Use remoteRegistries.pypi.packageName once set for connectors + connector_name = metadata["dockerRepository"].replace("airbyte/", "") + + # create a venv and install the connector + venv_name = f".venv-{connector_name}" + venv_path = Path(venv_name) + if not venv_path.exists(): + _run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) + + pip_path = str(venv_path / "bin" / "pip") + + _run_subprocess_and_raise_on_failure([pip_path, "install", connector_dir]) + + # write basic registry to temp json file + registry = { + "sources": [ + { + "dockerRepository": f"airbyte/{connector_name}", + "dockerImageTag": "0.0.1", + "remoteRegistries": { + "pypi": {"packageName": "airbyte-{connector_name}", "enabled": True} + }, + }, + ], + } + + with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file: + temp_file.write(json.dumps(registry)) + temp_file.seek(0) + os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) + if validate_install_only: + install_only_test(connector_name) + else: + if not sample_config: + raise exc.AirbyteLibInputError( + input_value="--sample-config is required without --validate-install-only set" + ) + full_tests(connector_name, sample_config) diff --git a/airbyte-lib/airbyte_lib/version.py b/airbyte-lib/airbyte_lib/version.py new file mode 100644 index 000000000000..114a730a5e7c --- /dev/null +++ b/airbyte-lib/airbyte_lib/version.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import importlib.metadata + + +airbyte_lib_version = importlib.metadata.version("airbyte-lib") + + +def get_version() -> str: + return airbyte_lib_version diff --git a/airbyte-lib/docs.py b/airbyte-lib/docs.py new file mode 100644 index 000000000000..be5dea69b9ef --- /dev/null +++ b/airbyte-lib/docs.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import os +import pathlib +import shutil + +import pdoc + + +def run() -> None: + """Generate docs for all public modules in airbyte_lib and save them to docs/generated. + + Public modules are: + * The main airbyte_lib module + * All directory modules in airbyte_lib that don't start with an underscore. + """ + public_modules = ["airbyte_lib"] + + # recursively delete the docs/generated folder if it exists + if pathlib.Path("docs/generated").exists(): + shutil.rmtree("docs/generated") + + # All folders in `airbyte_lib` that don't start with "_" are treated as public modules. + for d in os.listdir("airbyte_lib"): + dir_path = pathlib.Path(f"airbyte_lib/{d}") + if dir_path.is_dir() and not d.startswith("_") and (dir_path / "__init__.py").exists(): + public_modules.append(dir_path) + + pdoc.render.configure(template_directory="docs", show_source=False, search=False) + pdoc.pdoc(*public_modules, output_directory=pathlib.Path("docs/generated")) diff --git a/airbyte-lib/docs/frame.html.jinja2 b/airbyte-lib/docs/frame.html.jinja2 new file mode 100644 index 000000000000..379ae376725f --- /dev/null +++ b/airbyte-lib/docs/frame.html.jinja2 @@ -0,0 +1,14 @@ + +
      + {% block module_contents %}{% endblock %} +
      + +{% filter minify_css %} + {% block style %} + {# The same CSS files as in pdoc's default template, except for layout.css. + You may leave out Bootstrap Reboot, which corrects inconsistences across browsers + but may conflict with you website's stylesheet. #} + + + {% endblock %} +{% endfilter %} diff --git a/airbyte-lib/docs/generated/airbyte_lib.html b/airbyte-lib/docs/generated/airbyte_lib.html new file mode 100644 index 000000000000..5c7d77861523 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib.html @@ -0,0 +1,889 @@ + +
      +
      +
      + + class + CachedDataset(airbyte_lib.datasets._sql.SQLDataset): + + +
      + + +

      A dataset backed by a SQL table cache.

      + +

      Because this dataset includes all records from the underlying table, we also expose the +underlying table as a SQLAlchemy Table object.

      +
      + + +
      +
      + + CachedDataset(cache: 'SQLCacheBase', stream_name: str) + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      + +

      The base implementation simply passes the record iterator to Panda's DataFrame constructor.

      +
      + + +
      +
      +
      + + def + to_sql_table(self) -> 'Table': + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      airbyte_lib.datasets._sql.SQLDataset
      +
      stream_name
      +
      with_filter
      + +
      +
      +
      +
      +
      +
      + + class + DuckDBCache(airbyte_lib.caches.duckdb.DuckDBCacheBase): + + +
      + + +

      A DuckDB implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

      +
      + + +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      airbyte_lib.caches.base.SQLCacheBase
      +
      SQLCacheBase
      +
      type_converter_class
      +
      use_singleton_connection
      +
      config
      +
      file_writer
      +
      type_converter
      +
      get_sql_alchemy_url
      +
      database_name
      +
      get_sql_engine
      +
      get_sql_connection
      +
      get_sql_table_name
      +
      get_sql_table
      +
      streams
      +
      get_records
      +
      get_pandas_dataframe
      +
      get_state
      +
      register_source
      + +
      +
      airbyte_lib.caches.duckdb.DuckDBCacheBase
      +
      config_class
      +
      supports_merge_insert
      +
      get_telemetry_info
      + +
      +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + DuckDBCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the DuckDB cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + db_path: pathlib.Path | str + + +
      + + +

      Normally db_path is a Path object.

      + +

      There are some cases, such as when connecting to MotherDuck, where it could be a string that +is not also a path, such as "md:" to connect the user's default MotherDuck DB.

      +
      + + +
      +
      +
      + schema_name: str + + +
      + + +

      The name of the schema to write to. Defaults to "main".

      +
      + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + def + get_available_connectors() -> list[str]: + + +
      + + +

      Return a list of all available connectors.

      + +

      Connectors will be returned in alphabetical order, with the standard prefix "source-".

      +
      + + +
      +
      +
      + + def + get_source( name: str, config: dict[str, typing.Any] | None = None, *, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> Source: + + +
      + + +

      Get a connector by name and version.

      + +

      Args: + name: connector name + config: connector config - if not provided, you need to set it later via the set_config + method. + version: connector version - if not provided, the currently installed version will be used. + If no version is installed, the latest available version will be used. The version can + also be set to "latest" to force the use of the latest available version. + pip_url: connector pip URL - if not provided, the pip url will be inferred from the + connector name. + local_executable: If set, the connector will be assumed to already be installed and will be + executed using this path or executable name. Otherwise, the connector will be installed + automatically in a virtual environment. + install_if_missing: Whether to install the connector if it is not available locally. This + parameter is ignored when local_executable is set.

      +
      + + +
      +
      +
      + + def + get_default_cache() -> DuckDBCache: + + +
      + + +

      Get a local cache for storing data, using the default database path.

      + +

      Cache files are stored in the .cache directory, relative to the current +working directory.

      +
      + + +
      +
      +
      + + def + get_secret( secret_name: str, source: SecretSource | list[SecretSource] = <SecretSource.ANY: 4>, *, prompt: bool = True) -> str: + + +
      + + +

      Get a secret from the environment.

      + +

      The optional source argument of enum type SecretSource or list of SecretSource options. +If left blank, the source arg will be SecretSource.ANY. If source is set to a specific +source, then only that source will be checked. If a list of SecretSource entries is passed, +then the sources will be checked using the provided ordering.

      + +

      If prompt to True or if SecretSource.PROMPT is declared in the source arg, then the +user will be prompted to enter the secret if it is not found in any of the other sources.

      +
      + + +
      +
      +
      + + def + new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> DuckDBCache: + + +
      + + +

      Get a local cache for storing data, using a name string to seed the path.

      + +

      Args: + cache_name: Name to use for the cache. Defaults to None. + cache_dir: Root directory to store the cache in. Defaults to None. + cleanup: Whether to clean up temporary files. Defaults to True.

      + +

      Cache files are stored in the .cache directory, relative to the current +working directory.

      +
      + + +
      +
      +
      + + class + ReadResult(collections.abc.Mapping[str, airbyte_lib.datasets._sql.CachedDataset]): + + +
      + + +

      A Mapping is a generic container for associating key/value +pairs.

      + +

      This class provides concrete generic implementations of all +methods except for __getitem__, __iter__, and __len__.

      +
      + + +
      +
      + + ReadResult( processed_records: int, cache: airbyte_lib.caches.base.SQLCacheBase, processed_streams: list[str]) + + +
      + + + + +
      +
      +
      + processed_records + + +
      + + + + +
      +
      +
      + + def + get_sql_engine(self) -> sqlalchemy.engine.base.Engine: + + +
      + + + + +
      +
      +
      + streams: collections.abc.Mapping[str, CachedDataset] + + +
      + + + + +
      +
      +
      + cache: airbyte_lib.caches.base.SQLCacheBase + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      collections.abc.Mapping
      +
      get
      +
      keys
      +
      items
      +
      values
      + +
      +
      +
      +
      +
      +
      + + class + SecretSource(enum.Enum): + + +
      + + +

      An enumeration.

      +
      + + +
      +
      + ENV = +<SecretSource.ENV: 1> + + +
      + + + + +
      +
      +
      + DOTENV = +<SecretSource.DOTENV: 2> + + +
      + + + + +
      +
      +
      + GOOGLE_COLAB = +<SecretSource.GOOGLE_COLAB: 3> + + +
      + + + + +
      +
      +
      + ANY = +<SecretSource.ANY: 4> + + +
      + + + + +
      +
      +
      + PROMPT = +<SecretSource.PROMPT: 5> + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      enum.Enum
      +
      name
      +
      value
      + +
      +
      +
      +
      +
      +
      + + class + Source: + + +
      + + +

      A class representing a source that can be called.

      +
      + + +
      +
      + + Source( executor: airbyte_lib._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None, *, validate: bool = False) + + +
      + + +

      Initialize the source.

      + +

      If config is provided, it will be validated against the spec if validate is True.

      +
      + + +
      +
      +
      + executor + + +
      + + + + +
      +
      +
      + name + + +
      + + + + +
      +
      +
      + + def + set_streams(self, streams: list[str]) -> None: + + +
      + + +

      Deprecated. See select_streams().

      +
      + + +
      +
      +
      + + def + select_all_streams(self) -> None: + + +
      + + +

      Select all streams.

      + +

      This is a more streamlined equivalent to:

      + +
      +

      source.select_streams(source.get_available_streams()).

      +
      +
      + + +
      +
      +
      + + def + select_streams(self, streams: list[str]) -> None: + + +
      + + +

      Select the stream names that should be read from the connector.

      + +

      Currently, if this is not set, all streams will be read.

      +
      + + +
      +
      +
      + + def + get_selected_streams(self) -> list[str]: + + +
      + + +

      Get the selected streams.

      + +

      If no streams are selected, return an empty list.

      +
      + + +
      +
      +
      + + def + set_config(self, config: dict[str, typing.Any], *, validate: bool = False) -> None: + + +
      + + +

      Set the config for the connector.

      + +

      If validate is True, raise an exception if the config fails validation.

      + +

      If validate is False, validation will be deferred until check() or validate_config() +is called.

      +
      + + +
      +
      +
      + + def + get_config(self) -> dict[str, typing.Any]: + + +
      + + +

      Get the config for the connector.

      +
      + + +
      +
      +
      + + def + validate_config(self, config: dict[str, typing.Any] | None = None) -> None: + + +
      + + +

      Validate the config against the spec.

      + +

      If config is not provided, the already-set config will be validated.

      +
      + + +
      +
      +
      + + def + get_available_streams(self) -> list[str]: + + +
      + + +

      Get the available streams from the spec.

      +
      + + +
      +
      +
      + docs_url: str + + +
      + + +

      Get the URL to the connector's documentation.

      +
      + + +
      +
      +
      + discovered_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog + + +
      + + +

      Get the raw catalog for the given streams.

      + +

      If the catalog is not yet known, we call discover to get it.

      +
      + + +
      +
      +
      + configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog + + +
      + + +

      Get the configured catalog for the given streams.

      + +

      If the raw catalog is not yet known, we call discover to get it.

      + +

      If no specific streams are selected, we return a catalog that syncs all available streams.

      + +

      TODO: We should consider disabling by default the streams that the connector would +disable by default. (For instance, streams that require a premium license are sometimes +disabled by default within the connector.)

      +
      + + +
      +
      +
      + + def + get_records(self, stream: str) -> airbyte_lib.datasets._lazy.LazyDataset: + + +
      + + +

      Read a stream from the connector.

      + +

      This involves the following steps:

      + +
        +
      • Call discover to get the catalog
        • +
      • Generate a configured catalog that syncs the given stream in full_refresh mode
        • +
      • Write the configured catalog and the config to a temporary file
        • +
      • execute the connector with read --config --catalog
        • +
      • Listen to the messages and return the first AirbyteRecordMessages that come along.
        • +
      • Make sure the subprocess is killed when the function returns.
        • +
      +
      + + +
      +
      +
      + + def + check(self) -> None: + + +
      + + +

      Call check on the connector.

      + +

      This involves the following steps:

      + +
        +
      • Write the config to a temporary file
        • +
      • execute the connector with check --config
        • +
      • Listen to the messages and return the first AirbyteCatalog that comes along.
        • +
      • Make sure the subprocess is killed when the function returns.
        • +
      +
      + + +
      +
      +
      + + def + install(self) -> None: + + +
      + + +

      Install the connector if it is not yet installed.

      +
      + + +
      +
      +
      + + def + uninstall(self) -> None: + + +
      + + +

      Uninstall the connector if it is installed.

      + +

      This only works if the use_local_install flag wasn't used and installation is managed by +airbyte-lib.

      +
      + + +
      +
      +
      + + def + read( self, cache: airbyte_lib.caches.base.SQLCacheBase | None = None, *, write_strategy: str | airbyte_lib.strategies.WriteStrategy = <WriteStrategy.AUTO: 'auto'>, force_full_refresh: bool = False) -> ReadResult: + + +
      + + +

      Read from the connector and write to the cache.

      + +

      Args: + cache: The cache to write to. If None, a default cache will be used. + write_strategy: The strategy to use when writing to the cache. If a string, it must be + one of "append", "upsert", "replace", or "auto". If a WriteStrategy, it must be one + of WriteStrategy.APPEND, WriteStrategy.UPSERT, WriteStrategy.REPLACE, or + WriteStrategy.AUTO. + force_full_refresh: If True, the source will operate in full refresh mode. Otherwise, + streams will be read in incremental mode if supported by the connector. This option + must be True when using the "replace" strategy.

      +
      + + +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/airbyte_lib/caches.html b/airbyte-lib/docs/generated/airbyte_lib/caches.html new file mode 100644 index 000000000000..cf1eb7276567 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib/caches.html @@ -0,0 +1,992 @@ + +
      +
      +
      + + class + DuckDBCache(airbyte_lib.caches.duckdb.DuckDBCacheBase): + + +
      + + +

      A DuckDB implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

      +
      + + +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib.caches.duckdb.DuckDBCacheBase
      +
      config_class
      +
      supports_merge_insert
      +
      get_telemetry_info
      + +
      +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + DuckDBCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the DuckDB cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + db_path: pathlib.Path | str + + +
      + + +

      Normally db_path is a Path object.

      + +

      There are some cases, such as when connecting to MotherDuck, where it could be a string that +is not also a path, such as "md:" to connect the user's default MotherDuck DB.

      +
      + + +
      +
      +
      + schema_name: str + + +
      + + +

      The name of the schema to write to. Defaults to "main".

      +
      + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + PostgresCache(airbyte_lib.caches.SQLCacheBase): + + +
      + + +

      A Postgres implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

      + +

      TOOD: Add optimized bulk load path for Postgres. Could use an alternate file writer +or another import method. (Relatively low priority, since for now it works fine as-is.)

      +
      + + +
      +
      + config_class = +<class 'PostgresCacheConfig'> + + +
      + + + + +
      +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      + supports_merge_insert = +False + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + PostgresCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the Postgres cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + host: str + + +
      + + + + +
      +
      +
      + port: int + + +
      + + + + +
      +
      +
      + username: str + + +
      + + + + +
      +
      +
      + password: str + + +
      + + + + +
      +
      +
      + database: str + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      schema_name
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + SQLCacheBase(airbyte_lib._processors.RecordProcessor): + + +
      + + +

      A base class to be used for SQL Caches.

      + +

      Optionally we can use a file cache to store the data in parquet files.

      +
      + + +
      +
      + type_converter_class: type[airbyte_lib.types.SQLTypeConverter] = +<class 'airbyte_lib.types.SQLTypeConverter'> + + +
      + + + + +
      +
      +
      + config_class: type[airbyte_lib.caches.base.SQLCacheConfigBase] + + +
      + + + + +
      +
      +
      + file_writer_class: type[airbyte_lib._file_writers.base.FileWriterBase] + + +
      + + + + +
      +
      +
      + supports_merge_insert = +False + + +
      + + + + +
      +
      +
      + use_singleton_connection = +False + + +
      + + + + +
      +
      +
      + config: airbyte_lib.caches.base.SQLCacheConfigBase + + +
      + + + + +
      +
      +
      + file_writer + + +
      + + + + +
      +
      +
      + type_converter + + +
      + + + + +
      +
      +
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + database_name: str + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      +
      @final
      + + def + get_sql_engine(self) -> sqlalchemy.engine.base.Engine: + + +
      + + +

      Return a new SQL engine to use.

      +
      + + +
      +
      +
      +
      @contextmanager
      + + def + get_sql_connection( self) -> collections.abc.Generator[sqlalchemy.engine.base.Connection, None, None]: + + +
      + + +

      A context manager which returns a new SQL connection for running queries.

      + +

      If the connection needs to close, it will be closed automatically.

      +
      + + +
      +
      +
      + + def + get_sql_table_name(self, stream_name: str) -> str: + + +
      + + +

      Return the name of the SQL table for the given stream.

      +
      + + +
      +
      +
      +
      @final
      + + def + get_sql_table(self, stream_name: str) -> sqlalchemy.sql.schema.Table: + + +
      + + +

      Return the main table object for the stream.

      +
      + + +
      +
      +
      + streams: dict[str, airbyte_lib.datasets._sql.CachedDataset] + + +
      + + +

      Return a temporary table name.

      +
      + + +
      +
      +
      + + def + get_records(self, stream_name: str) -> airbyte_lib.datasets._sql.CachedDataset: + + +
      + + +

      Uses SQLAlchemy to select all rows from the table.

      +
      + + +
      +
      +
      + + def + get_pandas_dataframe(self, stream_name: str) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a Pandas data frame with the stream's data.

      +
      + + +
      +
      +
      + + def + get_state(self) -> list[dict]: + + +
      + + +

      Return the current state of the source.

      +
      + + +
      +
      +
      +
      @overrides
      + + def + register_source( self, source_name: str, incoming_source_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog, stream_names: set[str]) -> None: + + +
      + + +

      Register the source with the cache.

      + +

      We use stream_names to determine which streams will receive data, and +we only register the stream if is expected to receive data.

      + +

      This method is called by the source when it is initialized.

      +
      + + +
      +
      +
      +
      @abc.abstractmethod
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + SnowflakeCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the Snowflake cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + account: str + + +
      + + + + +
      +
      +
      + username: str + + +
      + + + + +
      +
      +
      + password: str + + +
      + + + + +
      +
      +
      + warehouse: str + + +
      + + + + +
      +
      +
      + database: str + + +
      + + + + +
      +
      +
      + role: str + + +
      + + + + +
      +
      +
      + dedupe_mode + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      schema_name
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + SnowflakeSQLCache(airbyte_lib.caches.SQLCacheBase): + + +
      + + +

      A Snowflake implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading.

      +
      + + +
      +
      + config_class = +<class 'SnowflakeCacheConfig'> + + +
      + + + + +
      +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      + type_converter_class = +<class 'airbyte_lib.caches.snowflake.SnowflakeTypeConverter'> + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/airbyte_lib/datasets.html b/airbyte-lib/docs/generated/airbyte_lib/datasets.html new file mode 100644 index 000000000000..76089344eca0 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib/datasets.html @@ -0,0 +1,258 @@ + +
      +
      +
      + + class + CachedDataset(airbyte_lib.datasets.SQLDataset): + + +
      + + +

      A dataset backed by a SQL table cache.

      + +

      Because this dataset includes all records from the underlying table, we also expose the +underlying table as a SQLAlchemy Table object.

      +
      + + +
      +
      + + CachedDataset(cache: 'SQLCacheBase', stream_name: str) + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      + +

      The base implementation simply passes the record iterator to Panda's DataFrame constructor.

      +
      + + +
      +
      +
      + + def + to_sql_table(self) -> 'Table': + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      +
      +
      +
      +
      + + class + DatasetBase(abc.ABC): + + +
      + + +

      Base implementation for all datasets.

      +
      + + +
      +
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      + +

      The base implementation simply passes the record iterator to Panda's DataFrame constructor.

      +
      + + +
      +
      +
      +
      + + class + DatasetMap(collections.abc.Mapping): + + +
      + + +

      A generic interface for a set of streams or datasets.

      +
      + + +
      +
      Inherited Members
      +
      +
      collections.abc.Mapping
      +
      get
      +
      keys
      +
      items
      +
      values
      + +
      +
      +
      +
      +
      +
      + + class + LazyDataset(airbyte_lib.datasets.DatasetBase): + + +
      + + +

      A dataset that is loaded incrementally from a source or a SQL query.

      +
      + + +
      +
      + + LazyDataset( iterator: collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]) + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      +
      +
      +
      +
      + + class + SQLDataset(airbyte_lib.datasets.DatasetBase): + + +
      + + +

      A dataset that is loaded incrementally from a SQL query.

      + +

      The CachedDataset class is a subclass of this class, which simply passes a SELECT over the full +table as the query statement.

      +
      + + +
      +
      + + SQLDataset( cache: 'SQLCacheBase', stream_name: str, query_statement: 'Selectable') + + +
      + + + + +
      +
      +
      + stream_name: str + + +
      + + + + +
      +
      +
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      + +

      The base implementation simply passes the record iterator to Panda's DataFrame constructor.

      +
      + + +
      +
      +
      + + def + with_filter( self, *filter_expressions: 'ClauseElement | str') -> SQLDataset: + + +
      + + +

      Filter the dataset by a set of column values.

      + +

      Filters can be specified as either a string or a SQLAlchemy expression.

      + +

      Filters are lazily applied to the dataset, so they can be chained together. For example:

      + +
          dataset.with_filter("id > 5").with_filter("id < 10")
+
      + +

      is equivalent to:

      + +
          dataset.with_filter("id > 5", "id < 10")
+
      +
      + + +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/index.html b/airbyte-lib/docs/generated/index.html new file mode 100644 index 000000000000..6dfc876b8f9c --- /dev/null +++ b/airbyte-lib/docs/generated/index.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/airbyte-lib/examples/run_faker.py b/airbyte-lib/examples/run_faker.py new file mode 100644 index 000000000000..758cd07e2216 --- /dev/null +++ b/airbyte-lib/examples/run_faker.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""A simple test of AirbyteLib, using the Faker source connector. + +Usage (from airbyte-lib root directory): +> poetry run python ./examples/run_faker.py + +No setup is needed, but you may need to delete the .venv-source-faker folder +if your installation gets interrupted or corrupted. +""" +from __future__ import annotations + +import airbyte_lib as ab + + +SCALE = 500_000 # Number of records to generate between users and purchases. + +# This is a dummy secret, just to test functionality. +DUMMY_SECRET = ab.get_secret("DUMMY_SECRET") + + +print("Installing Faker source...") +source = ab.get_source( + "source-faker", + config={"count": SCALE / 2}, + install_if_missing=True, +) +print("Faker source installed.") +source.check() +source.select_streams(["products", "users", "purchases"]) + +result = source.read() + +for name, records in result.streams.items(): + print(f"Stream {name}: {len(records)} records") diff --git a/airbyte-lib/examples/run_github.py b/airbyte-lib/examples/run_github.py new file mode 100644 index 000000000000..253e1275a541 --- /dev/null +++ b/airbyte-lib/examples/run_github.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""A simple test of AirbyteLib, using the Faker source connector. + +Usage (from airbyte-lib root directory): +> poetry run python ./examples/run_github.py + +No setup is needed, but you may need to delete the .venv-source-faker folder +if your installation gets interrupted or corrupted. +""" +from __future__ import annotations + +import airbyte_lib as ab + + +# Create a token here: https://github.com/settings/tokens +GITHUB_TOKEN = ab.get_secret("GITHUB_PERSONAL_ACCESS_TOKEN") + + +source = ab.get_source("source-github") +source.set_config( + { + "repositories": ["airbytehq/airbyte-lib-private-beta"], + "credentials": {"personal_access_token": GITHUB_TOKEN}, + } +) +source.check() +source.select_streams(["issues", "pull_requests", "commits", "collaborators", "deployments"]) + +result = source.read(cache=ab.new_local_cache("github")) +print(result.processed_records) + +for name, records in result.streams.items(): + print(f"Stream {name}: {len(records)} records") diff --git a/airbyte-lib/examples/run_pokeapi.py b/airbyte-lib/examples/run_pokeapi.py new file mode 100644 index 000000000000..9b710bd625fa --- /dev/null +++ b/airbyte-lib/examples/run_pokeapi.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""A simple test of AirbyteLib, using the PokeAPI source connector. + +Usage (from airbyte-lib root directory): +> poetry run python ./examples/run_pokeapi.py + +No setup is needed, but you may need to delete the .venv-source-pokeapi folder +if your installation gets interrupted or corrupted. +""" +from __future__ import annotations + +import airbyte_lib as ab + + +source = ab.get_source( + "source-pokeapi", + config={"pokemon_name": "bulbasaur"}, + install_if_missing=True, +) +source.check() + +# print(list(source.get_records("pokemon"))) +source.read(cache=ab.new_local_cache("poke")) diff --git a/airbyte-lib/examples/run_snowflake_faker.py b/airbyte-lib/examples/run_snowflake_faker.py new file mode 100644 index 000000000000..56d8af8f10ef --- /dev/null +++ b/airbyte-lib/examples/run_snowflake_faker.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import os + +from google.cloud import secretmanager + +import airbyte_lib as ab +from airbyte_lib.caches import SnowflakeCacheConfig, SnowflakeSQLCache + + +source = ab.get_source( + "source-faker", + config={"count": 10000, "seed": 0, "parallelism": 1, "always_updated": False}, + install_if_missing=True, +) + +# load secrets from GSM using the GCP_GSM_CREDENTIALS env variable +secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( + json.loads(os.environ["GCP_GSM_CREDENTIALS"]) +) +secret = json.loads( + secret_client.access_secret_version( + name="projects/dataline-integration-testing/secrets/AIRBYTE_LIB_SNOWFLAKE_CREDS/versions/latest" + ).payload.data.decode("UTF-8") +) + +cache = SnowflakeSQLCache( + SnowflakeCacheConfig( + account=secret["account"], + username=secret["username"], + password=secret["password"], + database=secret["database"], + warehouse=secret["warehouse"], + role=secret["role"], + ) +) + +source.check() + +source.select_streams(["products"]) +result = source.read(cache) + +for name in ["products"]: + print(f"Stream {name}: {len(list(result[name]))} records") diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py new file mode 100644 index 000000000000..f2695d7ff695 --- /dev/null +++ b/airbyte-lib/examples/run_spacex.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from itertools import islice + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-spacex-api +# source .venv-source-spacex-api/bin/activate +# pip install -e ../airbyte-integrations/connectors/source-spacex-api +# In separate terminal: +# poetry run python examples/run_spacex.py + +source = ab.get_source( + "source-spacex-api", + config={"id": "605b4b6aaa5433645e37d03f"}, + install_if_missing=True, +) +cache = ab.new_local_cache() + +source.check() + +source.select_streams(["launches", "rockets", "capsules"]) + +result = source.read(cache) + +print(islice(source.get_records("capsules"), 10)) + +for name, records in result.cache.streams.items(): + print(f"Stream {name}: {len(list(records))} records") diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py new file mode 100644 index 000000000000..e448f0f8b96c --- /dev/null +++ b/airbyte-lib/examples/run_test_source.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import os + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-test +# source .venv-source-test/bin/activate +# pip install -e ./tests/integration_tests/fixtures/source-test +# In separate terminal: +# poetry run python examples/run_test_source.py + +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + +source = ab.get_source("source-test", config={"apiKey": "test"}) +cache = ab.new_local_cache("cache_test") + +source.check() + +print(source.get_available_streams()) + +result = source.read(cache) + +print(result.processed_records) +print(list(result["stream1"])) + +different_cache = ab.new_local_cache("cache_test") +print(list(different_cache["stream1"])) diff --git a/airbyte-lib/examples/run_test_source_single_stream.py b/airbyte-lib/examples/run_test_source_single_stream.py new file mode 100644 index 000000000000..9b695979c833 --- /dev/null +++ b/airbyte-lib/examples/run_test_source_single_stream.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import os + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-test +# source .venv-source-test/bin/activate +# pip install -e ./tests/integration_tests/fixtures/source-test +# In separate terminal: +# poetry run python examples/run_test_source.py + +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + +source = ab.get_source("source-test", config={"apiKey": "test"}) + +print(list(source.read_stream("stream1"))) diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock new file mode 100644 index 000000000000..86d14143870c --- /dev/null +++ b/airbyte-lib/poetry.lock @@ -0,0 +1,2687 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.9" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.9.tar.gz", hash = "sha256:e749bd4aab0911bd93c710e3ab2fcdde45d7a0bed2c0032d873006d3df701478"}, + {file = "airbyte_cdk-0.58.9-py3-none-any.whl", hash = "sha256:45dfbac2d0ae86dd5872c07c140ce16be8481452b7b8f65b228bc9f892843871"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "airbyte-source-faker" +version = "6.0.1" +description = "Source implementation for fake but realistic looking data." +optional = false +python-versions = "*" +files = [ + {file = "airbyte-source-faker-6.0.1.tar.gz", hash = "sha256:8173a48551fbfe0eb6e9c331fec650fa490f283736aef0d58e2f14e55f8cf90a"}, + {file = "airbyte_source_faker-6.0.1-py3-none-any.whl", hash = "sha256:622cd123589218cffe69755727addfe85873d7563002cf8d5f949586604e0d9f"}, +] + +[package.dependencies] +airbyte-cdk = ">=0.2,<1.0" +mimesis = "6.1.1" + +[package.extras] +tests = ["pytest (>=6.2,<7.0)", "pytest-mock (>=3.6.1,<3.7.0)", "requests-mock (>=1.9.3,<1.10.0)"] + +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +optional = false +python-versions = "*" +files = [ + {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, + {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "41.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "docker" +version = "7.0.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.0.0-py3-none-any.whl", hash = "sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b"}, + {file = "docker-7.0.0.tar.gz", hash = "sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "duckdb" +version = "0.10.0" +description = "DuckDB in-process database" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "duckdb-0.10.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bd0ffb3fddef0f72a150e4d76e10942a84a1a0447d10907df1621b90d6668060"}, + {file = "duckdb-0.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f3d709d5c7c1a12b5e10d0b05fa916c670cd2b50178e3696faa0cc16048a1745"}, + {file = "duckdb-0.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9114aa22ec5d591a20ce5184be90f49d8e5b5348ceaab21e102c54560d07a5f8"}, + {file = "duckdb-0.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a37877efadf39caf7cadde0f430fedf762751b9c54750c821e2f1316705a21"}, + {file = "duckdb-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87cbc9e1d9c3fc9f14307bea757f99f15f46843c0ab13a6061354410824ed41f"}, + {file = "duckdb-0.10.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0bfec79fed387201550517d325dff4fad2705020bc139d936cab08b9e845662"}, + {file = "duckdb-0.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c5622134d2d9796b15e09de810e450859d4beb46d9b861357ec9ae40a61b775c"}, + {file = "duckdb-0.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:089ee8e831ccaef1b73fc89c43b661567175eed0115454880bafed5e35cda702"}, + {file = "duckdb-0.10.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a05af63747f1d7021995f0811c333dee7316cec3b06c0d3e4741b9bdb678dd21"}, + {file = "duckdb-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:072d6eba5d8a59e0069a8b5b4252fed8a21f9fe3f85a9129d186a39b3d0aea03"}, + {file = "duckdb-0.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a77b85668f59b919042832e4659538337f1c7f197123076c5311f1c9cf077df7"}, + {file = "duckdb-0.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96a666f1d2da65d03199a977aec246920920a5ea1da76b70ae02bd4fb1ffc48c"}, + {file = "duckdb-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ec76a4262b783628d26612d184834852d9c92fb203e91af789100c17e3d7173"}, + {file = "duckdb-0.10.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:009dd9d2cdbd3b061a9efbdfc79f2d1a8377bcf49f1e5f430138621f8c083a6c"}, + {file = "duckdb-0.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:878f06766088090dad4a2e5ee0081555242b2e8dcb29415ecc97e388cf0cf8d8"}, + {file = "duckdb-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:713ff0a1fb63a6d60f454acf67f31656549fb5d63f21ac68314e4f522daa1a89"}, + {file = "duckdb-0.10.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9c0ee450dfedfb52dd4957244e31820feef17228da31af6d052979450a80fd19"}, + {file = "duckdb-0.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ff79b2ea9994398b545c0d10601cd73565fbd09f8951b3d8003c7c5c0cebc7cb"}, + {file = "duckdb-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6bdf1aa71b924ef651062e6b8ff9981ad85bec89598294af8a072062c5717340"}, + {file = "duckdb-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0265bbc8216be3ced7b377ba8847128a3fc0ef99798a3c4557c1b88e3a01c23"}, + {file = "duckdb-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d418a315a07707a693bd985274c0f8c4dd77015d9ef5d8d3da4cc1942fd82e0"}, + {file = "duckdb-0.10.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2828475a292e68c71855190b818aded6bce7328f79e38c04a0c75f8f1c0ceef0"}, + {file = "duckdb-0.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c3aaeaae2eba97035c65f31ffdb18202c951337bf2b3d53d77ce1da8ae2ecf51"}, + {file = "duckdb-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:c51790aaaea97d8e4a58a114c371ed8d2c4e1ca7cbf29e3bdab6d8ccfc5afc1e"}, + {file = "duckdb-0.10.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8af1ae7cc77a12206b6c47ade191882cc8f49f750bb3e72bb86ac1d4fa89926a"}, + {file = "duckdb-0.10.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa4f7e8e8dc0e376aeb280b83f2584d0e25ec38985c27d19f3107b2edc4f4a97"}, + {file = "duckdb-0.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28ae942a79fad913defa912b56483cd7827a4e7721f4ce4bc9025b746ecb3c89"}, + {file = "duckdb-0.10.0-cp37-cp37m-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01b57802898091455ca2a32c1335aac1e398da77c99e8a96a1e5de09f6a0add9"}, + {file = "duckdb-0.10.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52e1ad4a55fa153d320c367046b9500578192e01c6d04308ba8b540441736f2c"}, + {file = "duckdb-0.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:904c47d04095af745e989c853f0bfc0776913dfc40dfbd2da7afdbbb5f67fed0"}, + {file = "duckdb-0.10.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:184ae7ea5874f3b8fa51ab0f1519bdd088a0b78c32080ee272b1d137e2c8fd9c"}, + {file = "duckdb-0.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd33982ecc9bac727a032d6cedced9f19033cbad56647147408891eb51a6cb37"}, + {file = "duckdb-0.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f59bf0949899105dd5f8864cb48139bfb78454a8c017b8258ba2b5e90acf7afc"}, + {file = "duckdb-0.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:395f3b18948001e35dceb48a4423d574e38656606d033eef375408b539e7b076"}, + {file = "duckdb-0.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b8eb2b803be7ee1df70435c33b03a4598cdaf676cd67ad782b288dcff65d781"}, + {file = "duckdb-0.10.0-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31b2ddd331801064326c8e3587a4db8a31d02aef11332c168f45b3bd92effb41"}, + {file = "duckdb-0.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c8b89e76a041424b8c2026c5dc1f74b53fbbc6c6f650d563259885ab2e7d093d"}, + {file = "duckdb-0.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:79084a82f16c0a54f6bfb7ded5600400c2daa90eb0d83337d81a56924eaee5d4"}, + {file = "duckdb-0.10.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:79799b3a270dcd9070f677ba510f1e66b112df3068425691bac97c5e278929c7"}, + {file = "duckdb-0.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e8fc394bfe3434920cdbcfbdd0ac3ba40902faa1dbda088db0ba44003a45318a"}, + {file = "duckdb-0.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c116605551b4abf5786243a59bcef02bd69cc51837d0c57cafaa68cdc428aa0c"}, + {file = "duckdb-0.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3191170c3b0a43b0c12644800326f5afdea00d5a4621d59dbbd0c1059139e140"}, + {file = "duckdb-0.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fee69a50eb93c72dc77e7ab1fabe0c38d21a52c5da44a86aa217081e38f9f1bd"}, + {file = "duckdb-0.10.0-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5f449e87dacb16b0d145dbe65fa6fdb5a55b2b6911a46d74876e445dd395bac"}, + {file = "duckdb-0.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4487d0df221b17ea4177ad08131bc606b35f25cfadf890987833055b9d10cdf6"}, + {file = "duckdb-0.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:c099ae2ff8fe939fda62da81704f91e2f92ac45e48dc0e37c679c9d243d01e65"}, + {file = "duckdb-0.10.0.tar.gz", hash = "sha256:c02bcc128002aa79e3c9d89b9de25e062d1096a8793bc0d7932317b7977f6845"}, +] + +[[package]] +name = "duckdb-engine" +version = "0.10.0" +description = "SQLAlchemy driver for duckdb" +optional = false +python-versions = ">=3.7" +files = [ + {file = "duckdb_engine-0.10.0-py3-none-any.whl", hash = "sha256:c408d002e83630b6bbb05fc3b26a43406085b1c22dd43e8cab00bf0b9c011ea8"}, + {file = "duckdb_engine-0.10.0.tar.gz", hash = "sha256:5e3dad3b3513f055a4f5ec5430842249cfe03015743a7597ed1dcc0447dca565"}, +] + +[package.dependencies] +duckdb = ">=0.4.0" +sqlalchemy = ">=1.3.22" + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "faker" +version = "21.0.1" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Faker-21.0.1-py3-none-any.whl", hash = "sha256:0afc67ec898a2d71842a3456e9302620ebc35fab6ad4f3829693fdf151fa4a3a"}, + {file = "Faker-21.0.1.tar.gz", hash = "sha256:bb404bba449b87e6b54a8c50b4602765e9c1a42eaf48abfceb025e42fed01608"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" + +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-api-core" +version = "2.17.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"}, + {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +grpcio = [ + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, +] +grpcio-status = [ + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.28.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.28.0.tar.gz", hash = "sha256:3cfc1b6e4e64797584fb53fc9bd0b7afa9b7c0dba2004fa7dcc9349e58cc3195"}, + {file = "google_auth-2.28.0-py2.py3-none-any.whl", hash = "sha256:7634d29dcd1e101f5226a23cbc4a0c6cda6394253bf80e281d9c5c6797869c53"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-secret-manager" +version = "2.18.1" +description = "Google Cloud Secret Manager API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-secret-manager-2.18.1.tar.gz", hash = "sha256:310555f3c8cb977f4a46d4454eca2c83fed6a09f3c4b35b84f6fa1f8fef55024"}, + {file = "google_cloud_secret_manager-2.18.1-py2.py3-none-any.whl", hash = "sha256:38e00ece9abf466cb449991b1a141a69690c6d51fe18456e531faf4935fbade3"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" +proto-plus = ">=1.22.3,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +grpcio = {version = ">=1.44.0,<2.0.0.dev0", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.13.0" +description = "IAM API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpc-google-iam-v1-0.13.0.tar.gz", hash = "sha256:fad318608b9e093258fbf12529180f400d1c44453698a33509cc6ecf005b294e"}, + {file = "grpc_google_iam_v1-0.13.0-py2.py3-none-any.whl", hash = "sha256:53902e2af7de8df8c1bd91373d9be55b0743ec267a7428ea638db3775becae89"}, +] + +[package.dependencies] +googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} +grpcio = ">=1.44.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "grpcio" +version = "1.60.1" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.60.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:14e8f2c84c0832773fb3958240c69def72357bc11392571f87b2d7b91e0bb092"}, + {file = "grpcio-1.60.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:33aed0a431f5befeffd9d346b0fa44b2c01aa4aeae5ea5b2c03d3e25e0071216"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:fead980fbc68512dfd4e0c7b1f5754c2a8e5015a04dea454b9cada54a8423525"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:082081e6a36b6eb5cf0fd9a897fe777dbb3802176ffd08e3ec6567edd85bc104"}, + {file = "grpcio-1.60.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ccb7db5a665079d68b5c7c86359ebd5ebf31a19bc1a91c982fd622f1e31ff2"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b54577032d4f235452f77a83169b6527bf4b77d73aeada97d45b2aaf1bf5ce0"}, + {file = "grpcio-1.60.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7d142bcd604166417929b071cd396aa13c565749a4c840d6c702727a59d835eb"}, + {file = "grpcio-1.60.1-cp310-cp310-win32.whl", hash = "sha256:2a6087f234cb570008a6041c8ffd1b7d657b397fdd6d26e83d72283dae3527b1"}, + {file = "grpcio-1.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:f2212796593ad1d0235068c79836861f2201fc7137a99aa2fea7beeb3b101177"}, + {file = "grpcio-1.60.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:79ae0dc785504cb1e1788758c588c711f4e4a0195d70dff53db203c95a0bd303"}, + {file = "grpcio-1.60.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4eec8b8c1c2c9b7125508ff7c89d5701bf933c99d3910e446ed531cd16ad5d87"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8c9554ca8e26241dabe7951aa1fa03a1ba0856688ecd7e7bdbdd286ebc272e4c"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91422ba785a8e7a18725b1dc40fbd88f08a5bb4c7f1b3e8739cab24b04fa8a03"}, + {file = "grpcio-1.60.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba6209c96828711cb7c8fcb45ecef8c8859238baf15119daa1bef0f6c84bfe7"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c71be3f86d67d8d1311c6076a4ba3b75ba5703c0b856b4e691c9097f9b1e8bd2"}, + {file = "grpcio-1.60.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5ef6cfaf0d023c00002ba25d0751e5995fa0e4c9eec6cd263c30352662cbce"}, + {file = "grpcio-1.60.1-cp311-cp311-win32.whl", hash = "sha256:a09506eb48fa5493c58f946c46754ef22f3ec0df64f2b5149373ff31fb67f3dd"}, + {file = "grpcio-1.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:49c9b6a510e3ed8df5f6f4f3c34d7fbf2d2cae048ee90a45cd7415abab72912c"}, + {file = "grpcio-1.60.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b58b855d0071575ea9c7bc0d84a06d2edfbfccec52e9657864386381a7ce1ae9"}, + {file = "grpcio-1.60.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a731ac5cffc34dac62053e0da90f0c0b8560396a19f69d9703e88240c8f05858"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:cf77f8cf2a651fbd869fbdcb4a1931464189cd210abc4cfad357f1cacc8642a6"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c557e94e91a983e5b1e9c60076a8fd79fea1e7e06848eb2e48d0ccfb30f6e073"}, + {file = "grpcio-1.60.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:069fe2aeee02dfd2135d562d0663fe70fbb69d5eed6eb3389042a7e963b54de8"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb0af13433dbbd1c806e671d81ec75bd324af6ef75171fd7815ca3074fe32bfe"}, + {file = "grpcio-1.60.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2f44c32aef186bbba254129cea1df08a20be414144ac3bdf0e84b24e3f3b2e05"}, + {file = "grpcio-1.60.1-cp312-cp312-win32.whl", hash = "sha256:a212e5dea1a4182e40cd3e4067ee46be9d10418092ce3627475e995cca95de21"}, + {file = "grpcio-1.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:6e490fa5f7f5326222cb9f0b78f207a2b218a14edf39602e083d5f617354306f"}, + {file = "grpcio-1.60.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:4216e67ad9a4769117433814956031cb300f85edc855252a645a9a724b3b6594"}, + {file = "grpcio-1.60.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:73e14acd3d4247169955fae8fb103a2b900cfad21d0c35f0dcd0fdd54cd60367"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6ecf21d20d02d1733e9c820fb5c114c749d888704a7ec824b545c12e78734d1c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33bdea30dcfd4f87b045d404388469eb48a48c33a6195a043d116ed1b9a0196c"}, + {file = "grpcio-1.60.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b69e79d00f78c81eecfb38f4516080dc7f36a198b6b37b928f1c13b3c063e9"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:39aa848794b887120b1d35b1b994e445cc028ff602ef267f87c38122c1add50d"}, + {file = "grpcio-1.60.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:72153a0d2e425f45b884540a61c6639436ddafa1829a42056aa5764b84108b8e"}, + {file = "grpcio-1.60.1-cp37-cp37m-win_amd64.whl", hash = "sha256:50d56280b482875d1f9128ce596e59031a226a8b84bec88cb2bf76c289f5d0de"}, + {file = "grpcio-1.60.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6d140bdeb26cad8b93c1455fa00573c05592793c32053d6e0016ce05ba267549"}, + {file = "grpcio-1.60.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:bc808924470643b82b14fe121923c30ec211d8c693e747eba8a7414bc4351a23"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:70c83bb530572917be20c21f3b6be92cd86b9aecb44b0c18b1d3b2cc3ae47df0"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b106bc52e7f28170e624ba61cc7dc6829566e535a6ec68528f8e1afbed1c41f"}, + {file = "grpcio-1.60.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e980cd6db1088c144b92fe376747328d5554bc7960ce583ec7b7d81cd47287"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0c5807e9152eff15f1d48f6b9ad3749196f79a4a050469d99eecb679be592acc"}, + {file = "grpcio-1.60.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1c3dc536b3ee124e8b24feb7533e5c70b9f2ef833e3b2e5513b2897fd46763a"}, + {file = "grpcio-1.60.1-cp38-cp38-win32.whl", hash = "sha256:d7404cebcdb11bb5bd40bf94131faf7e9a7c10a6c60358580fe83913f360f929"}, + {file = "grpcio-1.60.1-cp38-cp38-win_amd64.whl", hash = "sha256:c8754c75f55781515a3005063d9a05878b2cfb3cb7e41d5401ad0cf19de14872"}, + {file = "grpcio-1.60.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:0250a7a70b14000fa311de04b169cc7480be6c1a769b190769d347939d3232a8"}, + {file = "grpcio-1.60.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:660fc6b9c2a9ea3bb2a7e64ba878c98339abaf1811edca904ac85e9e662f1d73"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76eaaba891083fcbe167aa0f03363311a9f12da975b025d30e94b93ac7a765fc"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d97c65ea7e097056f3d1ead77040ebc236feaf7f71489383d20f3b4c28412a"}, + {file = "grpcio-1.60.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb2a2911b028f01c8c64d126f6b632fcd8a9ac975aa1b3855766c94e4107180"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5a1ebbae7e2214f51b1f23b57bf98eeed2cf1ba84e4d523c48c36d5b2f8829ff"}, + {file = "grpcio-1.60.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a66f4d2a005bc78e61d805ed95dedfcb35efa84b7bba0403c6d60d13a3de2d6"}, + {file = "grpcio-1.60.1-cp39-cp39-win32.whl", hash = "sha256:8d488fbdbf04283f0d20742b64968d44825617aa6717b07c006168ed16488804"}, + {file = "grpcio-1.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b7199cd2a55e62e45bfb629a35b71fc2c0cb88f686a047f25b1112d3810904"}, + {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.60.1)"] + +[[package]] +name = "grpcio-status" +version = "1.60.1" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.60.1.tar.gz", hash = "sha256:61b5aab8989498e8aa142c20b88829ea5d90d18c18c853b9f9e6d407d37bf8b4"}, + {file = "grpcio_status-1.60.1-py3-none-any.whl", hash = "sha256:3034fdb239185b6e0f3169d08c268c4507481e4b8a434c21311a03d9eb5889a0"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.60.1" +protobuf = ">=4.21.6" + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "mimesis" +version = "6.1.1" +description = "Mimesis: Fake Data Generator." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "mimesis-6.1.1-py3-none-any.whl", hash = "sha256:eabe41d7afa23b01dffb51ebd9e10837df6417fef02fa9841989ca886e479790"}, + {file = "mimesis-6.1.1.tar.gz", hash = "sha256:044ac378c61db0e06832ff722548fd6e604881d36bc938002e0bd5b85eeb6a98"}, +] + +[[package]] +name = "mypy" +version = "1.8.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "objprint" +version = "0.2.3" +description = "A library that can print Python objects in human readable format" +optional = false +python-versions = ">=3.6" +files = [ + {file = "objprint-0.2.3-py3-none-any.whl", hash = "sha256:1721e6f97bae5c5b86c2716a0d45a9dd2c9a4cd9f52cfc8a0dfbe801805554cb"}, + {file = "objprint-0.2.3.tar.gz", hash = "sha256:73d0ad5a7c3151fce634c8892e5c2a050ccae3b1a353bf1316f08b7854da863b"}, +] + +[[package]] +name = "orjson" +version = "3.9.14" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.9.14-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:793f6c9448ab6eb7d4974b4dde3f230345c08ca6c7995330fbceeb43a5c8aa5e"}, + {file = "orjson-3.9.14-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6bc7928d161840096adc956703494b5c0193ede887346f028216cac0af87500"}, + {file = "orjson-3.9.14-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:58b36f54da759602d8e2f7dad958752d453dfe2c7122767bc7f765e17dc59959"}, + {file = "orjson-3.9.14-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:abcda41ecdc950399c05eff761c3de91485d9a70d8227cb599ad3a66afe93bcc"}, + {file = "orjson-3.9.14-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df76ecd17b1b3627bddfd689faaf206380a1a38cc9f6c4075bd884eaedcf46c2"}, + {file = "orjson-3.9.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d450a8e0656efb5d0fcb062157b918ab02dcca73278975b4ee9ea49e2fcf5bd5"}, + {file = "orjson-3.9.14-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:95c03137b0cf66517c8baa65770507a756d3a89489d8ecf864ea92348e1beabe"}, + {file = "orjson-3.9.14-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20837e10835c98973673406d6798e10f821e7744520633811a5a3d809762d8cc"}, + {file = "orjson-3.9.14-cp310-none-win32.whl", hash = "sha256:1f7b6f3ef10ae8e3558abb729873d033dbb5843507c66b1c0767e32502ba96bb"}, + {file = "orjson-3.9.14-cp310-none-win_amd64.whl", hash = "sha256:ea890e6dc1711aeec0a33b8520e395c2f3d59ead5b4351a788e06bf95fc7ba81"}, + {file = "orjson-3.9.14-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c19009ff37f033c70acd04b636380379499dac2cba27ae7dfc24f304deabbc81"}, + {file = "orjson-3.9.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19cdea0664aec0b7f385be84986d4defd3334e9c3c799407686ee1c26f7b8251"}, + {file = "orjson-3.9.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:135d518f73787ce323b1a5e21fb854fe22258d7a8ae562b81a49d6c7f826f2a3"}, + {file = "orjson-3.9.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2cf1d0557c61c75e18cf7d69fb689b77896e95553e212c0cc64cf2087944b84"}, + {file = "orjson-3.9.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7c11667421df2d8b18b021223505dcc3ee51be518d54e4dc49161ac88ac2b87"}, + {file = "orjson-3.9.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eefc41ba42e75ed88bc396d8fe997beb20477f3e7efa000cd7a47eda452fbb2"}, + {file = "orjson-3.9.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:917311d6a64d1c327c0dfda1e41f3966a7fb72b11ca7aa2e7a68fcccc7db35d9"}, + {file = "orjson-3.9.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4dc1c132259b38d12c6587d190cd09cd76e3b5273ce71fe1372437b4cbc65f6f"}, + {file = "orjson-3.9.14-cp311-none-win32.whl", hash = "sha256:6f39a10408478f4c05736a74da63727a1ae0e83e3533d07b19443400fe8591ca"}, + {file = "orjson-3.9.14-cp311-none-win_amd64.whl", hash = "sha256:26280a7fcb62d8257f634c16acebc3bec626454f9ab13558bbf7883b9140760e"}, + {file = "orjson-3.9.14-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:08e722a8d06b13b67a51f247a24938d1a94b4b3862e40e0eef3b2e98c99cd04c"}, + {file = "orjson-3.9.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2591faa0c031cf3f57e5bce1461cfbd6160f3f66b5a72609a130924917cb07d"}, + {file = "orjson-3.9.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2450d87dd7b4f277f4c5598faa8b49a0c197b91186c47a2c0b88e15531e4e3e"}, + {file = "orjson-3.9.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:90903d2908158a2c9077a06f11e27545de610af690fb178fd3ba6b32492d4d1c"}, + {file = "orjson-3.9.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce6f095eef0026eae76fc212f20f786011ecf482fc7df2f4c272a8ae6dd7b1ef"}, + {file = "orjson-3.9.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:751250a31fef2bac05a2da2449aae7142075ea26139271f169af60456d8ad27a"}, + {file = "orjson-3.9.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9a1af21160a38ee8be3f4fcf24ee4b99e6184cadc7f915d599f073f478a94d2c"}, + {file = "orjson-3.9.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:449bf090b2aa4e019371d7511a6ea8a5a248139205c27d1834bb4b1e3c44d936"}, + {file = "orjson-3.9.14-cp312-none-win_amd64.whl", hash = "sha256:a603161318ff699784943e71f53899983b7dee571b4dd07c336437c9c5a272b0"}, + {file = "orjson-3.9.14-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:814f288c011efdf8f115c5ebcc1ab94b11da64b207722917e0ceb42f52ef30a3"}, + {file = "orjson-3.9.14-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a88cafb100af68af3b9b29b5ccd09fdf7a48c63327916c8c923a94c336d38dd3"}, + {file = "orjson-3.9.14-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ba3518b999f88882ade6686f1b71e207b52e23546e180499be5bbb63a2f9c6e6"}, + {file = "orjson-3.9.14-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:978f416bbff9da8d2091e3cf011c92da68b13f2c453dcc2e8109099b2a19d234"}, + {file = "orjson-3.9.14-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75fc593cf836f631153d0e21beaeb8d26e144445c73645889335c2247fcd71a0"}, + {file = "orjson-3.9.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d1528db3c7554f9d6eeb09df23cb80dd5177ec56eeb55cc5318826928de506"}, + {file = "orjson-3.9.14-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:7183cc68ee2113b19b0b8714221e5e3b07b3ba10ca2bb108d78fd49cefaae101"}, + {file = "orjson-3.9.14-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:df3266d54246cb56b8bb17fa908660d2a0f2e3f63fbc32451ffc1b1505051d07"}, + {file = "orjson-3.9.14-cp38-none-win32.whl", hash = "sha256:7913079b029e1b3501854c9a78ad938ed40d61fe09bebab3c93e60ff1301b189"}, + {file = "orjson-3.9.14-cp38-none-win_amd64.whl", hash = "sha256:29512eb925b620e5da2fd7585814485c67cc6ba4fe739a0a700c50467a8a8065"}, + {file = "orjson-3.9.14-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5bf597530544db27a8d76aced49cfc817ee9503e0a4ebf0109cd70331e7bbe0c"}, + {file = "orjson-3.9.14-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac650d49366fa41fe702e054cb560171a8634e2865537e91f09a8d05ea5b1d37"}, + {file = "orjson-3.9.14-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:236230433a9a4968ab895140514c308fdf9f607cb8bee178a04372b771123860"}, + {file = "orjson-3.9.14-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3014ccbda9be0b1b5f8ea895121df7e6524496b3908f4397ff02e923bcd8f6dd"}, + {file = "orjson-3.9.14-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ac0c7eae7ad3a223bde690565442f8a3d620056bd01196f191af8be58a5248e1"}, + {file = "orjson-3.9.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fca33fdd0b38839b01912c57546d4f412ba7bfa0faf9bf7453432219aec2df07"}, + {file = "orjson-3.9.14-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f75823cc1674a840a151e999a7dfa0d86c911150dd6f951d0736ee9d383bf415"}, + {file = "orjson-3.9.14-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6f52ac2eb49e99e7373f62e2a68428c6946cda52ce89aa8fe9f890c7278e2d3a"}, + {file = "orjson-3.9.14-cp39-none-win32.whl", hash = "sha256:0572f174f50b673b7df78680fb52cd0087a8585a6d06d295a5f790568e1064c6"}, + {file = "orjson-3.9.14-cp39-none-win_amd64.whl", hash = "sha256:ab90c02cb264250b8a58cedcc72ed78a4a257d956c8d3c8bebe9751b818dfad8"}, + {file = "orjson-3.9.14.tar.gz", hash = "sha256:06fb40f8e49088ecaa02f1162581d39e2cf3fd9dbbfe411eb2284147c99bad79"}, +] + +[[package]] +name = "overrides" +version = "7.7.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = false +python-versions = ">=3.6" +files = [ + {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, + {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.1.4" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "pandas-stubs" +version = "2.1.4.231227" +description = "Type annotations for pandas" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas_stubs-2.1.4.231227-py3-none-any.whl", hash = "sha256:211fc23e6ae87073bdf41dbf362c4a4d85e1e3477cb078dbac3da6c7fdaefba8"}, + {file = "pandas_stubs-2.1.4.231227.tar.gz", hash = "sha256:3ea29ef001e9e44985f5ebde02d4413f94891ef6ec7e5056fb07d125be796c23"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version < \"3.13\""} +types-pytz = ">=2022.1.1" + +[[package]] +name = "pdoc" +version = "14.4.0" +description = "API Documentation for Python Projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pdoc-14.4.0-py3-none-any.whl", hash = "sha256:6ea4fe07620b1f7601e2708a307a257636ec206e20b5611640b30f2e3cab47d6"}, + {file = "pdoc-14.4.0.tar.gz", hash = "sha256:c92edc425429ccbe287ace2a027953c24f13de53eab484c1a6d31ca72dd2fda9"}, +] + +[package.dependencies] +Jinja2 = ">=2.11.0" +MarkupSafe = "*" +pygments = ">=2.12.0" + +[package.extras] +dev = ["hypothesis", "mypy", "pdoc-pyo3-sample-library (==1.0.11)", "pygments (>=2.14.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "proto-plus" +version = "1.23.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, + {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, +] + +[[package]] +name = "pyarrow" +version = "14.0.2" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-14.0.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9fe808596c5dbd08b3aeffe901e5f81095baaa28e7d5118e01354c64f22807"}, + {file = "pyarrow-14.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a768987a16bb46220cef490c56c671993fbee8fd0475febac0b3e16b00a10e"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dbba05e98f247f17e64303eb876f4a80fcd32f73c7e9ad975a83834d81f3fda"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a898d134d00b1eca04998e9d286e19653f9d0fcb99587310cd10270907452a6b"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:87e879323f256cb04267bb365add7208f302df942eb943c93a9dfeb8f44840b1"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:76fc257559404ea5f1306ea9a3ff0541bf996ff3f7b9209fc517b5e83811fa8e"}, + {file = "pyarrow-14.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0c4a18e00f3a32398a7f31da47fefcd7a927545b396e1f15d0c85c2f2c778cd"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:87482af32e5a0c0cce2d12eb3c039dd1d853bd905b04f3f953f147c7a196915b"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:059bd8f12a70519e46cd64e1ba40e97eae55e0cbe1695edd95384653d7626b23"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f16111f9ab27e60b391c5f6d197510e3ad6654e73857b4e394861fc79c37200"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06ff1264fe4448e8d02073f5ce45a9f934c0f3db0a04460d0b01ff28befc3696"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6dd4f4b472ccf4042f1eab77e6c8bce574543f54d2135c7e396f413046397d5a"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:32356bfb58b36059773f49e4e214996888eeea3a08893e7dbde44753799b2a02"}, + {file = "pyarrow-14.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:52809ee69d4dbf2241c0e4366d949ba035cbcf48409bf404f071f624ed313a2b"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:c87824a5ac52be210d32906c715f4ed7053d0180c1060ae3ff9b7e560f53f944"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a25eb2421a58e861f6ca91f43339d215476f4fe159eca603c55950c14f378cc5"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c1da70d668af5620b8ba0a23f229030a4cd6c5f24a616a146f30d2386fec422"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cc61593c8e66194c7cdfae594503e91b926a228fba40b5cf25cc593563bcd07"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:78ea56f62fb7c0ae8ecb9afdd7893e3a7dbeb0b04106f5c08dbb23f9c0157591"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:37c233ddbce0c67a76c0985612fef27c0c92aef9413cf5aa56952f359fcb7379"}, + {file = "pyarrow-14.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:e4b123ad0f6add92de898214d404e488167b87b5dd86e9a434126bc2b7a5578d"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e354fba8490de258be7687f341bc04aba181fc8aa1f71e4584f9890d9cb2dec2"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:20e003a23a13da963f43e2b432483fdd8c38dc8882cd145f09f21792e1cf22a1"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc0de7575e841f1595ac07e5bc631084fd06ca8b03c0f2ecece733d23cd5102a"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e986dc859712acb0bd45601229021f3ffcdfc49044b64c6d071aaf4fa49e98"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f7d029f20ef56673a9730766023459ece397a05001f4e4d13805111d7c2108c0"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:209bac546942b0d8edc8debda248364f7f668e4aad4741bae58e67d40e5fcf75"}, + {file = "pyarrow-14.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1e6987c5274fb87d66bb36816afb6f65707546b3c45c44c28e3c4133c010a881"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a01d0052d2a294a5f56cc1862933014e696aa08cc7b620e8c0cce5a5d362e976"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a51fee3a7db4d37f8cda3ea96f32530620d43b0489d169b285d774da48ca9785"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64df2bf1ef2ef14cee531e2dfe03dd924017650ffaa6f9513d7a1bb291e59c15"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c0fa3bfdb0305ffe09810f9d3e2e50a2787e3a07063001dcd7adae0cee3601a"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c65bf4fd06584f058420238bc47a316e80dda01ec0dfb3044594128a6c2db794"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:63ac901baec9369d6aae1cbe6cca11178fb018a8d45068aaf5bb54f94804a866"}, + {file = "pyarrow-14.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:75ee0efe7a87a687ae303d63037d08a48ef9ea0127064df18267252cfe2e9541"}, + {file = "pyarrow-14.0.2.tar.gz", hash = "sha256:36cef6ba12b499d864d1def3e990f97949e0b79400d08b7cf74504ffbd3eb025"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyarrow-stubs" +version = "10.0.1.7" +description = "Type annotations for pyarrow" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pyarrow_stubs-10.0.1.7-py3-none-any.whl", hash = "sha256:cccc7a46eddeea4e3cb85330eb8972c116a615da6188b8ae1f7a44cb724b21ac"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyopenssl" +version = "23.3.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyOpenSSL-23.3.0-py3-none-any.whl", hash = "sha256:6756834481d9ed5470f4a9393455154bc92fe7a64b7bc6ee2c804e78c52099b2"}, + {file = "pyOpenSSL-23.3.0.tar.gz", hash = "sha256:6b2cba5cc46e822750ec3e5a81ee12819850b11303630d575e98108a079c2b12"}, +] + +[package.dependencies] +cryptography = ">=41.0.5,<42" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +test = ["flaky", "pretend", "pytest (>=3.0.1)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-docker" +version = "2.2.0" +description = "Simple pytest fixtures for Docker and Docker Compose based tests" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-docker-2.2.0.tar.gz", hash = "sha256:b083fd2ae69212369390033c22228d3263555a5f3b4bef87b74160e07218f377"}, + {file = "pytest_docker-2.2.0-py3-none-any.whl", hash = "sha256:8ee9c9742d58ac079c81c03635bb830881f7f4d529f0f53f4ba2c89ffc9c7137"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +pytest = ">=4.0,<8.0" + +[package.extras] +docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] +tests = ["pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)"] + +[[package]] +name = "pytest-mypy" +version = "0.10.3" +description = "Mypy static type checker plugin for Pytest" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-mypy-0.10.3.tar.gz", hash = "sha256:f8458f642323f13a2ca3e2e61509f7767966b527b4d8adccd5032c3e7b4fd3db"}, + {file = "pytest_mypy-0.10.3-py3-none-any.whl", hash = "sha256:7638d0d3906848fc1810cb2f5cc7fceb4cc5c98524aafcac58f28620e3102053"}, +] + +[package.dependencies] +attrs = ">=19.0" +filelock = ">=3.0" +mypy = [ + {version = ">=0.900", markers = "python_version >= \"3.11\""}, + {version = ">=0.780", markers = "python_version >= \"3.9\" and python_version < \"3.11\""}, +] +pytest = [ + {version = ">=6.2", markers = "python_version >= \"3.10\""}, + {version = ">=4.6", markers = "python_version >= \"3.6\" and python_version < \"3.10\""}, +] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-ulid" +version = "2.2.0" +description = "Universally unique lexicographically sortable identifier" +optional = false +python-versions = ">=3.9" +files = [ + {file = "python_ulid-2.2.0-py3-none-any.whl", hash = "sha256:ec2e69292c0b7c338a07df5e15b05270be6823675c103383e74d1d531945eab5"}, + {file = "python_ulid-2.2.0.tar.gz", hash = "sha256:9ec777177d396880d94be49ac7eb4ae2cd4a7474448bfdbfe911537add970aeb"}, +] + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "referencing" +version = "0.33.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.33.0-py3-none-any.whl", hash = "sha256:39240f2ecc770258f28b642dd47fd74bc8b02484de54e1882b74b35ebd779bd5"}, + {file = "referencing-0.33.0.tar.gz", hash = "sha256:c775fedf74bc0f9189c2a3be1c12fd03e8c23f4d371dce795df44e06c5b412f7"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "rich" +version = "13.7.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rpds-py" +version = "0.18.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.18.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e"}, + {file = "rpds_py-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88"}, + {file = "rpds_py-0.18.0-cp310-none-win32.whl", hash = "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337"}, + {file = "rpds_py-0.18.0-cp310-none-win_amd64.whl", hash = "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836"}, + {file = "rpds_py-0.18.0-cp311-none-win32.whl", hash = "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1"}, + {file = "rpds_py-0.18.0-cp311-none-win_amd64.whl", hash = "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7"}, + {file = "rpds_py-0.18.0-cp312-none-win32.whl", hash = "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98"}, + {file = "rpds_py-0.18.0-cp312-none-win_amd64.whl", hash = "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594"}, + {file = "rpds_py-0.18.0-cp38-none-win32.whl", hash = "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"}, + {file = "rpds_py-0.18.0-cp38-none-win_amd64.whl", hash = "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20"}, + {file = "rpds_py-0.18.0-cp39-none-win32.whl", hash = "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7"}, + {file = "rpds_py-0.18.0-cp39-none-win_amd64.whl", hash = "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f"}, + {file = "rpds_py-0.18.0.tar.gz", hash = "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d"}, +] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "ruff" +version = "0.1.15" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5fe8d54df166ecc24106db7dd6a68d44852d14eb0729ea4672bb4d96c320b7df"}, + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f0bfbb53c4b4de117ac4d6ddfd33aa5fc31beeaa21d23c45c6dd249faf9126f"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d432aec35bfc0d800d4f70eba26e23a352386be3a6cf157083d18f6f5881c8"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9405fa9ac0e97f35aaddf185a1be194a589424b8713e3b97b762336ec79ff807"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c66ec24fe36841636e814b8f90f572a8c0cb0e54d8b5c2d0e300d28a0d7bffec"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6f8ad828f01e8dd32cc58bc28375150171d198491fc901f6f98d2a39ba8e3ff5"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86811954eec63e9ea162af0ffa9f8d09088bab51b7438e8b6488b9401863c25e"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd4025ac5e87d9b80e1f300207eb2fd099ff8200fa2320d7dc066a3f4622dc6b"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b17b93c02cdb6aeb696effecea1095ac93f3884a49a554a9afa76bb125c114c1"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ddb87643be40f034e97e97f5bc2ef7ce39de20e34608f3f829db727a93fb82c5"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:abf4822129ed3a5ce54383d5f0e964e7fef74a41e48eb1dfad404151efc130a2"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6c629cf64bacfd136c07c78ac10a54578ec9d1bd2a9d395efbee0935868bf852"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1bab866aafb53da39c2cadfb8e1c4550ac5340bb40300083eb8967ba25481447"}, + {file = "ruff-0.1.15-py3-none-win32.whl", hash = "sha256:2417e1cb6e2068389b07e6fa74c306b2810fe3ee3476d5b8a96616633f40d14f"}, + {file = "ruff-0.1.15-py3-none-win_amd64.whl", hash = "sha256:3837ac73d869efc4182d9036b1405ef4c73d9b1f88da2413875e34e0d6919587"}, + {file = "ruff-0.1.15-py3-none-win_arm64.whl", hash = "sha256:9a933dfb1c14ec7a33cceb1e49ec4a16b51ce3c20fd42663198746efc0427360"}, + {file = "ruff-0.1.15.tar.gz", hash = "sha256:f6dfa8c1b21c913c326919056c390966648b680966febcb796cc9d1aaab8564e"}, +] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "snowflake-connector-python" +version = "3.6.0" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake-connector-python-3.6.0.tar.gz", hash = "sha256:15667a918780d79da755e6a60bbf6918051854951e8f56ccdf5692283e9a8479"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4093b38cf9abf95c38119f0b23b07e23dc7a8689b956cd5d34975e1875741f20"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:cf5a964fe01b177063f8c44d14df3a72715580bcd195788ec2822090f37330a5"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55a6418cec585b050e6f05404f25e62b075a3bbea587dc1f903de15640565c58"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7c76aea92b87f6ecd604e9c934aac8a779f2e20f3be1d990d53bb5b6d87b009"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:9dfcf178271e892e64e4092b9e011239a066ce5de848afd2efe3f13197a9f8b3"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4916f9b4a0efd7c96d1fa50a157e05907b6935f91492cca7f200b43cc178a25e"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:f15024c66db5e87d359216ec733a2974d7562aa38f3f18c8b6e65489839e00d7"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcbd3102f807ebbbae52b1b5683d45cd7b3dcb0eaec131233ba6b156e8d70fa4"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7662e2de25b885abe08ab866cf7c7b026ad1af9faa39c25e2c25015ef807abe3"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:d1fa102f55ee166cc766aeee3f9333b17b4bede6fb088eee1e1f022df15b6d81"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fde1e0727e2f23c2a07b49b30e1bc0f49977f965d08ddfda10015b24a2beeb76"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:1b51fe000c8cf6372d30b73c7136275e52788e6af47010cd1984c9fb03378e86"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a11699689a19916e65794ce58dca72b8a40fe6a7eea06764931ede10b47bcc"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d810be5b180c6f47ce9b6f989fe64b9984383e4b77e30b284a83e33f229a3a82"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:b5db47d4164d6b7a07c413a46f9edc4a1d687e3df44fd9d5fa89a89aecb94a8e"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bf8c1ad5aab5304fefa2a4178061a24c96da45e3e3db9d901621e9953e005402"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:1058ab5c98cc62fde8b3f021f0a5076cb7865b5cdab8a9bccde0df88b9e91334"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b93f55989f80d69278e0f40a7a1c0e737806b7c0ddb0351513a752b837243e8"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50dd954ea5918d3242ded69225b72f701963cd9c043ee7d9ab35dc22211611c8"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4ad42613b87f31441d07a8ea242f4c28ed5eb7b6e05986f9e94a7e44b96d3d1e"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset-normalizer = ">=2,<4" +cryptography = ">=3.1.0,<42.0.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +packaging = "*" +platformdirs = ">=2.6.0,<4.0.0" +pyjwt = "<3.0.0" +pyOpenSSL = ">=16.2.0,<24.0.0" +pytz = "*" +requests = "<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing-extensions = ">=4.3,<5" +urllib3 = {version = ">=1.21.1,<2.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.27.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<2.2.0)", "pyarrow"] +secure-local-storage = ["keyring (!=16.1.0,<25.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.5.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.7" +files = [ + {file = "snowflake-sqlalchemy-1.5.1.tar.gz", hash = "sha256:4f1383402ffc89311974bd810dee22003aef4af0f312a0fdb55778333ad1abf7"}, + {file = "snowflake_sqlalchemy-1.5.1-py2.py3-none-any.whl", hash = "sha256:df022fb73bc04d68dfb3216ebf7a1bfbd14d22def9c38bbe05275beb258adcd0"}, +] + +[package.dependencies] +snowflake-connector-python = "<4.0.0" +sqlalchemy = ">=1.4.0,<2.0.0" + +[package.extras] +development = ["mock", "numpy", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytz"] +pandas = ["snowflake-connector-python[pandas] (<4.0.0)"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + +[[package]] +name = "sqlalchemy" +version = "1.4.51" +description = "Database Abstraction Library" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "SQLAlchemy-1.4.51-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win32.whl", hash = "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win_amd64.whl", hash = "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win32.whl", hash = "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win_amd64.whl", hash = "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win32.whl", hash = "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win_amd64.whl", hash = "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win32.whl", hash = "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win_amd64.whl", hash = "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win32.whl", hash = "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win_amd64.whl", hash = "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win32.whl", hash = "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win_amd64.whl", hash = "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win32.whl", hash = "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win_amd64.whl", hash = "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba"}, + {file = "SQLAlchemy-1.4.51.tar.gz", hash = "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql", "pymysql (<1)"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.12.3" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, +] + +[[package]] +name = "types-jsonschema" +version = "4.21.0.20240118" +description = "Typing stubs for jsonschema" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-jsonschema-4.21.0.20240118.tar.gz", hash = "sha256:31aae1b5adc0176c1155c2d4f58348b22d92ae64315e9cc83bd6902168839232"}, + {file = "types_jsonschema-4.21.0.20240118-py3-none-any.whl", hash = "sha256:77a4ac36b0be4f24274d5b9bf0b66208ee771c05f80e34c4641de7d63e8a872d"}, +] + +[package.dependencies] +referencing = "*" + +[[package]] +name = "types-pytz" +version = "2024.1.0.20240203" +description = "Typing stubs for pytz" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-pytz-2024.1.0.20240203.tar.gz", hash = "sha256:c93751ee20dfc6e054a0148f8f5227b9a00b79c90a4d3c9f464711a73179c89e"}, + {file = "types_pytz-2024.1.0.20240203-py3-none-any.whl", hash = "sha256:9679eef0365db3af91ef7722c199dbb75ee5c1b67e3c4dd7bfbeb1b8a71c21a3"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "types-requests" +version = "2.31.0.4" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.31.0.4.tar.gz", hash = "sha256:a111041148d7e04bf100c476bc4db3ee6b0a1cd0b4018777f6a660b1c4f1318d"}, + {file = "types_requests-2.31.0.4-py3-none-any.whl", hash = "sha256:c7a9d6b62776f21b169a94a0e9d2dfcae62fa9149f53594ff791c3ae67325490"}, +] + +[package.dependencies] +types-urllib3 = "*" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "ulid" +version = "1.1" +description = "Pyhton version of this: https://github.com/alizain/ulid" +optional = false +python-versions = "*" +files = [ + {file = "ulid-1.1.tar.gz", hash = "sha256:0943e8a751ec10dfcdb4df2758f96dffbbfbc055d0b49288caf2f92125900d49"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "viztracer" +version = "0.16.2" +description = "A debugging and profiling tool that can trace and visualize python code execution" +optional = false +python-versions = ">=3.8" +files = [ + {file = "viztracer-0.16.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:bdc62e90a2957e4119632e98f8b77d0ff1ab4db7029dd2e265bb3748e0fc0e05"}, + {file = "viztracer-0.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:789ac930e1c9621f04d275ee3ebb75a5d6109bcd4634796a77934608c60424d0"}, + {file = "viztracer-0.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee504771e3182045996a966d94d95d71693e59717b2643199162ec754a6e2400"}, + {file = "viztracer-0.16.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef9ecf4110d379245f17429d2a10391f3612f60b5618d0d61a30c110e9df2313"}, + {file = "viztracer-0.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57c2574cc15b688eb0ce4e24a2c30f06c1df3bbe1dd16a1d18676e411e785f96"}, + {file = "viztracer-0.16.2-cp310-cp310-win32.whl", hash = "sha256:9fe652834f5073bf99debc25d8ba6084690fa2f26420621ca38a09efcae71b2f"}, + {file = "viztracer-0.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:d59f57e3e46e116ce77e144f419739d1d8d976a903c51a822ba4ef167e5b37d4"}, + {file = "viztracer-0.16.2-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:b0bd434c43b7f87f76ddd21cf7371d910edb74b131aaff670a8fcc9f28251e67"}, + {file = "viztracer-0.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1bbbb9c80b08db692993c67e7b10d7b06db3eedc6c38f0d93a40ea31de82076e"}, + {file = "viztracer-0.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1e7842e437d81fb47ef8266b2dde76bf755c95305014eeec8346b2fce9711c0"}, + {file = "viztracer-0.16.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bddfe6a6f2a66f363fcca79a694986b0602ba0dc3dede57dc182cdd6d0823585"}, + {file = "viztracer-0.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc4a2639e6f18200b73a70f3e7dca4cbb3ba08e3807023fd526f44ebf2185d1e"}, + {file = "viztracer-0.16.2-cp311-cp311-win32.whl", hash = "sha256:371496734ebb3eafd6a6e033dbf04960618089e021dc7eded95179a8f3700c40"}, + {file = "viztracer-0.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:d9c7670e7fb077fe48c92036766a6772e10a3caf41455d6244b8b1c8d48bbd87"}, + {file = "viztracer-0.16.2-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2fd8b5aa8143b5be4d696e53e8ac5027c20187c178396839f39f8aa610d5873d"}, + {file = "viztracer-0.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3a8ddc4990154f2d400b09deefc9236d963a733d458b2825bd590ced7e7bf89"}, + {file = "viztracer-0.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcf8b14dc8dd1567bca3f8cb13e31665a3cbf2ee95552de0afe9179e3a7bde22"}, + {file = "viztracer-0.16.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:309cf5d545222adb2581ae6aeb48d3d03d7241d335142408d87c49f1d0793f85"}, + {file = "viztracer-0.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee749a2a3f4ed662d35eb9378ff0648907aa6321befa16ad1d8bec6034b4d260"}, + {file = "viztracer-0.16.2-cp312-cp312-win32.whl", hash = "sha256:a082dab37b6b8cea43438b80a11a6e859f1b45522b8684a2fb9af03539d83803"}, + {file = "viztracer-0.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:03cd21181fe9a630ac5fb9ff1ee83fb7a67814e51e130f0ed83300e163fbac23"}, + {file = "viztracer-0.16.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:e920d383abae1b9314f2a60dd94e04c83998bfe759556af49d3c422d1d64d11e"}, + {file = "viztracer-0.16.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb9941b198fed8ba5b3f9d8105e59d37ab15f7f00b9a576686b1073990806d12"}, + {file = "viztracer-0.16.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1b7030aa6f934ff02882dfd48eca5a9442951b8be24c1dc5dc99fabbfb1997c"}, + {file = "viztracer-0.16.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:258087076c06d065d2786dc8a0f1f017d655d3753a8fe6836640c005c66a0c43"}, + {file = "viztracer-0.16.2-cp38-cp38-win32.whl", hash = "sha256:f0fd53e2fec972f9332677e6d11332ba789fcccf59060d7b9f309041602dc712"}, + {file = "viztracer-0.16.2-cp38-cp38-win_amd64.whl", hash = "sha256:ab067398029a50cc784d5456c5e8bef339b4bffaa1c3f0f9384a26b57c0efdaa"}, + {file = "viztracer-0.16.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:45879cf54ad9116245e2a6115660307f98ae3aa98a77347f2b336a904f260370"}, + {file = "viztracer-0.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abc61cfc36b33a301b950554d9e9027a506d580ebf1e764aa6656af0acfa3354"}, + {file = "viztracer-0.16.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:419f738bba8204e7ddb422faff3a40576896d030bbbf4fb79ace006147ca60e7"}, + {file = "viztracer-0.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c594022093bf9eee57ad2b9656f836dca2ed9c0b8e4d94a9d13a6cbc531386fe"}, + {file = "viztracer-0.16.2-cp39-cp39-win32.whl", hash = "sha256:4f98da282e87013a93917c2ae080ba52845e98ed5280faecdc42ee0c7fb74a4a"}, + {file = "viztracer-0.16.2-cp39-cp39-win_amd64.whl", hash = "sha256:64b97120374a572d2320fb795473c051c92d39dfc99fb74754e61e4c212e7617"}, + {file = "viztracer-0.16.2.tar.gz", hash = "sha256:8dff5637a7b42ffdbc1ed3768ce43979e71b09893ff370bc3c3ede54afed93ee"}, +] + +[package.dependencies] +objprint = ">0.1.3" + +[package.extras] +full = ["orjson"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "3b46151e994684f0953be1041a850ac8efcedea10632f8fa86aaaa6d20385174" diff --git a/airbyte-lib/poetry.toml b/airbyte-lib/poetry.toml new file mode 100644 index 000000000000..ab1033bd3722 --- /dev/null +++ b/airbyte-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml new file mode 100644 index 000000000000..8634bf4d1f3b --- /dev/null +++ b/airbyte-lib/pyproject.toml @@ -0,0 +1,261 @@ +[tool.poetry] +name = "airbyte-lib" +description = "AirbyteLib" +version = "0.1.0" +authors = ["Airbyte "] +readme = "README.md" +packages = [{include = "airbyte_lib"}] + +[tool.poetry.dependencies] +python = "^3.9" + +airbyte-cdk = "^0.58.3" +# airbyte-protocol-models = "^1.0.1" # Conflicts with airbyte-cdk # TODO: delete or resolve +jsonschema = "3.2.0" +orjson = "^3.9.10" +overrides = "^7.4.0" +pandas = "2.1.4" # 2.2.0 breaks sqlalchemy interop - TODO: optionally retest higher versions +psycopg2-binary = "^2.9.9" +python-ulid = "^2.2.0" +types-pyyaml = "^6.0.12.12" +ulid = "^1.1" +sqlalchemy = "1.4.51" +snowflake-connector-python = "3.6.0" +snowflake-sqlalchemy = "^1.5.1" +duckdb-engine = "^0.10.0" +requests = "^2.31.0" +pyarrow = "^14.0.2" + +# Psycopg3 is not supported in SQLAlchemy 1.x: +# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +rich = "^13.7.0" +pendulum = "<=3.0.0" +python-dotenv = "^1.0.1" + + +[tool.poetry.group.dev.dependencies] +docker = "^7.0.0" +faker = "^21.0.0" +mypy = "^1.7.1" +pandas-stubs = "^2.1.4.231218" +pdoc = "^14.3.0" +pyarrow-stubs = "^10.0.1.7" +pytest = "^7.4.3" +pytest-docker = "^2.0.1" +pytest-mypy = "^0.10.3" +ruff = "^0.1.11" +types-jsonschema = "^4.20.0.0" +google-cloud-secret-manager = "^2.17.0" +types-requests = "2.31.0.4" +freezegun = "^1.4.0" +airbyte-source-faker = "^6.0.0" +viztracer = "^0.16.2" +tomli = "^2.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "requires_creds: marks a test as requiring credentials (skip when secrets unavailable)" +] + +[tool.ruff.pylint] +max-args = 8 # Relaxed from default of 5 +max-branches = 15 # Relaxed from default of 12 + +[tool.ruff] +target-version = "py39" +select = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + "A", # flake8-builtins + "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async + "B", # flake8-bugbear + "FBT", # flake8-boolean-trap + "BLE", # Blind except + "C4", # flake8-comprehensions + "C90", # mccabe (complexity) + "COM", # flake8-commas + "CPY", # missing copyright notice + # "D", # pydocstyle # TODO: Re-enable when adding docstrings + "DTZ", # flake8-datetimez + "E", # pycodestyle (errors) + "ERA", # flake8-eradicate (commented out code) + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FIX", # flake8-fixme + "FLY", # flynt + "FURB", # Refurb + "I", # isort + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "LOG", # flake8-logging + "N", # pep8-naming + "PD", # pandas-vet + "PERF", # Perflint + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PL", # Pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "SIM", # flake8-simplify + "SLF", # flake8-self + "SLOT", # flake8-slots + "T10", # debugger calls + # "T20", # flake8-print # TODO: Re-enable once we have logging + "TCH", # flake8-type-checking + "TD", # flake8-todos + "TID", # flake8-tidy-imports + "TRY", # tryceratops + "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead. + "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur. + "UP", # pyupgrade + "W", # pycodestyle (warnings) + "YTT", # flake8-2020 +] +ignore = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + + # These we don't agree with or don't want to prioritize to enforce: + "ANN003", # kwargs missing type annotations + "ANN101", # Type annotations for 'self' args + "COM812", # Because it conflicts with ruff auto-format + "EM", # flake8-errmsgs (may reconsider later) + "DJ", # Django linting + "G", # flake8-logging-format + "ISC001", # Conflicts with ruff auto-format + "NPY", # NumPy-specific rules + "PIE790", # Allow unnecssary 'pass' (sometimes useful for readability) + "PERF203", # exception handling in loop + "S", # flake8-bandit (noisy, security related) + "SIM910", # Allow "None" as second argument to Dict.get(). "Explicit is better than implicit." + "TD002", # Require author for TODOs + "TRIO", # flake8-trio (opinionated, noisy) + "INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py. + + # TODO: Consider re-enabling these before release: + "A003", # Class attribute 'type' is shadowing a Python builtin + "BLE001", # Do not catch blind exception: Exception + "ERA001", # Remove commented-out code + "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003) + "PLW0603", # Using the global statement to update _cache is discouraged + "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002 +] +fixable = ["ALL"] +unfixable = [ + "ERA001", # Commented-out code (avoid silent loss of code) + "T201" # print() calls (avoid silent loss of code / log messages) +] +line-length = 100 +extend-exclude = ["docs", "test", "tests"] +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.isort] +force-sort-within-sections = false +lines-after-imports = 2 +known-first-party = ["airbyte_cdk", "airbyte_protocol"] +known-local-folder = ["airbyte_lib"] +required-imports = ["from __future__ import annotations"] +known-third-party = [] +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder" +] + +[tool.ruff.mccabe] +max-complexity = 24 + +[tool.ruff.pycodestyle] +ignore-overlong-task-comments = true + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.flake8-annotations] +allow-star-arg-any = false +ignore-fully-untyped = false + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +preview = false +docstring-code-format = true + +[tool.mypy] +# Platform configuration +python_version = "3.9" +# imports related +ignore_missing_imports = true +follow_imports = "silent" +# None and Optional handling +no_implicit_optional = true +strict_optional = true +# Configuring warnings +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +warn_return_any = false +# Untyped definitions and calls +check_untyped_defs = true +disallow_untyped_calls = false +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = false +# Disallow dynamic typing +disallow_subclassing_any = true +disallow_any_unimported = false +disallow_any_expr = false +disallow_any_decorated = false +disallow_any_explicit = false +disallow_any_generics = false +# Miscellaneous strictness flags +allow_untyped_globals = false +allow_redefinition = false +local_partial_types = false +implicit_reexport = true +strict_equality = true +# Configuring error messages +show_error_context = false +show_column_numbers = false +show_error_codes = true +exclude = ["docs", "test", "tests"] + +[[tool.mypy.overrides]] +module = [ + "airbyte_protocol", + "airbyte_protocol.models" +] +ignore_missing_imports = true # No stubs yet (😢) + +[tool.poetry.scripts] +generate-docs = "docs:run" +airbyte-lib-validate-source = "airbyte_lib.validate:run" + +[tool.poe.tasks] +test = "pytest tests" + +[tool.airbyte_ci] +extra_poetry_groups = ["dev"] +poe_tasks = ["test"] +required_environment_variables = ["GCP_GSM_CREDENTIALS"] +side_car_docker_engine = true diff --git a/airbyte-lib/tests/conftest.py b/airbyte-lib/tests/conftest.py new file mode 100644 index 000000000000..0824d77f3eb8 --- /dev/null +++ b/airbyte-lib/tests/conftest.py @@ -0,0 +1,250 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Global pytest fixtures.""" + +import json +import logging +import os +import shutil +import socket +import subprocess +import time + +import ulid +from airbyte_lib.caches.snowflake import SnowflakeCacheConfig + +import docker +import psycopg2 as psycopg +import pytest +from _pytest.nodes import Item +from google.cloud import secretmanager +from pytest_docker.plugin import get_docker_ip +from sqlalchemy import create_engine + +from airbyte_lib.caches import PostgresCacheConfig + +logger = logging.getLogger(__name__) + + +PYTEST_POSTGRES_IMAGE = "postgres:13" +PYTEST_POSTGRES_CONTAINER = "postgres_pytest_container" +PYTEST_POSTGRES_PORT = 5432 + +LOCAL_TEST_REGISTRY_URL = "./tests/integration_tests/fixtures/registry.json" + + +def pytest_collection_modifyitems(items: list[Item]) -> None: + """Override default pytest behavior, sorting our tests in a sensible execution order. + + In general, we want faster tests to run first, so that we can get feedback faster. + + Running lint tests first is helpful because they are fast and can catch typos and other errors. + + Otherwise tests are run based on an alpha-based natural sort, where 'unit' tests run after + 'integration' tests because 'u' comes after 'i' alphabetically. + """ + def test_priority(item: Item) -> int: + if item.get_closest_marker(name="slow"): + return 9 # slow tests have the lowest priority + elif 'lint_tests' in str(item.fspath): + return 1 # lint tests have high priority + elif 'unit_tests' in str(item.fspath): + return 2 # unit tests have highest priority + elif 'docs_tests' in str(item.fspath): + return 3 # doc tests have medium priority + elif 'integration_tests' in str(item.fspath): + return 4 # integration tests have the lowest priority + else: + return 5 # all other tests have lower priority + + # Sort the items list in-place based on the test_priority function + items.sort(key=test_priority) + + +def is_port_in_use(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + +@pytest.fixture(scope="session", autouse=True) +def remove_postgres_container(): + client = docker.from_env() + if is_port_in_use(PYTEST_POSTGRES_PORT): + try: + container = client.containers.get( + PYTEST_POSTGRES_CONTAINER, + ) + container.stop() + container.remove() + except docker.errors.NotFound: + pass # Container not found, nothing to do. + + +def test_pg_connection(host) -> bool: + pg_url = f"postgresql://postgres:postgres@{host}:{PYTEST_POSTGRES_PORT}/postgres" + + max_attempts = 120 + for attempt in range(max_attempts): + try: + conn = psycopg.connect(pg_url) + conn.close() + return True + except psycopg.OperationalError: + logger.info(f"Waiting for postgres to start (attempt {attempt + 1}/{max_attempts})") + time.sleep(1.0) + + else: + return False + + +@pytest.fixture(scope="session") +def pg_dsn(): + client = docker.from_env() + try: + client.images.get(PYTEST_POSTGRES_IMAGE) + except docker.errors.ImageNotFound: + # Pull the image if it doesn't exist, to avoid failing our sleep timer + # if the image needs to download on-demand. + client.images.pull(PYTEST_POSTGRES_IMAGE) + + try: + previous_container = client.containers.get(PYTEST_POSTGRES_CONTAINER) + previous_container.remove() + except docker.errors.NotFound: + pass + + postgres_is_running = False + postgres = client.containers.run( + image=PYTEST_POSTGRES_IMAGE, + name=PYTEST_POSTGRES_CONTAINER, + environment={"POSTGRES_USER": "postgres", "POSTGRES_PASSWORD": "postgres", "POSTGRES_DB": "postgres"}, + ports={"5432/tcp": PYTEST_POSTGRES_PORT}, + detach=True, + ) + + attempts = 10 + while not postgres_is_running and attempts > 0: + try: + postgres.reload() + postgres_is_running = postgres.status == "running" + except docker.errors.NotFound: + attempts -= 1 + time.sleep(3) + if not postgres_is_running: + raise Exception(f"Failed to start the PostgreSQL container. Status: {postgres.status}.") + + final_host = None + if host := os.environ.get("DOCKER_HOST_NAME"): + final_host = host if test_pg_connection(host) else None + else: + # Try to connect to the database using localhost and the docker host IP + for host in ["127.0.0.1", "localhost", "host.docker.internal", "172.17.0.1"]: + if test_pg_connection(host): + final_host = host + break + + if final_host is None: + raise Exception(f"Failed to connect to the PostgreSQL database on host {host}.") + + yield final_host + # Stop and remove the container after the tests are done + postgres.stop() + postgres.remove() + + +@pytest.fixture +def new_pg_cache_config(pg_dsn): + """Fixture to return a fresh cache. + + Each test that uses this fixture will get a unique table prefix. + """ + config = PostgresCacheConfig( + host=pg_dsn, + port=PYTEST_POSTGRES_PORT, + username="postgres", + password="postgres", + database="postgres", + schema_name="public", + + # TODO: Move this to schema name when we support it (breaks as of 2024-01-31): + table_prefix=f"test{str(ulid.ULID())[-6:]}_", + ) + yield config + + +@pytest.fixture +def snowflake_config(): + if "GCP_GSM_CREDENTIALS" not in os.environ: + raise Exception("GCP_GSM_CREDENTIALS env variable not set, can't fetch secrets for Snowflake. Make sure they are set up as described: https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access") + secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( + json.loads(os.environ["GCP_GSM_CREDENTIALS"]) + ) + secret = json.loads( + secret_client.access_secret_version( + name="projects/dataline-integration-testing/secrets/AIRBYTE_LIB_SNOWFLAKE_CREDS/versions/latest" + ).payload.data.decode("UTF-8") + ) + config = SnowflakeCacheConfig( + account=secret["account"], + username=secret["username"], + password=secret["password"], + database=secret["database"], + warehouse=secret["warehouse"], + role=secret["role"], + schema_name=f"test{str(ulid.ULID()).lower()[-6:]}", + ) + + yield config + + engine = create_engine(config.get_sql_alchemy_url()) + with engine.begin() as connection: + connection.execute(f"DROP SCHEMA IF EXISTS {config.schema_name}") + + +@pytest.fixture(autouse=True) +def source_test_registry(monkeypatch): + """ + Set environment variables for the test source. + + These are applied to this test file only. + + This means the normal registry is not usable. Expect AirbyteConnectorNotRegisteredError for + other connectors. + """ + env_vars = { + "AIRBYTE_LOCAL_REGISTRY": LOCAL_TEST_REGISTRY_URL, + } + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + + +@pytest.fixture(autouse=True) +def do_not_track(monkeypatch): + """ + Set environment variables for the test source. + + These are applied to this test file only. + """ + env_vars = { + "DO_NOT_TRACK": "true" + } + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + + +@pytest.fixture(scope="package") +def source_test_installation(): + """ + Prepare test environment. This will pre-install the test source from the fixtures array and set + the environment variable to use the local json file as registry. + """ + venv_dir = ".venv-source-test" + if os.path.exists(venv_dir): + shutil.rmtree(venv_dir) + + subprocess.run(["python", "-m", "venv", venv_dir], check=True) + subprocess.run([f"{venv_dir}/bin/pip", "install", "-e", "./tests/integration_tests/fixtures/source-test"], check=True) + + yield + + shutil.rmtree(venv_dir) diff --git a/airbyte-lib/tests/docs_tests/__init__.py b/airbyte-lib/tests/docs_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/docs_tests/test_docs_checked_in.py b/airbyte-lib/tests/docs_tests/test_docs_checked_in.py new file mode 100644 index 000000000000..54614c7cd621 --- /dev/null +++ b/airbyte-lib/tests/docs_tests/test_docs_checked_in.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +import docs + + +def test_docs_checked_in(): + """ + Docs need to be generated via `poetry run generate-docs` and checked in to the repo. + + This test runs the docs generation and compares the output with the checked in docs. + It will fail if there are any differences. + """ + + docs.run() + + # compare the generated docs with the checked in docs + diff = os.system("git diff --exit-code docs/generated") + + # if there is a diff, fail the test + assert diff == 0, "Docs are out of date. Please run `poetry run generate-docs` and commit the changes." diff --git a/airbyte-lib/tests/docs_tests/test_validate_changelog.py b/airbyte-lib/tests/docs_tests/test_validate_changelog.py new file mode 100644 index 000000000000..7481d014af2a --- /dev/null +++ b/airbyte-lib/tests/docs_tests/test_validate_changelog.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import tomli + + +def test_validate_changelog(): + """ + Publishing a version involves bumping the version in pyproject.toml and adding a changelog entry. + This test ensures that the changelog entry is present. + """ + + # get the version from pyproject.toml + with open("pyproject.toml") as f: + contents = tomli.loads(f.read()) + version = contents["tool"]["poetry"]["version"] + + # get the changelog + with open("README.md") as f: + readme = f.read() + changelog = readme.split("## Changelog")[-1] + + # check that the changelog contains the version + assert version in changelog, f"Version {version} is missing from the changelog in README.md. Please add it." diff --git a/airbyte-lib/tests/integration_tests/__init__.py b/airbyte-lib/tests/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json b/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json new file mode 100644 index 000000000000..3ce4b45a3209 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json @@ -0,0 +1 @@ +{ "apiKey": "wrong" } diff --git a/airbyte-lib/tests/integration_tests/fixtures/registry.json b/airbyte-lib/tests/integration_tests/fixtures/registry.json new file mode 100644 index 000000000000..d356be8c0ccd --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/registry.json @@ -0,0 +1,88 @@ +{ + "sources": [ + { + "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe363", + "name": "Test Source", + "dockerRepository": "airbyte/source-test", + "dockerImageTag": "0.0.1", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "icon": "test.svg", + "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", + "sourceType": "api", + "remoteRegistries": { + "pypi": { + "packageName": "airbyte-source-test", + "enabled": true + } + }, + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service" + } + } + } + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "supportLevel": "community", + "ab_internal": { + "sl": 100, + "ql": 200 + }, + "tags": ["language:python"], + "githubIssueLabel": "source-test", + "license": "MIT" + }, + { + "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe333", + "name": "Non-published source", + "dockerRepository": "airbyte/source-non-published", + "dockerImageTag": "0.0.1", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "icon": "test.svg", + "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", + "sourceType": "api", + "remoteRegistries": { + "pypi": { + "packageName": "airbyte-source-non-published", + "enabled": false + } + }, + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service" + } + } + } + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "supportLevel": "community", + "ab_internal": { + "sl": 100, + "ql": 200 + }, + "tags": ["language:python"], + "githubIssueLabel": "source-source-non-published", + "license": "MIT" + } + ] +} diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml b/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml new file mode 100644 index 000000000000..6f1494d43b27 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml @@ -0,0 +1,17 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 + dockerImageTag: 0.0.0 + dockerRepository: airbyte/source-broken + githubIssueLabel: source-broken + name: Test + releaseDate: 2023-08-25 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-broken +metadataSpecVersion: "1.0" diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py new file mode 100644 index 000000000000..516112718b7e --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import setup + +setup( + name="airbyte-source-broken", + version="0.0.1", + description="Test Soutce", + author="Airbyte", + author_email="contact@airbyte.io", + packages=["source_broken"], + entry_points={ + "console_scripts": [ + "source-broken=source_broken.run:run", + ], + }, +) diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py new file mode 100644 index 000000000000..c777271f249a --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py @@ -0,0 +1,4 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +def run(): + raise Exception("Could not run") \ No newline at end of file diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml b/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml new file mode 100644 index 000000000000..8712af99c05d --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml @@ -0,0 +1,17 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 + dockerImageTag: 0.0.0 + dockerRepository: airbyte/source-test + githubIssueLabel: source-test + name: Test + releaseDate: 2023-08-25 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-test +metadataSpecVersion: "1.0" diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py new file mode 100644 index 000000000000..0035f1eda76a --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import setup + +setup( + name="airbyte-source-test", + version="0.0.1", + description="Test Soutce", + author="Airbyte", + author_email="contact@airbyte.io", + packages=["source_test"], + entry_points={ + "console_scripts": [ + "source-test=source_test.run:run", + ], + }, +) diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/__init__.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py new file mode 100644 index 000000000000..5f4ae3f1e939 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py @@ -0,0 +1,150 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import sys + +sample_catalog = { + "type": "CATALOG", + "catalog": { + "streams": [ + { + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["column1"], + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + }, + }, + }, + { + "name": "stream2", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + "empty_column": {"type": "string"}, + }, + }, + }, + { + "name": "always-empty-stream", + "description": "This stream always emits zero records, to test handling of empty datasets.", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + "empty_column": {"type": "string"}, + }, + }, + }, + ] + }, +} + +sample_connection_specification = { + "type": "SPEC", + "spec": { + "documentationUrl": "https://example.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service", + } + }, + }, + }, +} + +sample_connection_check_success = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "SUCCEEDED"}, +} + +sample_connection_check_failure = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "FAILED", "message": "An error"}, +} + +sample_record1_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record2_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value2", "column2": 2}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record_stream2 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream2", + "emitted_at": 123456789, + }, +} + + +def parse_args(): + arg_dict = {} + args = sys.argv[2:] + for i in range(0, len(args), 2): + arg_dict[args[i]] = args[i + 1] + + return arg_dict + + +def get_json_file(path): + with open(path, "r") as f: + return json.load(f) + + +def run(): + args = sys.argv[1:] + if args[0] == "spec": + print(json.dumps(sample_connection_specification)) + elif args[0] == "discover": + print(json.dumps(sample_catalog)) + elif args[0] == "check": + args = parse_args() + config = get_json_file(args["--config"]) + if config.get("apiKey").startswith("test"): + print(json.dumps(sample_connection_check_success)) + else: + print(json.dumps(sample_connection_check_failure)) + elif args[0] == "read": + args = parse_args() + catalog = get_json_file(args["--catalog"]) + config = get_json_file(args["--config"]) + print(json.dumps({"type": "LOG", "log": {"level": "INFO", "message": "Starting sync"}})) + for stream in catalog["streams"]: + if stream["stream"]["name"] == "stream1": + print(json.dumps(sample_record1_stream1)) + if config.get("apiKey") == "test_fail_during_sync": + raise Exception("An error") + print(json.dumps(sample_record2_stream1)) + elif stream["stream"]["name"] == "stream2": + print(json.dumps(sample_record_stream2)) diff --git a/airbyte-lib/tests/integration_tests/fixtures/valid_config.json b/airbyte-lib/tests/integration_tests/fixtures/valid_config.json new file mode 100644 index 000000000000..fbe094d80a44 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/valid_config.json @@ -0,0 +1 @@ +{ "apiKey": "test" } diff --git a/airbyte-lib/tests/integration_tests/test_install.py b/airbyte-lib/tests/integration_tests/test_install.py new file mode 100644 index 000000000000..3350d54cbfb5 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_install.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from gettext import install +import pytest + +from airbyte_lib._factories.connector_factories import get_source +from airbyte_lib import exceptions as exc + + +def test_install_failure_log_pypi(): + """Test that the install log is created and contains the expected content.""" + with pytest.raises(exc.AirbyteConnectorNotRegisteredError): + source = get_source("source-not-found") + + with pytest.raises(exc.AirbyteConnectorInstallationError) as exc_info: + source = get_source( + "source-not-found", + pip_url="https://pypi.org/project/airbyte-not-found", + install_if_missing=True, + ) + + # Check that the stderr log contains the expected content from a failed pip install + assert 'Could not install requirement' in str(exc_info.value.__cause__.log_text) diff --git a/airbyte-lib/tests/integration_tests/test_snowflake_cache.py b/airbyte-lib/tests/integration_tests/test_snowflake_cache.py new file mode 100644 index 000000000000..4ac08f4bebe3 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_snowflake_cache.py @@ -0,0 +1,156 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Integration tests which leverage the source-faker connector to test the framework end-to-end. + +Since source-faker is included in dev dependencies, we can assume `source-faker` is installed +and available on PATH for the poetry-managed venv. +""" +from __future__ import annotations +from collections.abc import Generator +import os +import sys +import shutil +from pathlib import Path + +import pytest +import ulid +import viztracer + +from airbyte_cdk.models import ConfiguredAirbyteCatalog + +import airbyte_lib as ab +from airbyte_lib import caches + + +# Product count is always the same, regardless of faker scale. +NUM_PRODUCTS = 100 + +SEED_A = 1234 +SEED_B = 5678 + +# Number of records in each of the 'users' and 'purchases' streams. +FAKER_SCALE_A = 200 +# We want this to be different from FAKER_SCALE_A. +FAKER_SCALE_B = 300 + + +# Patch PATH to include the source-faker executable. + +@pytest.fixture(autouse=True) +def add_venv_bin_to_path(monkeypatch): + # Get the path to the bin directory of the virtual environment + venv_bin_path = os.path.join(sys.prefix, 'bin') + + # Add the bin directory to the PATH + new_path = f"{venv_bin_path}:{os.environ['PATH']}" + monkeypatch.setenv('PATH', new_path) + + +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker_seed_a() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + source = ab.get_source( + "source-faker", + local_executable="source-faker", + config={ + "count": FAKER_SCALE_A, + "seed": SEED_A, + "parallelism": 16, # Otherwise defaults to 4. + }, + install_if_missing=False, # Should already be on PATH + ) + source.check() + source.select_streams([ + "users", + ]) + return source + + +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker_seed_b() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + source = ab.get_source( + "source-faker", + local_executable="source-faker", + config={ + "count": FAKER_SCALE_B, + "seed": SEED_B, + "parallelism": 16, # Otherwise defaults to 4. + }, + install_if_missing=False, # Should already be on PATH + ) + source.check() + source.select_streams([ + "users", + ]) + return source + + +@pytest.fixture(scope="function") +def snowflake_cache(snowflake_config) -> Generator[caches.SnowflakeCache, None, None]: + """Fixture to return a fresh cache.""" + cache: caches.SnowflakeCache = caches.SnowflakeSQLCache(snowflake_config) + yield cache + # TODO: Delete cache DB file after test is complete. + return + + +# Uncomment this line if you want to see performance trace logs. +# You can render perf traces using the viztracer CLI or the VS Code VizTracer Extension. +#@viztracer.trace_and_save(output_dir=".pytest_cache/snowflake_trace/") +@pytest.mark.requires_creds +@pytest.mark.slow +def test_faker_read_to_snowflake( + source_faker_seed_a: ab.Source, + snowflake_cache: ab.SnowflakeCache, +) -> None: + """Test that the append strategy works as expected.""" + result = source_faker_seed_a.read( + snowflake_cache, write_strategy="replace", force_full_refresh=True + ) + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_A + + +@pytest.mark.requires_creds +@pytest.mark.slow +def test_replace_strategy( + source_faker_seed_a: ab.Source, + snowflake_cache: ab.SnowflakeCache, +) -> None: + """Test that the append strategy works as expected.""" + for _ in range(2): + result = source_faker_seed_a.read( + snowflake_cache, write_strategy="replace", force_full_refresh=True + ) + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_A + + +@pytest.mark.requires_creds +@pytest.mark.slow +def test_merge_strategy( + source_faker_seed_a: ab.Source, + source_faker_seed_b: ab.Source, + snowflake_cache: ab.DuckDBCache, +) -> None: + """Test that the merge strategy works as expected. + + Since all streams have primary keys, we should expect the auto strategy to be identical to the + merge strategy. + """ + # First run, seed A (counts should match the scale or the product count) + result = source_faker_seed_a.read(snowflake_cache, write_strategy="merge") + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_A + + # Second run, also seed A (should have same exact data, no change in counts) + result = source_faker_seed_a.read(snowflake_cache, write_strategy="merge") + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_A + + # Third run, seed B - should increase record count to the scale of B, which is greater than A. + # TODO: See if we can reliably predict the exact number of records, since we use fixed seeds. + result = source_faker_seed_b.read(snowflake_cache, write_strategy="merge") + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_B + + # Third run, seed A again - count should stay at scale B, since A is smaller. + # TODO: See if we can reliably predict the exact number of records, since we use fixed seeds. + result = source_faker_seed_a.read(snowflake_cache, write_strategy="merge") + assert len(list(result.cache.streams["users"])) == FAKER_SCALE_B diff --git a/airbyte-lib/tests/integration_tests/test_source_faker_integration.py b/airbyte-lib/tests/integration_tests/test_source_faker_integration.py new file mode 100644 index 000000000000..244c52f0e901 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_source_faker_integration.py @@ -0,0 +1,282 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Integration tests which leverage the source-faker connector to test the framework end-to-end. + +Since source-faker is included in dev dependencies, we can assume `source-faker` is installed +and available on PATH for the poetry-managed venv. +""" +from __future__ import annotations +from collections.abc import Generator +import os +import sys +import shutil +from pathlib import Path + +import pytest +import ulid +import viztracer + +from airbyte_cdk.models import ConfiguredAirbyteCatalog + +import airbyte_lib as ab +from airbyte_lib import caches + + +# Product count is always the same, regardless of faker scale. +NUM_PRODUCTS = 100 + +SEED_A = 1234 +SEED_B = 5678 + +# Number of records in each of the 'users' and 'purchases' streams. +FAKER_SCALE_A = 200 +# We want this to be different from FAKER_SCALE_A. +FAKER_SCALE_B = 300 + + +# Patch PATH to include the source-faker executable. + +@pytest.fixture(autouse=True) +def add_venv_bin_to_path(monkeypatch): + # Get the path to the bin directory of the virtual environment + venv_bin_path = os.path.join(sys.prefix, 'bin') + + # Add the bin directory to the PATH + new_path = f"{venv_bin_path}:{os.environ['PATH']}" + monkeypatch.setenv('PATH', new_path) + + +def test_which_source_faker() -> None: + """Test that source-faker is available on PATH.""" + assert shutil.which("source-faker") is not None, \ + f"Can't find source-faker on PATH: {os.environ['PATH']}" + + +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker_seed_a() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + source = ab.get_source( + "source-faker", + local_executable="source-faker", + config={ + "count": FAKER_SCALE_A, + "seed": SEED_A, + "parallelism": 16, # Otherwise defaults to 4. + }, + install_if_missing=False, # Should already be on PATH + ) + source.check() + source.select_streams([ + "users", + "products", + "purchases", + ]) + return source + + +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker_seed_b() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + source = ab.get_source( + "source-faker", + local_executable="source-faker", + config={ + "count": FAKER_SCALE_B, + "seed": SEED_B, + "parallelism": 16, # Otherwise defaults to 4. + }, + install_if_missing=False, # Should already be on PATH + ) + source.check() + source.select_streams([ + "products", + "purchases", + "users", + ]) + return source + + +@pytest.fixture(scope="function") +def duckdb_cache() -> Generator[caches.DuckDBCache, None, None]: + """Fixture to return a fresh cache.""" + cache: caches.DuckDBCache = ab.new_local_cache() + yield cache + # TODO: Delete cache DB file after test is complete. + return + + +@pytest.fixture(scope="function") +def postgres_cache(new_pg_cache_config) -> Generator[caches.PostgresCache, None, None]: + """Fixture to return a fresh cache.""" + cache: caches.PostgresCache = caches.PostgresCache(config=new_pg_cache_config) + yield cache + # TODO: Delete cache DB file after test is complete. + return + + +@pytest.fixture +def all_cache_types( + duckdb_cache: ab.DuckDBCache, + postgres_cache: ab.PostgresCache, +): + _ = postgres_cache + return [ + duckdb_cache, + postgres_cache, + ] + +def test_faker_pks( + source_faker_seed_a: ab.Source, + duckdb_cache: ab.DuckDBCache, +) -> None: + """Test that the append strategy works as expected.""" + + catalog: ConfiguredAirbyteCatalog = source_faker_seed_a.configured_catalog + + assert catalog.streams[0].primary_key + assert catalog.streams[1].primary_key + + read_result = source_faker_seed_a.read(duckdb_cache, write_strategy="append") + assert read_result.cache._get_primary_keys("products") == ["id"] + assert read_result.cache._get_primary_keys("purchases") == ["id"] + + +@pytest.mark.slow +def test_replace_strategy( + source_faker_seed_a: ab.Source, + all_cache_types: ab.DuckDBCache, +) -> None: + """Test that the append strategy works as expected.""" + for cache in all_cache_types: # Function-scoped fixtures can't be used in parametrized(). + for _ in range(2): + result = source_faker_seed_a.read( + cache, write_strategy="replace", force_full_refresh=True + ) + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_A + + +@pytest.mark.slow +def test_append_strategy( + source_faker_seed_a: ab.Source, + all_cache_types: ab.DuckDBCache, +) -> None: + """Test that the append strategy works as expected.""" + for cache in all_cache_types: # Function-scoped fixtures can't be used in parametrized(). + for iteration in range(1, 3): + result = source_faker_seed_a.read(cache, write_strategy="append") + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS * iteration + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_A * iteration + + +@pytest.mark.slow +@pytest.mark.parametrize("strategy", ["merge", "auto"]) +def test_merge_strategy( + strategy: str, + source_faker_seed_a: ab.Source, + source_faker_seed_b: ab.Source, + all_cache_types: ab.DuckDBCache, +) -> None: + """Test that the merge strategy works as expected. + + Since all streams have primary keys, we should expect the auto strategy to be identical to the + merge strategy. + """ + for cache in all_cache_types: # Function-scoped fixtures can't be used in parametrized(). + # First run, seed A (counts should match the scale or the product count) + result = source_faker_seed_a.read(cache, write_strategy=strategy) + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_A + + # Second run, also seed A (should have same exact data, no change in counts) + result = source_faker_seed_a.read(cache, write_strategy=strategy) + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_A + + # Third run, seed B - should increase record count to the scale of B, which is greater than A. + # TODO: See if we can reliably predict the exact number of records, since we use fixed seeds. + result = source_faker_seed_b.read(cache, write_strategy=strategy) + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_B + + # Third run, seed A again - count should stay at scale B, since A is smaller. + # TODO: See if we can reliably predict the exact number of records, since we use fixed seeds. + result = source_faker_seed_a.read(cache, write_strategy=strategy) + assert len(list(result.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result.cache.streams["purchases"])) == FAKER_SCALE_B + + +def test_incremental_sync( + source_faker_seed_a: ab.Source, + source_faker_seed_b: ab.Source, + duckdb_cache: ab.DuckDBCache, +) -> None: + config_a = source_faker_seed_a.get_config() + config_b = source_faker_seed_b.get_config() + config_a["always_updated"] = False + config_b["always_updated"] = False + source_faker_seed_a.set_config(config_a) + source_faker_seed_b.set_config(config_b) + + result1 = source_faker_seed_a.read(duckdb_cache) + assert len(list(result1.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result1.cache.streams["purchases"])) == FAKER_SCALE_A + assert result1.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2 + + assert not duckdb_cache.get_state() == [] + + # Second run should not return records as it picks up the state and knows it's up to date. + result2 = source_faker_seed_b.read(duckdb_cache) + + assert result2.processed_records == 0 + assert len(list(result2.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result2.cache.streams["purchases"])) == FAKER_SCALE_A + + +def test_incremental_state_cache_persistence( + source_faker_seed_a: ab.Source, + source_faker_seed_b: ab.Source, +) -> None: + config_a = source_faker_seed_a.get_config() + config_b = source_faker_seed_b.get_config() + config_a["always_updated"] = False + config_b["always_updated"] = False + source_faker_seed_a.set_config(config_a) + source_faker_seed_b.set_config(config_b) + cache_name = str(ulid.ULID()) + cache = ab.new_local_cache(cache_name) + result = source_faker_seed_a.read(cache) + assert result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2 + second_cache = ab.new_local_cache(cache_name) + # The state should be persisted across cache instances. + result2 = source_faker_seed_b.read(second_cache) + assert result2.processed_records == 0 + + assert not second_cache.get_state() == [] + assert len(list(result2.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result2.cache.streams["purchases"])) == FAKER_SCALE_A + + +def test_incremental_state_prefix_isolation( + source_faker_seed_a: ab.Source, + source_faker_seed_b: ab.Source, +) -> None: + """ + Test that state in the cache correctly isolates streams when different table prefixes are used + """ + config_a = source_faker_seed_a.get_config() + config_a["always_updated"] = False + source_faker_seed_a.set_config(config_a) + cache_name = str(ulid.ULID()) + db_path = Path(f"./.cache/{cache_name}.duckdb") + cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="prefix_")) + different_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="different_prefix_")) + + result = source_faker_seed_a.read(cache) + assert result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2 + + result2 = source_faker_seed_b.read(different_prefix_cache) + assert result2.processed_records == NUM_PRODUCTS + FAKER_SCALE_B * 2 + + assert len(list(result2.cache.streams["products"])) == NUM_PRODUCTS + assert len(list(result2.cache.streams["purchases"])) == FAKER_SCALE_B diff --git a/airbyte-lib/tests/integration_tests/test_source_test_fixture.py b/airbyte-lib/tests/integration_tests/test_source_test_fixture.py new file mode 100644 index 000000000000..5e3a05e683a9 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_source_test_fixture.py @@ -0,0 +1,820 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from collections.abc import Mapping +import os +import shutil +import itertools +from contextlib import nullcontext as does_not_raise +from typing import Any +from unittest.mock import Mock, call, patch +import tempfile +from pathlib import Path +from airbyte_lib.caches.base import SQLCacheBase + +from sqlalchemy import column, text + +import airbyte_lib as ab +from airbyte_lib.caches import SnowflakeCacheConfig, SnowflakeSQLCache +import pandas as pd +import pytest + +from airbyte_lib.caches import PostgresCache, PostgresCacheConfig +from airbyte_lib import registry +from airbyte_lib.version import get_version +from airbyte_lib.results import ReadResult +from airbyte_lib.datasets import CachedDataset, LazyDataset, SQLDataset +import airbyte_lib as ab + +from airbyte_lib.results import ReadResult +from airbyte_lib import exceptions as exc +import ulid + + +@pytest.fixture(scope="module", autouse=True) +def autouse_source_test_installation(source_test_installation): + return + + +@pytest.fixture(scope="function", autouse=True) +def autouse_source_test_registry(source_test_registry): + return + + +@pytest.fixture +def source_test(source_test_env) -> ab.Source: + return ab.get_source("source-test", config={"apiKey": "test"}) + + +@pytest.fixture +def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]: + return { + "stream1": [ + {"column1": "value1", "column2": 1}, + {"column1": "value2", "column2": 2}, + ], + "stream2": [ + {"column1": "value1", "column2": 1, "empty_column": None}, + ], + "always-empty-stream": [], + } + +def test_registry_get(): + metadata = registry.get_connector_metadata("source-test") + assert metadata.name == "source-test" + assert metadata.latest_available_version == "0.0.1" + + +def test_registry_list() -> None: + assert registry.get_available_connectors() == ["source-test"] + + +def test_list_streams(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source( + "source-test", config={"apiKey": "test"}, install_if_missing=False + ) + assert source.get_available_streams() == list(expected_test_stream_data.keys()) + + +def test_invalid_config(): + source = ab.get_source( + "source-test", config={"apiKey": 1234}, install_if_missing=False + ) + with pytest.raises(exc.AirbyteConnectorCheckFailedError): + source.check() + + +def test_ensure_installation_detection(): + """Assert that install isn't called, since the connector is already installed by the fixture.""" + with patch("airbyte_lib._executor.VenvExecutor.install") as mock_venv_install, \ + patch("airbyte_lib.source.Source.install") as mock_source_install, \ + patch("airbyte_lib._executor.VenvExecutor.ensure_installation") as mock_ensure_installed: + source = ab.get_source( + "source-test", + config={"apiKey": 1234}, + pip_url="https://pypi.org/project/airbyte-not-found", + install_if_missing=True, + ) + assert mock_ensure_installed.call_count == 1 + assert not mock_venv_install.called + assert not mock_source_install.called + + +def test_source_yaml_spec(): + source = ab.get_source( + "source-test", config={"apiKey": 1234}, install_if_missing=False + ) + assert source._yaml_spec.startswith("connectionSpecification:\n $schema:") + + +def test_non_existing_connector(): + with pytest.raises(Exception): + ab.get_source("source-not-existing", config={"apiKey": "abc"}) + +def test_non_enabled_connector(): + with pytest.raises(exc.AirbyteConnectorNotPyPiPublishedError): + ab.get_source("source-non-published", config={"apiKey": "abc"}) + +@pytest.mark.parametrize( + "latest_available_version, requested_version, raises", + [ + ("0.0.1", "latest", False), + ("0.0.1", "0.0.1", False), + ("0.0.1", None, False), + ("1.2.3", None, False), # Don't raise if a version is not requested + ("1.2.3", "latest", True), + ("1.2.3", "1.2.3", True), + ]) +def test_version_enforcement( + raises: bool, + latest_available_version, + requested_version, +): + """" + Ensures version enforcement works as expected: + * If no version is specified, the current version is accepted + * If the version is specified as "latest", only the latest available version is accepted + * If the version is specified as a semantic version, only the exact version is accepted + + In this test, the actually installed version is 0.0.1 + """ + patched_entry = registry.ConnectorMetadata( + name="source-test", latest_available_version=latest_available_version, pypi_package_name="airbyte-source-test" + ) + + # We need to initialize the cache before we can patch it. + _ = registry._get_registry_cache() + with patch.dict("airbyte_lib.registry.__cache", {"source-test": patched_entry}, clear=False): + if raises: + with pytest.raises(Exception): + source = ab.get_source( + "source-test", + version=requested_version, + config={"apiKey": "abc"}, + install_if_missing=False, + ) + source.executor.ensure_installation(auto_fix=False) + else: + source = ab.get_source( + "source-test", + version=requested_version, + config={"apiKey": "abc"}, + install_if_missing=False, + ) + if requested_version: # Don't raise if a version is not requested + assert source.executor._get_installed_version(raise_on_error=True) == ( + requested_version or latest_available_version + ).replace("latest", latest_available_version) + source.executor.ensure_installation(auto_fix=False) + + +def test_check(): + source = ab.get_source( + "source-test", + config={"apiKey": "test"}, + install_if_missing=False, + ) + source.check() + + +def test_check_fail(): + source = ab.get_source("source-test", config={"apiKey": "wrong"}) + + with pytest.raises(Exception): + source.check() + + +def test_file_write_and_cleanup() -> None: + """Ensure files are written to the correct location and cleaned up afterwards.""" + with tempfile.TemporaryDirectory() as temp_dir_1, tempfile.TemporaryDirectory() as temp_dir_2: + cache_w_cleanup = ab.new_local_cache(cache_dir=temp_dir_1, cleanup=True) + cache_wo_cleanup = ab.new_local_cache(cache_dir=temp_dir_2, cleanup=False) + + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + _ = source.read(cache_w_cleanup) + _ = source.read(cache_wo_cleanup) + + assert len(list(Path(temp_dir_1).glob("*.parquet"))) == 0, "Expected files to be cleaned up" + assert len(list(Path(temp_dir_2).glob("*.parquet"))) == 3, "Expected files to exist" + + +def assert_cache_data(expected_test_stream_data: dict[str, list[dict[str, str | int]]], cache: SQLCacheBase, streams: list[str] = None): + for stream_name in streams or expected_test_stream_data.keys(): + if len(cache[stream_name]) > 0: + pd.testing.assert_frame_equal( + cache[stream_name].to_pandas(), + pd.DataFrame(expected_test_stream_data[stream_name]), + check_dtype=False, + ) + else: + # stream is empty + assert len(expected_test_stream_data[stream_name]) == 0 + + # validate that the cache doesn't contain any other streams + if streams: + assert len(list(cache.__iter__())) == len(streams) + + +def test_sync_to_duckdb(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = ab.new_local_cache() + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + assert_cache_data(expected_test_stream_data, cache) + + +def test_read_result_mapping(): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + result: ReadResult = source.read(ab.new_local_cache()) + assert len(result) == 3 + assert isinstance(result, Mapping) + assert "stream1" in result + assert "stream2" in result + assert "always-empty-stream" in result + assert "stream3" not in result + assert result.keys() == {"stream1", "stream2", "always-empty-stream"} + + +def test_dataset_list_and_len(expected_test_stream_data): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + result: ReadResult = source.read(ab.new_local_cache()) + stream_1 = result["stream1"] + assert len(stream_1) == 2 + assert len(list(stream_1)) == 2 + # Make sure we can iterate over the stream after calling len + assert list(stream_1) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] + # Make sure we can iterate over the stream a second time + assert list(stream_1) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] + + assert isinstance(result, Mapping) + assert "stream1" in result + assert "stream2" in result + assert "always-empty-stream" in result + assert "stream3" not in result + assert result.keys() == {"stream1", "stream2", "always-empty-stream"} + + +def test_read_from_cache(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """ + Test that we can read from a cache that already has data (identifier by name) + """ + cache_name = str(ulid.ULID()) + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = ab.new_local_cache(cache_name) + + source.read(cache) + + # Create a new cache pointing to the same duckdb file + second_cache = ab.new_local_cache(cache_name) + + + assert_cache_data(expected_test_stream_data, second_cache) + + +def test_read_isolated_by_prefix(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """ + Test that cache correctly isolates streams when different table prefixes are used + """ + cache_name = str(ulid.ULID()) + db_path = Path(f"./.cache/{cache_name}.duckdb") + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="prefix_")) + + source.read(cache) + + same_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="prefix_")) + different_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="different_prefix_")) + no_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix=None)) + + # validate that the cache with the same prefix has the data as expected, while the other two are empty + assert_cache_data(expected_test_stream_data, same_prefix_cache) + assert len(list(different_prefix_cache.__iter__())) == 0 + assert len(list(no_prefix_cache.__iter__())) == 0 + + # read partial data into the other two caches + source.select_streams(["stream1"]) + source.read(different_prefix_cache) + source.read(no_prefix_cache) + + second_same_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="prefix_")) + second_different_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix="different_prefix_")) + second_no_prefix_cache = ab.DuckDBCache(config=ab.DuckDBCacheConfig(db_path=db_path, table_prefix=None)) + + # validate that the first cache still has full data, while the other two have partial data + assert_cache_data(expected_test_stream_data, second_same_prefix_cache) + assert_cache_data(expected_test_stream_data, second_different_prefix_cache, streams=["stream1"]) + assert_cache_data(expected_test_stream_data, second_no_prefix_cache, streams=["stream1"]) + + +def test_merge_streams_in_cache(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """ + Test that we can extend a cache with new streams + """ + cache_name = str(ulid.ULID()) + source = ab.get_source("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache(cache_name) + + source.select_streams(["stream1"]) + source.read(cache) + + # Assert that the cache only contains stream1 + with pytest.raises(KeyError): + cache["stream2"] + + # Create a new cache with the same name + second_cache = ab.new_local_cache(cache_name) + source.select_streams(["stream2"]) + result = source.read(second_cache) + + third_cache = ab.new_local_cache(cache_name) + source.select_streams(["always-empty-stream"]) + result = source.read(third_cache) + + # Assert that the read result only contains stream2 + with pytest.raises(KeyError): + result["stream1"] + with pytest.raises(KeyError): + result["stream2"] + + assert_cache_data(expected_test_stream_data, third_cache) + + +def test_read_result_as_list(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = ab.new_local_cache() + + result: ReadResult = source.read(cache) + stream_1_list = list(result["stream1"]) + stream_2_list = list(result["stream2"]) + always_empty_stream_list = list(result["always-empty-stream"]) + assert stream_1_list == expected_test_stream_data["stream1"] + assert stream_2_list == expected_test_stream_data["stream2"] + assert always_empty_stream_list == expected_test_stream_data["always-empty-stream"] + + +def test_get_records_result_as_list(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + stream_1_list = list(source.get_records("stream1")) + stream_2_list = list(source.get_records("stream2")) + always_empty_stream_list = list(source.get_records("always-empty-stream")) + assert stream_1_list == expected_test_stream_data["stream1"] + assert stream_2_list == expected_test_stream_data["stream2"] + assert always_empty_stream_list == expected_test_stream_data["always-empty-stream"] + + + +def test_sync_with_merge_to_duckdb(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """Test that the merge strategy works as expected. + + In this test, we sync the same data twice. If the data is not duplicated, we assume + the merge was successful. + + # TODO: Add a check with a primary key to ensure that the merge strategy works as expected. + """ + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = ab.new_local_cache() + + # Read twice to test merge strategy + result: ReadResult = source.read(cache) + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + if len(cache[stream_name]) > 0: + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + else: + # stream is empty + assert len(expected_test_stream_data[stream_name]) == 0 + + +def test_cached_dataset( + expected_test_stream_data: dict[str, list[dict[str, str | int]]], +) -> None: + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + result: ReadResult = source.read(ab.new_local_cache()) + + stream_name = "stream1" + not_a_stream_name = "not_a_stream" + + # Check that the stream appears in mapping-like attributes + assert stream_name in result.cache._streams_with_data + assert stream_name in result + assert stream_name in result.cache + assert stream_name in result.cache.streams + assert stream_name in result.streams + + stream_get_a: CachedDataset = result[stream_name] + stream_get_b: CachedDataset = result.streams[stream_name] + stream_get_c: CachedDataset = result.cache[stream_name] + stream_get_d: CachedDataset = result.cache.streams[stream_name] + + # Check that each get method is syntactically equivalent + + assert isinstance(stream_get_a, CachedDataset) + assert isinstance(stream_get_b, CachedDataset) + assert isinstance(stream_get_c, CachedDataset) + assert isinstance(stream_get_d, CachedDataset) + + assert stream_get_a == stream_get_b + assert stream_get_b == stream_get_c + assert stream_get_c == stream_get_d + + # Check that we can iterate over the stream + + list_from_iter_a = list(stream_get_a) + list_from_iter_b = [row for row in stream_get_a] + + # Make sure that we get a key error if we try to access a stream that doesn't exist + with pytest.raises(KeyError): + result[not_a_stream_name] + with pytest.raises(KeyError): + result.streams[not_a_stream_name] + with pytest.raises(KeyError): + result.cache[not_a_stream_name] + with pytest.raises(KeyError): + result.cache.streams[not_a_stream_name] + + # Make sure we can use "result.streams.items()" + for stream_name, cached_dataset in result.streams.items(): + assert isinstance(cached_dataset, CachedDataset) + assert isinstance(stream_name, str) + + list_data = list(cached_dataset) + assert list_data == expected_test_stream_data[stream_name] + + # Make sure we can use "result.cache.streams.items()" + for stream_name, cached_dataset in result.cache.streams.items(): + assert isinstance(cached_dataset, CachedDataset) + assert isinstance(stream_name, str) + + list_data = list(cached_dataset) + assert list_data == expected_test_stream_data[stream_name] + + +def test_cached_dataset_filter(): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + result: ReadResult = source.read(ab.new_local_cache()) + + stream_name = "stream1" + + # Check the many ways to add a filter: + cached_dataset: CachedDataset = result[stream_name] + filtered_dataset_a: SQLDataset = cached_dataset.with_filter("column2 == 1") + filtered_dataset_b: SQLDataset = cached_dataset.with_filter(text("column2 == 1")) + filtered_dataset_c: SQLDataset = cached_dataset.with_filter(column("column2") == 1) + + assert isinstance(cached_dataset, CachedDataset) + all_records = list(cached_dataset) + assert len(all_records) == 2 + + for filtered_dataset, case in [ + (filtered_dataset_a, "a"), + (filtered_dataset_b, "b"), + (filtered_dataset_c, "c"), + ]: + assert isinstance(filtered_dataset, SQLDataset) + + # Check that we can iterate over each stream + + filtered_records: list[Mapping[str, Any]] = [row for row in filtered_dataset] + + # Check that the filter worked + assert len(filtered_records) == 1, f"Case '{case}' had incorrect number of records." + + # Assert the stream name still matches + assert filtered_dataset.stream_name == stream_name, \ + f"Case '{case}' had incorrect stream name." + + # Check that chaining filters works + chained_dataset = filtered_dataset.with_filter("column1 == 'value1'") + chained_records = [row for row in chained_dataset] + assert len(chained_records) == 1, \ + f"Case '{case}' had incorrect number of records after chaining filters." + + +def test_lazy_dataset_from_source( + expected_test_stream_data: dict[str, list[dict[str, str | int]]], +) -> None: + source = ab.get_source("source-test", config={"apiKey": "test"}) + + stream_name = "stream1" + not_a_stream_name = "not_a_stream" + + lazy_dataset_a = source.get_records(stream_name) + lazy_dataset_b = source.get_records(stream_name) + + assert isinstance(lazy_dataset_a, LazyDataset) + + # Check that we can iterate over the stream + + list_from_iter_a = list(lazy_dataset_a) + list_from_iter_b = [row for row in lazy_dataset_b] + + assert list_from_iter_a == list_from_iter_b + + # Make sure that we get a key error if we try to access a stream that doesn't exist + with pytest.raises(exc.AirbyteLibInputError): + source.get_records(not_a_stream_name) + + # Make sure we can iterate on all available streams + for stream_name in source.get_available_streams(): + assert isinstance(stream_name, str) + + lazy_dataset: LazyDataset = source.get_records(stream_name) + assert isinstance(lazy_dataset, LazyDataset) + + list_data = list(lazy_dataset) + assert list_data == expected_test_stream_data[stream_name] + + +@pytest.mark.parametrize( + "method_call", + [ + pytest.param(lambda source: source.check(), id="check"), + pytest.param(lambda source: list(source.get_records("stream1")), id="read_stream"), + pytest.param(lambda source: source.read(), id="read"), + ], +) +def test_check_fail_on_missing_config(method_call): + source = ab.get_source("source-test") + + with pytest.raises(exc.AirbyteConnectorConfigurationMissingError): + method_call(source) + +def test_sync_with_merge_to_postgres(new_pg_cache_config: PostgresCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """Test that the merge strategy works as expected. + + In this test, we sync the same data twice. If the data is not duplicated, we assume + the merge was successful. + + # TODO: Add a check with a primary key to ensure that the merge strategy works as expected. + """ + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = PostgresCache(config=new_pg_cache_config) + + # Read twice to test merge strategy + result: ReadResult = source.read(cache) + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + if len(cache[stream_name]) > 0: + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + else: + # stream is empty + assert len(expected_test_stream_data[stream_name]) == 0 + + +def test_airbyte_lib_version() -> None: + assert get_version() + assert isinstance(get_version(), str) + + # Ensure the version is a valid semantic version (x.y.z or x.y.z.alpha0) + assert 3 <= len(get_version().split(".")) <= 4 + + +@patch.dict('os.environ', {'DO_NOT_TRACK': ''}) +@patch('airbyte_lib.telemetry.requests') +@patch('airbyte_lib.telemetry.datetime') +@pytest.mark.parametrize( + "raises, api_key, expected_state, expected_number_of_records, request_call_fails, extra_env, expected_flags, cache_type, number_of_records_read", + [ + pytest.param(pytest.raises(Exception), "test_fail_during_sync", "failed", 1, False, {"CI": ""}, {"CI": False}, "duckdb", None, id="fail_during_sync"), + pytest.param(does_not_raise(), "test", "succeeded", 3, False, {"CI": ""}, {"CI": False}, "duckdb", None, id="succeed_during_sync"), + pytest.param(does_not_raise(), "test", "succeeded", 3, True, {"CI": ""}, {"CI": False}, "duckdb", None,id="fail_request_without_propagating"), + pytest.param(does_not_raise(), "test", "succeeded", 3, False, {"CI": ""}, {"CI": False}, "duckdb", None,id="falsy_ci_flag"), + pytest.param(does_not_raise(), "test", "succeeded", 3, False, {"CI": "true"}, {"CI": True}, "duckdb", None,id="truthy_ci_flag"), + pytest.param(pytest.raises(Exception), "test_fail_during_sync", "failed", 1, False, {"CI": ""}, {"CI": False}, "streaming", 3, id="streaming_fail_during_sync"), + pytest.param(does_not_raise(), "test", "succeeded", 2, False, {"CI": ""}, {"CI": False}, "streaming", 2, id="streaming_succeed"), + pytest.param(does_not_raise(), "test", "succeeded", 1, False, {"CI": ""}, {"CI": False}, "streaming", 1, id="streaming_partial_read"), + ], +) +def test_tracking( + mock_datetime: Mock, + mock_requests: Mock, + raises, api_key: str, + expected_state: str, + expected_number_of_records: int, + request_call_fails: bool, + extra_env: dict[str, str], + expected_flags: dict[str, bool], + cache_type: str, + number_of_records_read: int +): + """ + Test that the telemetry is sent when the sync is successful. + This is done by mocking the requests.post method and checking that it is called with the right arguments. + """ + now_date = Mock() + mock_datetime.datetime = Mock() + mock_datetime.datetime.utcnow.return_value = now_date + now_date.isoformat.return_value = "2021-01-01T00:00:00.000000" + + mock_post = Mock() + mock_requests.post = mock_post + + source = ab.get_source("source-test", config={"apiKey": api_key}) + source.select_all_streams() + + cache = ab.new_local_cache() + + if request_call_fails: + mock_post.side_effect = Exception("test exception") + + with patch.dict('os.environ', extra_env): + with raises: + if cache_type == "streaming": + list(itertools.islice(source.get_records("stream1"), number_of_records_read)) + else: + source.read(cache) + + mock_post.assert_has_calls([ + call("https://api.segment.io/v1/track", + auth=("cukeSffc0G6gFQehKDhhzSurDzVSZ2OP", ""), + json={ + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": {'name': 'source-test', 'version': '0.0.1', 'type': 'venv'}, + "state": "started", + "cache": {"type": cache_type}, + "ip": "0.0.0.0", + "flags": expected_flags + }, + "timestamp": "2021-01-01T00:00:00.000000", + } + ), + call( + "https://api.segment.io/v1/track", + auth=("cukeSffc0G6gFQehKDhhzSurDzVSZ2OP", ""), + json={ + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": {'name': 'source-test', 'version': '0.0.1', 'type': 'venv'}, + "state": expected_state, + "number_of_records": expected_number_of_records, + "cache": {"type": cache_type}, + "ip": "0.0.0.0", + "flags": expected_flags + }, + "timestamp": "2021-01-01T00:00:00.000000", + } + ) + ]) + + +def test_sync_to_postgres(new_pg_cache_config: PostgresCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = PostgresCache(config=new_pg_cache_config) + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + if len(cache[stream_name]) > 0: + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + else: + # stream is empty + assert len(expected_test_stream_data[stream_name]) == 0 + +@pytest.mark.slow +@pytest.mark.requires_creds +def test_sync_to_snowflake(snowflake_config: SnowflakeCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_source("source-test", config={"apiKey": "test"}) + source.select_all_streams() + + cache = SnowflakeSQLCache(config=snowflake_config) + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + if len(cache[stream_name]) > 0: + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + else: + # stream is empty + assert len(expected_test_stream_data[stream_name]) == 0 + + +def test_sync_limited_streams(expected_test_stream_data): + source = ab.get_source("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + source.select_streams(["stream2"]) + + result = source.read(cache) + + assert result.processed_records == 1 + pd.testing.assert_frame_equal( + result["stream2"].to_pandas(), + pd.DataFrame(expected_test_stream_data["stream2"]), + check_dtype=False, + ) + + +def test_read_stream(): + source = ab.get_source("source-test", config={"apiKey": "test"}) + + assert list(source.get_records("stream1")) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] + + +def test_read_stream_nonexisting(): + source = ab.get_source("source-test", config={"apiKey": "test"}) + + with pytest.raises(Exception): + list(source.get_records("non-existing")) + +def test_failing_path_connector(): + with pytest.raises(Exception): + ab.get_source("source-test", config={"apiKey": "test"}, use_local_install=True) + +def test_succeeding_path_connector(): + new_path = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" + + # Patch the PATH env var to include the test venv bin folder + with patch.dict(os.environ, {"PATH": new_path}): + source = ab.get_source( + "source-test", + config={"apiKey": "test"}, + local_executable="source-test", + ) + source.check() + +def test_install_uninstall(): + with tempfile.TemporaryDirectory() as temp_dir: + source = ab.get_source( + "source-test", + pip_url="./tests/integration_tests/fixtures/source-test", + config={"apiKey": "test"}, + install_if_missing=False, + ) + + # Override the install root to avoid conflicts with the test fixture + install_root = Path(temp_dir) + source.executor.install_root = install_root + + # assert that the venv is gone + assert not os.path.exists(install_root / ".venv-source-test") + + # use which to check if the executable is available + assert shutil.which("source-test") is None + + # assert that the connector is not available + with pytest.raises(Exception): + source.check() + + source.install() + + assert os.path.exists(install_root / ".venv-source-test") + assert os.path.exists(install_root / ".venv-source-test/bin/source-test") + + source.check() + + source.uninstall() + + assert not os.path.exists(install_root / ".venv-source-test") + assert not os.path.exists(install_root / ".venv-source-test/bin/source-test") diff --git a/airbyte-lib/tests/integration_tests/test_validation.py b/airbyte-lib/tests/integration_tests/test_validation.py new file mode 100644 index 000000000000..140a7d52023e --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_validation.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +import shutil + +import pytest +from airbyte_lib.validate import validate + + +@pytest.fixture(scope="module", autouse=True) +def autouse_source_test_installation(source_test_installation): + return + + +@pytest.fixture(scope="function", autouse=True) +def autouse_source_test_registry(source_test_registry): + return + + +def test_validate_success(): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=False) + +def test_validate_check_failure(): + with pytest.raises(Exception): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=False) + +def test_validate_success_install_only(): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=True) + +def test_validate_config_failure(): + with pytest.raises(Exception): + validate("./tests/integration_tests/fixtures/source-broken", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=True) diff --git a/airbyte-lib/tests/lint_tests/__init__.py b/airbyte-lib/tests/lint_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/lint_tests/test_mypy.py b/airbyte-lib/tests/lint_tests/test_mypy.py new file mode 100644 index 000000000000..df0997828079 --- /dev/null +++ b/airbyte-lib/tests/lint_tests/test_mypy.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import subprocess + +import pytest + + +def test_mypy_typing(): + # Run the check command + check_result = subprocess.run( + ["poetry", "run", "mypy", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert check_result.returncode == 0, ( + "MyPy checks failed:\n" + + f"{check_result.stdout.decode()}\n{check_result.stderr.decode()}\n\n" + + "Run `poetry run mypy .` to see all failures." + ) diff --git a/airbyte-lib/tests/lint_tests/test_ruff.py b/airbyte-lib/tests/lint_tests/test_ruff.py new file mode 100644 index 000000000000..57262a8f608c --- /dev/null +++ b/airbyte-lib/tests/lint_tests/test_ruff.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import subprocess + +import pytest + + +def test_ruff_linting(): + # Run the check command + check_result = subprocess.run( + ["poetry", "run", "ruff", "check", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert check_result.returncode == 0, ( + "Ruff checks failed:\n\n" + + f"{check_result.stdout.decode()}\n{check_result.stderr.decode()}\n\n" + + "Run `poetry run ruff check .` to view all issues." + ) + + +def test_ruff_linting_fixable(): + # Run the check command + fix_diff_result = subprocess.run( + ["poetry", "run", "ruff", "check", "--fix", "--diff", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert fix_diff_result.returncode == 0, ( + "Ruff checks revealed fixable issues:\n\n" + + f"{fix_diff_result.stdout.decode()}\n{fix_diff_result.stderr.decode()}\n\n" + + "Run `poetry run ruff check --fix .` to attempt automatic fixes." + ) + + +def test_ruff_format(): + # Define the command to run Ruff + command = ["poetry", "run", "ruff", "format", "--check", "--diff"] + + # Run the command + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Assert that the Ruff command exited without errors (exit code 0) + assert result.returncode == 0, ( + f"Ruff checks failed:\n\n{result.stdout.decode()}\n{result.stderr.decode()}\n\n" + + "Run `poetry run ruff format .` to attempt automatic fixes." + ) diff --git a/airbyte-lib/tests/unit_tests/__init__.py b/airbyte-lib/tests/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/unit_tests/test_caches.py b/airbyte-lib/tests/unit_tests/test_caches.py new file mode 100644 index 000000000000..5bc2ba4186cd --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_caches.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pathlib import Path + +import pytest + +from airbyte_lib._file_writers import ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.caches.duckdb import DuckDBCacheBase, DuckDBCacheConfig + + +def test_duck_db_cache_config_initialization(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.db_path == Path('test_path') + assert config.schema_name == 'test_schema' + +def test_duck_db_cache_config_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.schema_name == 'main' + +def test_get_sql_alchemy_url(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_get_sql_alchemy_url_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_inheritance(): + assert issubclass(DuckDBCacheConfig, SQLCacheConfigBase) + assert issubclass(DuckDBCacheConfig, ParquetWriterConfig) + +def test_duck_db_cache_config_get_sql_alchemy_url(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_get_database_name(): + config = DuckDBCacheConfig(db_path='test_path/test_db.duckdb', schema_name='test_schema') + assert config.get_database_name() == 'test_db' + +def test_duck_db_cache_base_inheritance(): + assert issubclass(DuckDBCacheBase, SQLCacheBase) + +def test_duck_db_cache_config_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.schema_name == 'main' + +def test_duck_db_cache_config_get_sql_alchemy_url_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_get_database_name_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path/test_db.duckdb') + assert config.get_database_name() == 'test_db' + +def test_duck_db_cache_config_inheritance_from_sql_cache_config_base(): + assert issubclass(DuckDBCacheConfig, SQLCacheConfigBase) + +def test_duck_db_cache_config_inheritance_from_parquet_writer_config(): + assert issubclass(DuckDBCacheConfig, ParquetWriterConfig) diff --git a/airbyte-lib/tests/unit_tests/test_exceptions.py b/airbyte-lib/tests/unit_tests/test_exceptions.py new file mode 100644 index 000000000000..ef5a391e47df --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_exceptions.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import inspect +import pytest +import inspect +import airbyte_lib.exceptions as exceptions_module + +def test_exceptions(): + exception_classes = [ + (name, obj) + for name, obj in inspect.getmembers(exceptions_module) + if inspect.isclass(obj) and name.endswith("Error") + ] + assert "AirbyteError" in [name for name, _ in exception_classes] + assert "NotAnError" not in [name for name, _ in exception_classes] + for name, obj in exception_classes: + instance = obj() + message = instance.get_message() + assert isinstance(message, str), "No message for class: " + name + assert message.count("\n") == 0 + assert message != "" + assert message.strip() == message + assert name.startswith("Airbyte") + assert name.endswith("Error") + + +if __name__ == "__main__": + pytest.main() diff --git a/airbyte-lib/tests/unit_tests/test_pip_helpers.py b/airbyte-lib/tests/unit_tests/test_pip_helpers.py new file mode 100644 index 000000000000..e99ba3e624ff --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_pip_helpers.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from airbyte_lib._util import github_pip_url, connector_pip_url + +@pytest.mark.parametrize('owner, repo, branch_or_ref, package_name, subdirectory, expected', [ + ('airbytehq', 'airbyte', None, None, None, 'git+https://github.com/airbytehq/airbyte.git'), + ('airbytehq', 'airbyte', 'master', None, None, 'git+https://github.com/airbytehq/airbyte.git@master'), + ('airbytehq', 'airbyte', 'my-branch', None, None, 'git+https://github.com/airbytehq/airbyte.git@my-branch'), + ('airbytehq', 'airbyte', 'my-branch', 'airbyte-lib', None, 'git+https://github.com/airbytehq/airbyte.git@my-branch#egg=airbyte-lib'), + ('airbytehq', 'airbyte', 'my-branch', 'airbyte-lib', 'airbyte-lib', 'git+https://github.com/airbytehq/airbyte.git@my-branch#egg=airbyte-lib&subdirectory=airbyte-lib'), +]) +def test_github_pip_url(owner, repo, branch_or_ref, package_name, subdirectory, expected): + result = github_pip_url(owner, repo, branch_or_ref=branch_or_ref, package_name=package_name, subdirectory=subdirectory) + assert result == expected + +@pytest.mark.parametrize('connector_name, branch, owner, expected', [ + ('source-coin-api', 'my-branch', None, 'git+https://github.com/airbytehq/airbyte.git@my-branch#egg=source-coin-api&subdirectory=airbyte-integrations/connectors/source-coin-api'), + ('source-coin-api', 'my-branch', 'my-fork', 'git+https://github.com/my-fork/airbyte.git@my-branch#egg=source-coin-api&subdirectory=airbyte-integrations/connectors/source-coin-api'), +]) +def test_connector_pip_url(connector_name, branch, owner, expected): + result = connector_pip_url( + connector_name, + branch, + owner=owner) + assert result == expected diff --git a/airbyte-lib/tests/unit_tests/test_progress.py b/airbyte-lib/tests/unit_tests/test_progress.py new file mode 100644 index 000000000000..377df860bb57 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_progress.py @@ -0,0 +1,174 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import datetime +from textwrap import dedent +import time +import pytest +from freezegun import freeze_time +from airbyte_lib.progress import ReadProgress, _get_elapsed_time_str, _to_time_str +from dateutil.tz import tzlocal + +# Calculate the offset from UTC in hours +tz_offset_hrs = int(datetime.datetime.now(tzlocal()).utcoffset().total_seconds() / 3600) + + +@freeze_time("2022-01-01") +def test_read_progress_initialization(): + progress = ReadProgress() + assert progress.num_streams_expected == 0 + assert progress.read_start_time == 1640995200.0 # Unix timestamp for 2022-01-01 + assert progress.total_records_read == 0 + assert progress.total_records_written == 0 + assert progress.total_batches_written == 0 + assert progress.written_stream_names == set() + assert progress.finalize_start_time is None + assert progress.finalize_end_time is None + assert progress.total_records_finalized == 0 + assert progress.total_batches_finalized == 0 + assert progress.finalized_stream_names == set() + assert progress.last_update_time is None + + +@freeze_time("2022-01-01") +def test_read_progress_reset(): + progress = ReadProgress() + progress.reset(5) + assert progress.num_streams_expected == 5 + assert progress.read_start_time == 1640995200.0 + assert progress.total_records_read == 0 + assert progress.total_records_written == 0 + assert progress.total_batches_written == 0 + assert progress.written_stream_names == set() + assert progress.finalize_start_time is None + assert progress.finalize_end_time is None + assert progress.total_records_finalized == 0 + assert progress.total_batches_finalized == 0 + assert progress.finalized_stream_names == set() + +@freeze_time("2022-01-01") +def test_read_progress_log_records_read(): + progress = ReadProgress() + progress.log_records_read(100) + assert progress.total_records_read == 100 + +@freeze_time("2022-01-01") +def test_read_progress_log_batch_written(): + progress = ReadProgress() + progress.log_batch_written("stream1", 50) + assert progress.total_records_written == 50 + assert progress.total_batches_written == 1 + assert progress.written_stream_names == {"stream1"} + +@freeze_time("2022-01-01") +def test_read_progress_log_batches_finalizing(): + progress = ReadProgress() + progress.log_batches_finalizing("stream1", 1) + assert progress.finalize_start_time == 1640995200.0 + +@freeze_time("2022-01-01") +def test_read_progress_log_batches_finalized(): + progress = ReadProgress() + progress.log_batches_finalized("stream1", 1) + assert progress.total_batches_finalized == 1 + +@freeze_time("2022-01-01") +def test_read_progress_log_stream_finalized(): + progress = ReadProgress() + progress.log_stream_finalized("stream1") + assert progress.finalized_stream_names == {"stream1"} + + +def test_get_elapsed_time_str(): + assert _get_elapsed_time_str(30) == "30 seconds" + assert _get_elapsed_time_str(90) == "1min 30s" + assert _get_elapsed_time_str(600) == "10min" + assert _get_elapsed_time_str(3600) == "1hr 0min" + + +@freeze_time("2022-01-01 0:00:00") +def test_get_time_str(): + assert _to_time_str(time.time()) == "00:00:00" + + +def _assert_lines(expected_lines, actual_lines: list[str] | str): + if isinstance(actual_lines, list): + actual_lines = "\n".join(actual_lines) + for line in expected_lines: + assert line in actual_lines, f"Missing line: {line}" + +def test_get_status_message_after_finalizing_records(): + + # Test that we can render the initial status message before starting to read + with freeze_time("2022-01-01 00:00:00"): + progress = ReadProgress() + expected_lines = [ + "Started reading at 00:00:00.", + "Read **0** records over **0 seconds** (0.0 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after reading some records + with freeze_time("2022-01-01 00:01:00"): + progress.log_records_read(100) + expected_lines = [ + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Advance the day and reset the progress + with freeze_time("2022-01-02 00:00:00"): + progress = ReadProgress() + progress.reset(1) + expected_lines = [ + "Started reading at 00:00:00.", + "Read **0** records over **0 seconds** (0.0 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after writing some records and starting to finalize + with freeze_time("2022-01-02 00:01:00"): + progress.log_records_read(100) + progress.log_batch_written("stream1", 50) + progress.log_batches_finalizing("stream1", 1) + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after finalizing some records + with freeze_time("2022-01-02 00:02:00"): + progress.log_batches_finalized("stream1", 1) + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + "Finalized **1** batches over 60 seconds.", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after finalizing all records + with freeze_time("2022-01-02 00:02:00"): + progress.log_stream_finalized("stream1") + message = progress._get_status_message() + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + "Finalized **1** batches over 60 seconds.", + "Completed 1 out of 1 streams:", + "- stream1", + "Total time elapsed: 2min 0s", + ] + _assert_lines(expected_lines, message) diff --git a/airbyte-lib/tests/unit_tests/test_type_translation.py b/airbyte-lib/tests/unit_tests/test_type_translation.py new file mode 100644 index 000000000000..a2c255c5b0d7 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_type_translation.py @@ -0,0 +1,82 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from sqlalchemy import types +from airbyte_lib.types import SQLTypeConverter, _get_airbyte_type + + +@pytest.mark.parametrize( + "json_schema_property_def, expected_sql_type", + [ + ({"type": "string"}, types.VARCHAR), + ({"type": ["boolean", "null"]}, types.BOOLEAN), + ({"type": ["null", "boolean"]}, types.BOOLEAN), + ({"type": "string"}, types.VARCHAR), + ({"type": ["null", "string"]}, types.VARCHAR), + ({"type": "boolean"}, types.BOOLEAN), + ({"type": "string", "format": "date"}, types.DATE), + ({"type": ["null", "string"]}, types.VARCHAR), + ({"type": ["null", "boolean"]}, types.BOOLEAN), + ({"type": ["null", "number"]}, types.DECIMAL), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, types.TIMESTAMP), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, types.TIMESTAMP), + ({"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}, types.TIME), + ({"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, types.TIME), + ({"type": "integer"}, types.BIGINT), + ({"type": "number", "airbyte_type": "integer"}, types.BIGINT), + ({"type": "number"}, types.DECIMAL), + ({"type": "array", "items": {"type": "object"}}, types.JSON), + ({"type": "object", "properties": {}}, types.JSON), + ], +) +def test_to_sql_type(json_schema_property_def, expected_sql_type): + converter = SQLTypeConverter() + sql_type = converter.to_sql_type(json_schema_property_def) + assert isinstance(sql_type, expected_sql_type) + + +@pytest.mark.parametrize( + "json_schema_property_def, expected_airbyte_type", + [ + ({"type": "string"}, "string"), + ({"type": ["boolean", "null"]}, "boolean"), + ({"type": ["null", "boolean"]}, "boolean"), + ({"type": "string"}, "string"), + ({"type": ["null", "string"]}, "string"), + ({"type": "boolean"}, "boolean"), + ({"type": "string", "format": "date"}, "date"), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, "timestamp_without_timezone"), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "timestamp_with_timezone"), + ({"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}, "time_without_timezone"), + ({"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, "time_with_timezone"), + ({"type": "integer"}, "integer"), + ({"type": "number", "airbyte_type": "integer"}, "integer"), + ({"type": "number"}, "number"), + ({"type": "array"}, "array"), + ({"type": "object"}, "object"), + ], +) +def test_to_airbyte_type(json_schema_property_def, expected_airbyte_type): + airbyte_type, _ = _get_airbyte_type(json_schema_property_def) + assert airbyte_type == expected_airbyte_type + + +@pytest.mark.parametrize( + "json_schema_property_def, expected_airbyte_type, expected_airbyte_subtype", + [ + ({"type": "string"}, "string", None), + ({"type": "number"}, "number", None), + ({"type": "array"}, "array", None), + ({"type": "object"}, "object", None), + ({"type": "array", "items": {"type": ["null", "string"]}}, "array", "string"), + ({"type": "array", "items": {"type": ["boolean"]}}, "array", "boolean"), + ], +) +def test_to_airbyte_subtype( + json_schema_property_def, + expected_airbyte_type, + expected_airbyte_subtype, +): + airbyte_type, subtype = _get_airbyte_type(json_schema_property_def) + assert airbyte_type == expected_airbyte_type + assert subtype == expected_airbyte_subtype diff --git a/airbyte-lib/tests/unit_tests/test_writers.py b/airbyte-lib/tests/unit_tests/test_writers.py new file mode 100644 index 000000000000..2578ae10b483 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_writers.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pathlib import Path +import pytest +from airbyte_lib._file_writers.base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase +from airbyte_lib._file_writers.parquet import ParquetWriter, ParquetWriterConfig +from numpy import source + + +def test_parquet_writer_config_initialization(): + config = ParquetWriterConfig(cache_dir='test_path') + assert config.cache_dir == Path('test_path') + +def test_parquet_writer_config_inheritance(): + assert issubclass(ParquetWriterConfig, FileWriterConfigBase) + +def test_parquet_writer_initialization(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert writer.config == config + +def test_parquet_writer_inheritance(): + assert issubclass(ParquetWriter, FileWriterBase) + +def test_parquet_writer_has_config(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert hasattr(writer, 'config') + +def test_parquet_writer_has_source_catalog(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + +def test_parquet_writer_source_catalog_is_none(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) diff --git a/build.gradle b/build.gradle index 80da7caf07ea..3d13e5f7b2d5 100644 --- a/build.gradle +++ b/build.gradle @@ -1,191 +1,49 @@ +import com.github.spotbugs.snom.Confidence +import com.github.spotbugs.snom.Effort import com.github.spotbugs.snom.SpotBugsTask - plugins { id 'base' - id 'com.github.node-gradle.node' version '3.5.1' - id 'com.github.spotbugs' version '5.0.13' - id 'version-catalog' - id 'ru.vyarus.use-python' -} - - -Properties env = new Properties() -rootProject.file('gradle.properties').withInputStream { env.load(it) } - -if (!env.containsKey('VERSION')) { - throw new Exception('Version not specified in .env file...') -} - -// `version` is used as the application build version for artifacts like jars -// `image_tag` is used as the docker tag applied to built images. -// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable. -// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'. -ext { - version = System.getenv("VERSION") ?: env.VERSION - image_tag = System.getenv("VERSION") ?: 'dev' - skipSlowTests = (System.getProperty('skipSlowTests', 'false') != 'false') -} -// Pyenv support. -try { - def pyenvRoot = "pyenv root".execute() - if (pyenvRoot.waitFor() == 0) { - ext.pyenvRoot = pyenvRoot.text.trim() - } -} catch (IOException _) { - // Swallow exception if pyenv is not installed. -} - -def isConnectorProject = { Project project -> - if (project.parent == null || project.parent.name != 'connectors') { - return false - } - return project.name.startsWith("source-") || project.name.startsWith("destination-") + id 'com.github.spotbugs' version '6.0.7' } allprojects { apply plugin: 'base' + apply plugin: 'java' + apply plugin: 'java-test-fixtures' + apply plugin: 'com.github.spotbugs' - // by default gradle uses directory as the project name. That works very well in a single project environment but + // By default gradle uses directory as the project name. That works very well in a single project environment but // projects clobber each other in an environments with subprojects when projects are in directories named identically. def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.') group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}" project.base.archivesName = "${project.group}-${project.name}" - version = rootProject.ext.version -} - -// python is required by the root project to run CAT tests for connectors -python { - envPath = '.venv' - minPythonVersion = '3.10' // should be 3.10 for local development - - // Amazon Linux support. - // The airbyte-ci tool runs gradle tasks in AL2023-based containers. - // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. - // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. - try { - if ("python3.11 --version".execute().waitFor() == 0) { - // python3.11 definitely exists at this point, use it instead of 'python3'. - pythonBinary "python3.11" - } - } catch (IOException _) { - // Swallow exception if python3.11 is not installed. - } - // Pyenv support. - try { - def pyenvRoot = "pyenv root".execute() - def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() - // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. - pythonBinary "python" - if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { - pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" - } - } catch (IOException _) { - // Swallow exception if pyenv is not installed. - } - - scope = 'VIRTUALENV' - installVirtualenv = true - // poetry is required for installing and running airbyte-ci - pip 'poetry:1.5.1' -} - -def cleanPythonVenv = rootProject.tasks.register('cleanPythonVenv', Exec) { - commandLine 'rm' - args '-rf', "${rootProject.projectDir.absolutePath}/.venv" -} -rootProject.tasks.named('clean').configure { - dependsOn cleanPythonVenv -} - - -def getCDKTargetVersion() { - def props = new Properties() - file("airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties").withInputStream { props.load(it) } - return props.getProperty('version') -} -static def getLatestFileModifiedTimeFromFiles(files) { - if (files.isEmpty()) { - return null - } - return files.findAll { it.isFile() } - .collect { it.lastModified() } - .max() -} -def checkCDKJarExists(requiredSnapshotVersion) { - if (requiredSnapshotVersion == null) { - // Connector does not require CDK snapshot. - return - } - final boolean checkFileChanges = true - final cdkTargetVersion = getCDKTargetVersion() - if (requiredSnapshotVersion != cdkTargetVersion) { - if (!cdkTargetVersion.contains("-SNAPSHOT")) { - throw new GradleException( - "CDK JAR version is not publishing snapshot but connector requires version ${requiredSnapshotVersion}.\n" + - "Please check that the version in the CDK properties file matches the connector build.gradle." - ) - } - throw new GradleException( - "CDK JAR version ${cdkTargetVersion} does not match connector's required version ${requiredSnapshotVersion}.\n" + - "Please check that the version in the CDK properties file matches the connector build.gradle." - ) - } - - def cdkJar = file("${System.properties['user.home']}/.m2/repository/io/airbyte/airbyte-cdk/${cdkTargetVersion}/airbyte-cdk-${cdkTargetVersion}.jar") - if (!cdkJar.exists()) { - println("WARNING: CDK JAR does not exist at ${cdkJar.path}.\nPlease run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.") - } - if (checkFileChanges) { - def latestJavaFileTimestamp = getLatestFileModifiedTimeFromFiles(file("${rootDir}/airbyte-cdk/java/airbyte-cdk/src").listFiles().findAll { it.isFile() }) - if (cdkJar.lastModified() < latestJavaFileTimestamp) { - throw new GradleException("CDK JAR is out of date. Please run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.") - } + // Produce reproducible archives + // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives) + tasks.withType(AbstractArchiveTask).configureEach { + preserveFileTimestamps = false + reproducibleFileOrder = true } -} -static def getCDKSnapshotRequirement(dependenciesList) { - def cdkSnapshotRequirement = dependenciesList.find { - it.requested instanceof ModuleComponentSelector && - it.requested.group == 'io.airbyte' && - it.requested.module == 'airbyte-cdk' && - it.requested.version.endsWith('-SNAPSHOT') - } - if (cdkSnapshotRequirement == null) { - return null - } else { - return cdkSnapshotRequirement.requested.version - } -} - -// Common configurations for 'assemble'. -allprojects { + // Common configurations for 'assemble'. tasks.withType(Tar).configureEach { duplicatesStrategy DuplicatesStrategy.INCLUDE } - tasks.withType(Zip).configureEach { duplicatesStrategy DuplicatesStrategy.INCLUDE // Disabling distZip causes the build to break for some reason, so: instead of disabling it, make it fast. entryCompression ZipEntryCompression.STORED } -} -// Java projects common configurations. -subprojects { subproj -> - - if (!subproj.file('src/main/java').directory) { - return - } + // Convenience task to list all dependencies per project + tasks.register('listAllDependencies', DependencyReportTask) {} - apply plugin: 'java' - apply plugin: 'jacoco' - apply plugin: 'com.github.spotbugs' + // Common java configurations java { - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 compileJava { options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing"] } @@ -194,69 +52,19 @@ subprojects { subproj -> //deprecation and removal are removed from error since we should still test those constructs. options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked,-deprecation,-removal"] } - } - - if (isConnectorProject(subproj)) { - // This is a Java connector project. - - // Evaluate CDK project before evaluating the connector. - evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk') - - if (!gradle.startParameter.taskNames.any { it.contains(':airbyte-cdk:') } && - gradle.startParameter.taskNames.any { it.contains(':source-') || it.contains(':destination-') }) { - // We are building a connector. Warn if the CDK JAR is missing or out of date. - final String cdkRelativePath = 'airbyte-cdk/java/airbyte-cdk' - afterEvaluate { - def cdkVersionNeeded = getCDKSnapshotRequirement(configurations.compileClasspath.incoming.resolutionResult.allDependencies) - checkCDKJarExists(cdkVersionNeeded) - } - } - } - - jacoco { - toolVersion = "0.8.8" - } - - jacocoTestReport { - reports { - html.required = true - xml.required = true - csv.required = false - } - } - def jacocoTestReportTask = tasks.named('jacocoTestReport') - jacocoTestReportTask.configure { - dependsOn tasks.named('test') - } - - jacocoTestCoverageVerification { - violationRules { - failOnViolation = false - rule { - element = 'CLASS' - excludes = ['**/*Test*', '**/generated*'] - limit { - counter = 'BRANCH' - minimum = 0.8 - } - limit { - counter = 'INSTRUCTION' - minimum = 0.8 - } - } + compileTestFixturesJava { + //rawtypes and unchecked are necessary for mockito + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked"] } } spotbugs { ignoreFailures = false - effort = 'max' + effort = Effort.valueOf(System.getProperty('skipSlowTests', 'false') == 'false' ? 'MAX' : 'MIN') excludeFilter.set rootProject.file('spotbugs-exclude-filter-file.xml') - reportLevel = 'high' + reportLevel = Confidence.valueOf('HIGH') showProgress = false - toolVersion = '4.7.3' - if (rootProject.ext.skipSlowTests && isConnectorProject(subproj)) { - effort = 'min' - } + toolVersion = '4.8.3' } test { @@ -267,6 +75,17 @@ subprojects { subproj -> // Swallow the logs when running in airbyte-ci, rely on test reports instead. showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI") } + reports { + junitXml { + outputPerTestCase = true + } + } + + // This is required by mockito, see https://github.com/mockito/mockito/issues/3037. + jvmArgs "-XX:+EnableDynamicAgentLoading" + // This is also required, to prevent stderr spam starting with + // "OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader cl..." + jvmArgs "-Xshare:off" // Set the timezone to UTC instead of picking up the host machine's timezone, // which on a developer's laptop is more likely to be PST. @@ -285,8 +104,9 @@ subprojects { subproj -> systemProperty 'junit.jupiter.execution.parallel.config.fixed.parallelism', 1 // Order test classes by annotation. systemProperty 'junit.jupiter.testclass.order.default', 'org.junit.jupiter.api.ClassOrderer$OrderAnnotation' + systemProperty 'junit.jupiter.extensions.autodetection.enabled', 'true' - if (!subproj.hasProperty('testExecutionConcurrency')) { + if (!project.hasProperty('testExecutionConcurrency')) { // By default, let gradle spawn as many independent workers as it wants. maxParallelForks = Runtime.runtime.availableProcessors() maxHeapSize = '3G' @@ -296,7 +116,7 @@ subprojects { subproj -> maxParallelForks = 1 maxHeapSize = '8G' // Manage test execution concurrency in JUnit. - String concurrency = subproj.property('testExecutionConcurrency').toString() + String concurrency = project.property('testExecutionConcurrency').toString() if (concurrency.isInteger() && (concurrency as int) > 0) { // Define a fixed number of threads when the property is set to a positive integer. systemProperty 'junit.jupiter.execution.parallel.config.fixed.parallelism', concurrency @@ -305,59 +125,41 @@ subprojects { subproj -> systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic' } } - - // Exclude all connector unit tests upon request. - if (rootProject.ext.skipSlowTests) { - exclude '**/io/airbyte/integrations/source/**' - exclude '**/io/airbyte/integrations/destination/**' - } - - jacoco { - enabled = !rootProject.ext.skipSlowTests - excludes = ['**/*Test*', '**/generated*'] - } - finalizedBy jacocoTestReportTask } - // TODO: These should be added to the CDK or to the individual projects that need them: - dependencies { - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) - - // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here. - implementation libs.bundles.jackson - implementation libs.guava - implementation libs.commons.io - implementation libs.bundles.apache - implementation libs.slf4j.api - - // SLF4J as a facade over Log4j2 required dependencies - implementation libs.bundles.log4j - implementation libs.appender.log4j2 - - // Bridges from other logging implementations to SLF4J - implementation libs.bundles.slf4j - - // Lombok dependencies - compileOnly libs.lombok - annotationProcessor libs.lombok - - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer - - // adds owasp plugin - spotbugsPlugins libs.findsecbugs.plugin - implementation libs.spotbugs.annotations - - // Airbyte dependencies. - implementation libs.airbyte.protocol + // Lombok dependencies. + def lombok = "org.projectlombok:lombok:1.18.30" + compileOnly lombok + annotationProcessor lombok + testCompileOnly lombok + testAnnotationProcessor lombok + testFixturesCompileOnly lombok + testFixturesAnnotationProcessor lombok + + // JUnit dependencies. + def vAssertJ = "3.21.0" + def vJUnit = "5.9.1" + def vJUnitJupiter = "5.10.0" + def vJUnitPioneer = "1.7.1" + + testFixturesImplementation "org.junit.jupiter:junit-jupiter-api:${vJUnit}" + testFixturesImplementation "org.junit.jupiter:junit-jupiter-params:${vJUnit}" + testFixturesImplementation "org.mockito:mockito-junit-jupiter:${vJUnitJupiter}" + testFixturesImplementation "org.assertj:assertj-core:${vAssertJ}" + testFixturesImplementation "org.junit-pioneer:junit-pioneer:${vJUnitPioneer}" + + testImplementation "org.junit.jupiter:junit-jupiter-api:${vJUnit}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${vJUnit}" + testImplementation "org.mockito:mockito-junit-jupiter:${vJUnitJupiter}" + testImplementation "org.assertj:assertj-core:${vAssertJ}" + testImplementation "org.junit-pioneer:junit-pioneer:${vJUnitPioneer}" + + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${vJUnit}" + + // Spotbugs dependencies. + def vSpotbugs = "4.8.3" + implementation "com.github.spotbugs:spotbugs-annotations:${vSpotbugs}" } tasks.withType(SpotBugsTask).configureEach { @@ -370,107 +172,3 @@ subprojects { subproj -> javadoc.options.addStringOption('Xdoclint:none', '-quiet') } - -// integration and performance test tasks per project -allprojects { - tasks.register('integrationTest') { - dependsOn tasks.matching { - [ - 'integrationTestJava', - 'integrationTestPython', - ].contains(it.name) - } - } - - tasks.register('performanceTest') { - dependsOn tasks.matching { - [ - 'performanceTestJava', - ].contains(it.name) - } - } -} - - -// convenience task to list all dependencies per project -subprojects { - tasks.register('listAllDependencies', DependencyReportTask) {} -} - -// airbyte-ci tasks for local development -def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) { - workingDir rootProject.file('airbyte-ci/connectors/pipelines') - commandLine rootProject.file('.venv/bin/python').absolutePath - args "-m", "poetry", "install", "--no-cache" -} -poetryInstallAirbyteCI.configure { - dependsOn tasks.named('pipInstall') -} -def poetryCleanVirtualenv = tasks.register('cleanVirtualenv', Exec) { - workingDir rootProject.file('airbyte-ci/connectors/pipelines') - commandLine rootProject.file('.venv/bin/python').absolutePath - args "-m", "poetry", "env", "remove", "--all" - onlyIf { - rootProject.file('.venv/bin/python').exists() - } -} -cleanPythonVenv.configure { - dependsOn poetryCleanVirtualenv -} - -subprojects { - if (!isConnectorProject(project)) { - return - } - def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs -> - def task = tasks.register(taskName, Exec) { - workingDir rootDir - environment "CI", "1" // set to use more suitable logging format - commandLine rootProject.file('.venv/bin/python').absolutePath - args "-m", "poetry" - args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}" - args "run" - args "airbyte-ci", "connectors", "--name=${project.name}" - args connectorsArgs - // Forbid these kinds of tasks from running concurrently. - // We can induce serial execution by giving them all a common output directory. - outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock") - outputs.upToDateWhen { false } - } - task.configure { dependsOn poetryInstallAirbyteCI } - return task - } - - // Build connector image as part of 'assemble' task. - // This is required for local 'integrationTest' execution. - def buildConnectorImage = airbyteCIConnectorsTask( - 'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar') - buildConnectorImage.configure { - // Images for java projects always rely on the distribution tarball. - dependsOn tasks.matching { it.name == 'distTar' } - // Ensure that all files exist beforehand. - dependsOn tasks.matching { it.name == 'generate' } - } - tasks.named('assemble').configure { - // We may revisit the dependency on assemble but the dependency should always be on a base task. - dependsOn buildConnectorImage - } - - // Convenience tasks for local airbyte-ci execution. - airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build') - airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test') -} - -// produce reproducible archives -// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives) -tasks.withType(AbstractArchiveTask).configureEach { - preserveFileTimestamps = false - reproducibleFileOrder = true -} - -// pin dependency versions according to ./deps.toml -catalog { - versionCatalog { - from(files("deps.toml")) - } -} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 0e2dd4d1f248..21109a19fa65 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -4,20 +4,7 @@ plugins { repositories { // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. - - // ## Prefer repos controlled by Airbyte. - // TODO: add airbyte-controlled proxy repos here - - // ## Look into other, public repos. - // Gradle plugin portal. gradlePluginPortal() - // Maven Central has most of everything. - mavenCentral() -} - -dependencies { - implementation 'ru.vyarus:gradle-use-python-plugin:2.3.0' - implementation 'org.apache.commons:commons-text:1.10.0' } tasks.withType(Jar).configureEach { diff --git a/buildSrc/src/main/groovy/DockerHelpers.groovy b/buildSrc/src/main/groovy/DockerHelpers.groovy deleted file mode 100644 index 875f6320de9e..000000000000 --- a/buildSrc/src/main/groovy/DockerHelpers.groovy +++ /dev/null @@ -1,23 +0,0 @@ -import java.nio.file.Paths - -class DockerHelpers { - static String extractLabelValue(String dockerFile, String labelName) { - def file = dockerFile instanceof File ? dockerFile : new File(dockerFile) - return file.readLines() - .grep({ it.startsWith('LABEL') && it.contains(labelName) }) - .get(0) - .split('=')[1] - } - - static String extractImageName(String dockerFile) { - return extractLabelValue(dockerFile, "io.airbyte.name") - } - - static String extractImageVersion(String dockerFile) { - return extractLabelValue(dockerFile, "io.airbyte.version") - } - - static String getDevTaggedImage(projectDir, dockerfileName) { - return "${extractImageName(Paths.get(projectDir.absolutePath, dockerfileName).toString())}:dev" - } -} diff --git a/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle b/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle deleted file mode 100644 index e323cf7c95cc..000000000000 --- a/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle +++ /dev/null @@ -1,331 +0,0 @@ -import java.nio.file.Paths -import java.security.MessageDigest -import java.util.concurrent.ConcurrentHashMap -import org.apache.commons.text.StringSubstitutor -import org.gradle.api.DefaultTask -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.file.ConfigurableFileTree -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFile -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.PathSensitive -import org.gradle.api.tasks.PathSensitivity -import org.gradle.api.tasks.TaskAction - -/** - * AirbyteDockerLegacyTask is the task which builds a docker image based on a Dockerfile. - * - * It and the other classes in this file have "Legacy" in their name because we want to get rid of this plugin in favor - * of dagger-pipeline-based tooling like `airbyte-ci`. As of the time of this writing this is already the case for - * connectors. There are still a few remaining usages outside of connectors and they are useful to support a smooth - * local java-centric development experience with gradle, especially around integration tests. - * - * Issue https://github.com/airbytehq/airbyte/issues/30708 tracks the complete removal of this plugin. - */ -@CacheableTask -abstract class AirbyteDockerLegacyTask extends DefaultTask { - - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - FileCollection filesInDockerImage - - @Input - Map baseImageHashes - - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - File dockerFile - - @OutputFile - File idFileOutput - - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - File buildScript = project.rootProject.file('tools/bin/build_image.sh') - - @TaskAction - def dockerTask() { - project.exec { - commandLine( - buildScript.absolutePath, - project.rootDir.absolutePath, - project.projectDir.absolutePath, - dockerFile.name, - DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile.name), - idFileOutput.absolutePath, - ) - } - } -} - -/** - * AirbyteDockerLegacyTaskFactory is a convenience object to avoid passing the current project around. - */ -class AirbyteDockerLegacyTaskFactory { - - private AirbyteDockerLegacyTaskFactory() {} - - Project project - String dockerFileName - - File dockerFile() { - return project.file(dockerFileName) - } - - // This hash of the full path to the Dockerfile is the name of the task's output file. - String dockerfilePathHash() { - return MessageDigest.getInstance("MD5") - .digest(dockerFile().absolutePath.getBytes()) - .encodeHex() - .toString() - } - - // A superset of the files which are COPYed into the image, defined as the project file set - // with the .dockerignore rules applied to it. - // We could be more precise by parsing the Dockerfile but this is good enough in practice. - FileCollection filteredProjectFiles() { - ConfigurableFileTree files = project.fileTree(project.projectDir) - def dockerignore = project.file('.dockerignore') - if (!dockerignore.exists()) { - return files.filter { - file -> !file.toString().contains(".venv") - } - } - for (def rule : dockerignore.readLines()) { - if (rule.startsWith("#")) { - continue - } - rule = rule.trim() - files = (rule.startsWith("!") ? files.include(rule.substring(1)) : files.exclude(rule)) as ConfigurableFileTree - } - return files - } - - // Queries docker for all images and their hashes. - static synchronized Map collectKnownImageHashes(Project project) { - def stdout = new ByteArrayOutputStream() - project.rootProject.exec { - commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}" - standardOutput = stdout - } - Map map = [:] - stdout.toString().eachLine {line -> - def splits = line.split() - map.put(splits[0], splits[1].trim()) - } - return map - } - - // Query all docker images at most once for all tasks, at task creation time. - static def lazyImageHashesAtTaskCreationTime = new LazyImageHashesCache() - - static class LazyImageHashesCache { - private Map lazyValue - - synchronized Map get(Project project) { - if (lazyValue == null) { - lazyValue = collectKnownImageHashes(project) - } - return lazyValue - } - } - - // Global mapping of tagged image name to gradle project. - // This is populated at configuration time and accessed at task creation time. - // All keys verify isTaggedImageOwnedByThisRepo. - static def taggedImageToProject = new ConcurrentHashMap() - - static boolean isTaggedImageOwnedByThisRepo(String taggedImage) { - if (!taggedImage.startsWith("airbyte/")) { - // Airbyte's docker images are all prefixed like this. - // Anything not with this prefix is therefore not owned by this repo. - return false - } - if (taggedImage.startsWith("airbyte/base-airbyte-protocol-python:")) { - // Special case: this image is not built by this repo. - return false - } - if (!taggedImage.endsWith(":dev")) { - // Special case: this image is owned by this repo but built separate. e.g. source-file-secure - return false - } - // Otherwise, assume the image is built by this repo. - return true - } - - // Returns a mapping of each base image referenced in the Dockerfile to the corresponding hash - // in the results of collectKnownImageHashes(). If no hash was found, map to "???" instead. - Map baseTaggedImagesAndHashes(Map allKnownImageHashes) { - def taggedImages = new HashSet() - - // Look for "FROM foo AS bar" directives, and add them to the map with .put("bar", "foo") - Map imageAliases = [:] - dockerFile().eachLine { line -> - def parts = line.split() - if (parts.length >= 4 && parts[0].equals("FROM") && parts[parts.length - 2].equals("AS")) { - imageAliases.put(parts[parts.length - 1], parts[1]) - } - } - - dockerFile().eachLine { line -> - if (line.startsWith("FROM ")) { - def image = line.split()[1] - assert !image.isEmpty() - taggedImages.add(image) - } else if (line.startsWith("COPY --from=")) { - def image = line.substring("COPY --from=".length()).split()[0] - assert !image.isEmpty() - if (imageAliases[image] != null) { - taggedImages.add(imageAliases[image]) - } else { - taggedImages.add(image) - } - } - } - - Map result = [:] - for (def taggedImage : taggedImages) { - // Some image tags rely on environment variables (e.g. "FROM amazoncorretto:${JDK_VERSION}"). - taggedImage = new StringSubstitutor(System.getenv()).replace(taggedImage).trim() - result.put(taggedImage, allKnownImageHashes.getOrDefault(taggedImage, "???")) - } - return result - } - - // Create the task lazily: we shouldn't invoke 'docker' unless the task is created as part of the build. - def createTask(String taskName) { - if (!dockerFile().exists()) { - // This might not actually be necessary. It doesn't seem harmful either. - return project.tasks.register(taskName) { - logger.info "Skipping ${taskName} because ${dockerFile()} does not exist." - } - } - - // Tagged name of the image to be built by this task. - def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFileName) - // Map this project to the tagged name of the image built by this task. - taggedImageToProject.put(taggedImage, project) - // Path to the ID file to be generated by this task. - // The ID file contains the hash of the image. - def idFilePath = Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', dockerfilePathHash()) - // Register the task (lazy creation). - def airbyteDockerTask = project.tasks.register(taskName, AirbyteDockerLegacyTask) { task -> - // Set inputs. - task.filesInDockerImage = filteredProjectFiles() - task.dockerFile = this.dockerFile() - task.baseImageHashes = baseTaggedImagesAndHashes(lazyImageHashesAtTaskCreationTime.get(project)) - // Set dependencies on base images built by this repo. - for (String taggedImageDependency : task.baseImageHashes.keySet()) { - if (isTaggedImageOwnedByThisRepo(taggedImageDependency)) { - task.logger.info("adding airbyteDocker task dependency: image ${taggedImage} is based on ${taggedImageDependency}") - def dependentProject = taggedImageToProject.get(taggedImageDependency) - if (dependentProject == null) { - throw new GradleException("no known project for image ${taggedImageDependency}") - } - // Depend on 'assemble' instead of 'airbyteDocker' or 'airbyteDockerTest', it's simpler that way. - task.dependsOn(dependentProject.tasks.named('assemble')) - } - } - // Set outputs. - task.idFileOutput = idFilePath.toFile() - task.outputs.upToDateWhen { - // Because the baseImageHashes is computed at task creation time, it may be stale - // at task execution time. Let's double-check. - - // Missing dependency declarations in the gradle build may result in the airbyteDocker tasks - // to be created in the wrong order. Not worth breaking the build over. - for (Map.Entry e : task.baseImageHashes) { - if (isTaggedImageOwnedByThisRepo(e.key) && e.value == "???") { - task.logger.info "Not up to date: missing at least one airbyte base image in docker" - return false - } - } - // Fetch the hashes of the required based images anew. - def allImageHashes = collectKnownImageHashes(task.project) - // If the image to be built by this task doesn't exist in docker, then it definitely should - // be built regardless of the status of the ID file. - // For instance, it's possible that a `docker image rm` occurred between consecutive - // identical gradle builds: the ID file remains untouched but the image still needs to be rebuilt. - if (!allImageHashes.containsKey(taggedImage)) { - task.logger.info "Not up to date: ID file exists but target image not found in docker" - return false - } - // If the depended-upon base images have changed in the meantime, then it follows that the target - // image needs to be rebuilt regardless of the status of the ID file. - def currentBaseImageHashes = baseTaggedImagesAndHashes(allImageHashes) - if (!task.baseImageHashes.equals(currentBaseImageHashes)) { - task.logger.info "Not up to date: at last one base image has changed in docker since task creation" - return false - } - // In all other cases, if the ID file hasn't been touched, then the task can be skipped. - return true - } - } - - airbyteDockerTask.configure { - // Images for java projects always rely on the distribution tarball. - dependsOn project.tasks.matching { it.name == 'distTar' } - // Ensure that all files exist beforehand. - dependsOn project.tasks.matching { it.name == 'generate' } - } - project.tasks.named('assemble').configure { - // We may revisit the dependency on assemble but the dependency should always be on a base task. - dependsOn airbyteDockerTask - } - // Add a task to clean up when doing a gradle clean. - // Don't actually mess with docker, just delete the output file. - def airbyteDockerCleanTask = project.tasks.register(taskName + "Clean", Delete) { - delete idFilePath - } - project.tasks.named('clean').configure { - dependsOn airbyteDockerCleanTask - } - return airbyteDockerTask - } - - static def build(Project project, String taskName, String dockerFileName) { - def f = new AirbyteDockerLegacyTaskFactory() - f.project = project - f.dockerFileName = dockerFileName - f.createTask(taskName) - } -} - -/** - * AirbyteDockerLegacyPlugin creates an airbyteDocker task for the project when a Dockerfile is present. - * - * Following the same logic, it creates airbyteDockerTest when Dockerfile.test is present, though - * that behavior is not used anywhere except in the source-mongo connector and is therefore deprecated - * through the use of airbyte-ci. - */ -class AirbyteDockerLegacyPlugin implements Plugin { - - void apply(Project project) { - AirbyteDockerLegacyTaskFactory.build(project, 'airbyteDocker', 'Dockerfile') - - // Used only for source-mongodb. Consider removing entirely. - if (project.name.endsWith('source-mongodb')) { - AirbyteDockerLegacyTaskFactory.build(project, 'airbyteDockerTest', 'Dockerfile.test') - } - - // Used for base-normalization. - if (project.name.endsWith('base-normalization')) { - ['airbyteDockerMSSql' : 'mssql', - 'airbyteDockerMySql' : 'mysql', - 'airbyteDockerOracle' : 'oracle', - 'airbyteDockerClickhouse': 'clickhouse', - 'airbyteDockerSnowflake' : 'snowflake', - 'airbyteDockerRedshift' : 'redshift', - 'airbyteDockerTiDB' : 'tidb', - 'airbyteDockerDuckDB' : 'duckdb' - ].forEach {taskName, customConnector -> - AirbyteDockerLegacyTaskFactory.build(project, taskName, "${customConnector}.Dockerfile") - } - } - } -} diff --git a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle deleted file mode 100644 index 1719bf93596a..000000000000 --- a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle +++ /dev/null @@ -1,61 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.testing.Test - -class AirbyteIntegrationTestJavaPlugin implements Plugin { - void apply(Project project) { - project.sourceSets { - integrationTestJava { - java { - srcDir 'src/test-integration/java' - } - resources { - srcDir 'src/test-integration/resources' - } - } - } - project.tasks.named('check').configure { - dependsOn project.tasks.matching { it.name == 'compileIntegrationTestJavaJava' } - dependsOn project.tasks.matching { it.name == 'spotbugsIntegrationTestJava' } - } - - project.configurations { - integrationTestJavaImplementation.extendsFrom testImplementation - integrationTestJavaRuntimeOnly.extendsFrom testRuntimeOnly - } - - def integrationTestJava = project.tasks.register('integrationTestJava', Test) { - testClassesDirs = project.sourceSets.integrationTestJava.output.classesDirs - classpath += project.sourceSets.integrationTestJava.runtimeClasspath - - useJUnitPlatform() - testLogging() { - events 'skipped', 'started', 'passed', 'failed' - exceptionFormat 'full' - // Swallow the logs when running in airbyte-ci, rely on test reports instead. - showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI") - } - - systemProperties = project.test.systemProperties - maxParallelForks = project.test.maxParallelForks - maxHeapSize = project.test.maxHeapSize - - // Tone down the JIT when running the containerized connector to improve overall performance. - // The JVM default settings are optimized for long-lived processes in steady-state operation. - // Unlike in production, the connector containers in these tests are always short-lived. - // It's very much worth injecting a JAVA_OPTS environment variable into the container with - // flags which will reduce startup time at the detriment of long-term performance. - environment 'JOB_DEFAULT_ENV_JAVA_OPTS', '-XX:TieredStopAtLevel=1' - - // Always re-run integration tests no matter what. - outputs.upToDateWhen { false } - } - integrationTestJava.configure { - mustRunAfter project.tasks.named('check') - dependsOn project.tasks.matching { it.name == 'assemble' } - } - project.tasks.named('build').configure { - dependsOn integrationTestJava - } - } -} diff --git a/buildSrc/src/main/groovy/airbyte-java-cdk.gradle b/buildSrc/src/main/groovy/airbyte-java-cdk.gradle deleted file mode 100644 index c97594543b0b..000000000000 --- a/buildSrc/src/main/groovy/airbyte-java-cdk.gradle +++ /dev/null @@ -1,70 +0,0 @@ -/* -This class facilites detecting the Java CDK target version via readCdkTargetVersion(). -*/ - -import java.util.Properties -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.testing.Test - -class AirbyteJavaCdkPlugin implements Plugin { - - static String CDK_VERSION_FILE = "airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties" - - String readCdkTargetVersion(Project project) { - Properties cdkVersionProps = new Properties() - project.file("${project.rootDir}/${CDK_VERSION_FILE}").withInputStream { - cdkVersionProps.load(it) - } - return cdkVersionProps.getProperty('version') ?: 'undefined' - } - - @Override - void apply(Project project) { - project.ext.getCdkTargetVersion = { - return readCdkTargetVersion(project) - } - project.getTasks().create("disableLocalCdkRefs", DisableLocalCdkRefsTask.class) - project.getTasks().create("assertNotUsingLocalCdk", AssertNotUsingLocalCdkTask.class) - } - - public static class DisableLocalCdkRefsTask extends DefaultTask { - @TaskAction - public void disableLocalCdkRefs() { - // Step through the project tree and set useLocalCdk to false on all connectors - getProject().fileTree(dir: '.', include: '**/build.gradle').forEach(file -> { - String content = file.getText() - if (content.contains("useLocalCdk = true")) { - content = content.replace("useLocalCdk = true", "useLocalCdk = false") - file.setText(content) - System.out.println("Updated " + file.getPath()) - } - }) - } - } - - public static class AssertNotUsingLocalCdkTask extends DefaultTask { - @TaskAction - public void assertNotUsingLocalCdk() { - List foundPaths = new ArrayList<>() - - for (File file : getProject().fileTree(dir: '.', include: '**/build.gradle')) { - String content = file.getText() - if (content.contains("useLocalCdk = true")) { - System.err.println("Found usage of 'useLocalCdk = true' in " + file.getPath()) - foundPaths.add(file.getPath()) - } - } - - if (!foundPaths.isEmpty()) { - String errorMessage = String.format( - "Detected usage of 'useLocalCdk = true' in the following files:\n%s\n" + - "This must be set to 'false' before merging to the main branch. \n" + - "NOTE: You can run './gradlew disableLocalCdkRefs' to automatically set it to 'false' on all projects.", - String.join("\n", foundPaths) - ) - throw new RuntimeException(errorMessage) - } - } - } -} diff --git a/buildSrc/src/main/groovy/airbyte-java-connector.gradle b/buildSrc/src/main/groovy/airbyte-java-connector.gradle index 9d8a60ed88c8..d7626c8e5f11 100644 --- a/buildSrc/src/main/groovy/airbyte-java-connector.gradle +++ b/buildSrc/src/main/groovy/airbyte-java-connector.gradle @@ -5,6 +5,7 @@ Also facilitates importing and working with the Java CDK. import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test class AirbyteJavaConnectorExtension { @@ -22,7 +23,7 @@ class AirbyteJavaConnectorExtension { addCdkDependencies() } - static final List IMPLEMENTATION = [ + static final List PRE_V019_IMPLEMENTATION = [ 'airbyte-commons', 'airbyte-json-validation', 'airbyte-commons-cli', @@ -31,14 +32,14 @@ class AirbyteJavaConnectorExtension { 'init-oss', ] - static final List TEST_IMPLEMENTATION = [ + static final List PRE_V019_TEST_IMPLEMENTATION = [ 'airbyte-commons', 'airbyte-json-validation', 'airbyte-api', 'config-models-oss', ] - static final List INTEGRATION_TEST_IMPLEMENTATION = [ + static final List PRE_V019_INTEGRATION_TEST_IMPLEMENTATION = [ 'config-models-oss', 'init-oss', 'acceptance-test-harness', @@ -47,16 +48,62 @@ class AirbyteJavaConnectorExtension { void addCdkDependencies() { def projectName = { ":airbyte-cdk:java:airbyte-cdk:${it}" } def jarName = { "io.airbyte.cdk:airbyte-cdk-${it}:${cdkVersionRequired}" } + project.processIntegrationTestJavaResources { + // The metadata.yaml file is required by DestinationAcceptanceTest. + from(project.projectDir) { + include 'metadata.yaml' + duplicatesStrategy DuplicatesStrategy.EXCLUDE + } + } project.dependencies { def dep = { useLocalCdk ? project.project(projectName(it)) : jarName(it) } def testFixturesDep = { useLocalCdk ? testFixtures(project.project(projectName(it))) : "${jarName(it)}:test-fixtures" } - - IMPLEMENTATION.each { - implementation dep(it) - testFixturesImplementation dep(it) + if (useLocalCdk || !cdkVersionRequired.matches("^0\\.[0-9]\\..*|^0\\.1[0-8]\\..*")) { + // v0.19+ module structure + implementation dep("dependencies") + testImplementation dep("dependencies") + testFixturesImplementation dep("dependencies") + integrationTestJavaImplementation dep("dependencies") + integrationTestJavaImplementation testFixturesDep("dependencies") + } else { + // pre-v0.19 module structure + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) + + implementation "com.fasterxml.jackson.core:jackson-annotations:2.15.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.15.2" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2" + + implementation "com.google.guava:guava:31.1-jre" + implementation "commons-io:commons-io:2.7" + implementation "org.apache.commons:commons-compress:1.20" + implementation "org.apache.commons:commons-lang3:3.11" + implementation "org.slf4j:slf4j-api:2.0.9" + + // SLF4J as a facade over Log4j2 required dependencies + implementation "org.apache.logging.log4j:log4j-api:2.21.1" + implementation "org.apache.logging.log4j:log4j-core:2.21.1" + implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.21.1" + implementation "org.apache.logging.log4j:log4j-web:2.21.1" + implementation "com.therealvan:appender-log4j2:3.6.0" + + // Bridges from other logging implementations to SLF4J + implementation "org.slf4j:jcl-over-slf4j:2.0.9" + implementation "org.slf4j:jul-to-slf4j:2.0.9" + implementation "org.slf4j:log4j-over-slf4j:2.0.9" + + // Airbyte dependencies. + implementation "io.airbyte.airbyte-protocol:protocol-models:0.3.6" + + // CDK dependencies. + PRE_V019_IMPLEMENTATION.each { + implementation jarName(it) + testFixturesImplementation jarName(it) + } + PRE_V019_TEST_IMPLEMENTATION.each {testImplementation jarName(it) } + PRE_V019_INTEGRATION_TEST_IMPLEMENTATION.each {integrationTestJavaImplementation jarName(it) } } - TEST_IMPLEMENTATION.each {testImplementation dep(it) } - INTEGRATION_TEST_IMPLEMENTATION.each {integrationTestJavaImplementation dep(it) } (["core"] + features).each { implementation dep(it) testFixturesImplementation dep(it) @@ -78,13 +125,107 @@ class AirbyteJavaConnectorPlugin implements Plugin { @Override void apply(Project project) { - project.plugins.apply('java-test-fixtures') - project.plugins.apply(AirbyteIntegrationTestJavaPlugin) - project.plugins.apply(AirbytePerformanceTestJavaPlugin) + project.plugins.apply('application') + + project.sourceSets { + integrationTestJava { + java { + srcDir 'src/test-integration/java' + } + resources { + srcDir 'src/test-integration/resources' + } + } + performanceTestJava { + java { + srcDir 'src/test-performance/java' + } + resources { + srcDir 'src/test-performance/resources' + } + } + } + + project.tasks.named('check').configure { + dependsOn project.tasks.matching { it.name ==~ /(compile|spotbugs)[a-zA-Z]*Java/ } + } project.configurations { testFixturesImplementation.extendsFrom implementation testFixturesRuntimeOnly.extendsFrom runtimeOnly + integrationTestJavaImplementation.extendsFrom testImplementation + integrationTestJavaRuntimeOnly.extendsFrom testRuntimeOnly + performanceTestJavaImplementation.extendsFrom testImplementation + performanceTestJavaRuntimeOnly.extendsFrom testRuntimeOnly + } + + boolean withSlowTests = System.getProperty('skipSlowTests', 'false') == 'false' + project.test { + onlyIf { withSlowTests } + } + + def integrationTestJava = project.tasks.register('integrationTestJava', Test) { + testClassesDirs = project.sourceSets.integrationTestJava.output.classesDirs + classpath += project.sourceSets.integrationTestJava.runtimeClasspath + + useJUnitPlatform() + testLogging() { + events 'skipped', 'started', 'passed', 'failed' + exceptionFormat 'full' + // Swallow the logs when running in airbyte-ci, rely on test reports instead. + showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI") + } + + jvmArgs = project.test.jvmArgs + systemProperties = project.test.systemProperties + maxParallelForks = project.test.maxParallelForks + maxHeapSize = project.test.maxHeapSize + + // Tone down the JIT when running the containerized connector to improve overall performance. + // The JVM default settings are optimized for long-lived processes in steady-state operation. + // Unlike in production, the connector containers in these tests are always short-lived. + // It's very much worth injecting a JAVA_OPTS environment variable into the container with + // flags which will reduce startup time at the detriment of long-term performance. + environment 'JOB_DEFAULT_ENV_JAVA_OPTS', '-XX:TieredStopAtLevel=1' + + // Always re-run integration tests no matter what. + outputs.upToDateWhen { false } + } + integrationTestJava.configure { + mustRunAfter project.tasks.named('check') + dependsOn project.tasks.matching { it.name == 'assemble' } + onlyIf { withSlowTests } + } + project.tasks.register('integrationTest').configure { + dependsOn integrationTestJava + } + project.tasks.named('build').configure { + dependsOn integrationTestJava + } + + def performanceTestJava = project.tasks.register('performanceTestJava', Test) { + testClassesDirs = project.sourceSets.performanceTestJava.output.classesDirs + classpath += project.sourceSets.performanceTestJava.runtimeClasspath + + systemProperty "cpuLimit", System.getProperty("cpuLimit") + systemProperty "memoryLimit", System.getProperty("memoryLimit") + useJUnitPlatform() + testLogging() { + events "passed", "failed" + exceptionFormat "full" + showStandardStreams = true + } + + outputs.upToDateWhen { false } + maxHeapSize = '3g' + } + performanceTestJava.configure { + mustRunAfter project.tasks.named('check') + dependsOn project.tasks.matching { it.name == 'assemble' } + onlyIf { withSlowTests } + } + project.tasks.register('performanceTest').configure { + dependsOn performanceTestJava } project.dependencies { diff --git a/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle b/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle deleted file mode 100644 index d5ccbbbd0bea..000000000000 --- a/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle +++ /dev/null @@ -1,48 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.testing.Test - -class AirbytePerformanceTestJavaPlugin implements Plugin { - void apply(Project project) { - project.sourceSets { - performanceTestJava { - java { - srcDir 'src/test-performance/java' - } - resources { - srcDir 'src/test-performance/resources' - } - } - } - project.tasks.named('check').configure { - dependsOn project.tasks.matching { it.name == 'compilePerformanceTestJavaJava' } - dependsOn project.tasks.matching { it.name == 'spotbugsPerformanceTestJava' } - } - - project.configurations { - performanceTestJavaImplementation.extendsFrom testImplementation - performanceTestJavaRuntimeOnly.extendsFrom testRuntimeOnly - } - - def performanceTestJava = project.tasks.register('performanceTestJava', Test) { - testClassesDirs = project.sourceSets.performanceTestJava.output.classesDirs - classpath += project.sourceSets.performanceTestJava.runtimeClasspath - - systemProperty "cpuLimit", System.getProperty("cpuLimit") - systemProperty "memoryLimit", System.getProperty("memoryLimit") - useJUnitPlatform() - testLogging() { - events "passed", "failed" - exceptionFormat "full" - showStandardStreams = true - } - - outputs.upToDateWhen { false } - maxHeapSize = '3g' - } - performanceTestJava.configure { - mustRunAfter project.tasks.named('check') - dependsOn project.tasks.matching { it.name == 'assemble' } - } - } -} diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle deleted file mode 100644 index 59f14890c75b..000000000000 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ /dev/null @@ -1,185 +0,0 @@ -import groovy.io.FileType -import groovy.io.FileVisitResult -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.Exec -import ru.vyarus.gradle.plugin.python.task.PythonTask - -class Helpers { - static addTestTaskIfTestFilesFound(Project project, String testFilesDirectory, String taskName, taskDependencies) { - """ - This method verifies if there are test files in a directory before adding the pytest task to run tests on that directory. This is needed - because if there are no tests in that dir and we run pytest on it, it exits with exit code 5 which gradle takes to mean that the process - failed, since it's non-zero. This means that if a module doesn't need a unit or integration test, it still needs to add a dummy test file - like: - - ``` - def make_ci_pass_test(): - assert True - ``` - - So we use this method to leverage pytest's test discovery rules (https://docs.pytest.org/en/6.2.x/goodpractices.html#conventions-for-python-test-discovery) - to selectively run pytest based on whether there seem to be test files in that directory. - Namely, if the directory contains a file whose name is test_*.py or *_test.py then it's a test. - - See https://github.com/airbytehq/airbyte/issues/4979 for original context - """ - - boolean requiresTasks = false - if (project.file(testFilesDirectory).exists()) { - def testDir = project.projectDir.toPath().resolve(testFilesDirectory) - testDir.traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) {file -> - requiresTasks = true - // If a file is found, terminate the traversal, thus causing this task to be declared at most once - return FileVisitResult.TERMINATE - } - } - if (!requiresTasks) { - return - } - - def coverageTask = project.tasks.register(taskName, PythonTask) { - def dataFile = "${testFilesDirectory}/.coverage.${taskName}" - def rcFile = project.rootProject.file('pyproject.toml').absolutePath - def testConfig = project.file('pytest.ini').exists() ? 'pytest.ini' : project.rootProject.file('pyproject.toml').absolutePath - - module = "coverage" - command = "run --data-file=${dataFile} --rcfile=${rcFile} -m pytest -s ${testFilesDirectory} -c ${testConfig}" - } - coverageTask.configure { - dependsOn taskDependencies - } - } -} - -class AirbytePythonPlugin implements Plugin { - - void apply(Project project) { - - def venvDirectoryName = '.venv' - - // Add a task that allows cleaning up venvs to every python project - def cleanPythonVenv = project.tasks.register('cleanPythonVenv', Exec) { - commandLine 'rm' - args '-rf', "${project.projectDir.absolutePath}/${venvDirectoryName}" - } - project.tasks.named('clean').configure { - dependsOn cleanPythonVenv - } - - project.plugins.apply 'ru.vyarus.use-python' - - // Configure gradle python plugin. - project.python { - envPath = venvDirectoryName - minPythonVersion '3.10' - - // Amazon Linux support. - // The airbyte-ci tool runs gradle tasks in AL2023-based containers. - // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. - // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. - try { - if ("python3.11 --version".execute().waitFor() == 0) { - // python3.11 definitely exists at this point, use it instead of 'python3'. - pythonBinary "python3.11" - } - } catch (IOException _) { - // Swallow exception if python3.11 is not installed. - } - // Pyenv support. - try { - def pyenvRoot = "pyenv root".execute() - def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() - // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. - pythonBinary "python" - if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { - pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" - } - } catch (IOException _) { - // Swallow exception if pyenv is not installed. - } - - scope 'VIRTUALENV' - installVirtualenv = true - pip 'pip:23.2.1' - pip 'mccabe:0.6.1' - // https://github.com/csachs/pyproject-flake8/issues/13 - pip 'flake8:4.0.1' - // flake8 doesn't support pyproject.toml files - // and thus there is the wrapper "pyproject-flake8" for this - pip 'pyproject-flake8:0.0.1a2' - pip 'pytest:6.2.5' - pip 'coverage[toml]:6.3.1' - } - - // Attempt to install anything in requirements.txt. - // By convention this should only be dependencies whose source is located in the project. - if (project.file('requirements.txt').exists()) { - project.tasks.register('installLocalReqs', PythonTask) { - module = "pip" - command = "install -r requirements.txt" - inputs.file('requirements.txt') - outputs.file('build/installedlocalreqs.txt') - } - } else if (project.file('setup.py').exists()) { - // If requirements.txt does not exists, install from setup.py instead, assume a dev or "tests" profile exists. - // In this case, there is no need to depend on the base python modules since everything should be contained in the setup.py. - project.tasks.register('installLocalReqs', PythonTask) { - module = "pip" - command = "install .[dev,tests]" - inputs.file('setup.py') - outputs.file('build/installedlocalreqs.txt') - } - } else { - return - } - - def installLocalReqs = project.tasks.named('installLocalReqs') - - def flakeCheck = project.tasks.register('flakeCheck', PythonTask) { - module = "pflake8" - command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" - } - - def installReqs = project.tasks.register('installReqs', PythonTask) { - module = "pip" - command = "install .[main]" - inputs.file('setup.py') - outputs.file('build/installedreqs.txt') - } - installReqs.configure { - dependsOn installLocalReqs - } - - project.tasks.named('check').configure { - dependsOn installReqs - dependsOn flakeCheck - } - - def installTestReqs = project.tasks.register('installTestReqs', PythonTask) { - module = "pip" - command = "install .[tests]" - inputs.file('setup.py') - outputs.file('build/installedtestreqs.txt') - } - installTestReqs.configure { - dependsOn installReqs - } - - Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'testPython', installTestReqs) - project.tasks.named('check').configure { - dependsOn project.tasks.matching { it.name == 'testPython' } - } - - Helpers.addTestTaskIfTestFilesFound(project, 'integration_tests', 'integrationTestPython', installTestReqs) - def integrationTestTasks = project.tasks.matching { it.name == 'integrationTestPython' } - integrationTestTasks.configureEach { - dependsOn project.tasks.named('assemble') - mustRunAfter project.tasks.named('check') - } - project.tasks.named('build').configure { - dependsOn integrationTestTasks - } - } -} diff --git a/deps.toml b/deps.toml index e6d34339ec3a..1caea5603cc8 100644 --- a/deps.toml +++ b/deps.toml @@ -1,5 +1,5 @@ [versions] -airbyte-protocol = "0.3.6" +airbyte-protocol = "0.5.0" commons_io = "2.7" testcontainers = "1.19.0" datadog-version = "0.111.0" @@ -12,12 +12,7 @@ jooq = "3.13.4" junit-jupiter = "5.9.1" kotlin = "1.9.0" log4j = "2.21.1" -lombok = "1.18.24" -micronaut = "3.8.3" -micronaut-data = "3.9.4" -micronaut-jaxrs = "3.4.0" -micronaut-security = "3.9.2" -micronaut-test = "3.8.0" +lombok = "1.18.30" postgresql = "42.6.0" reactor = "3.5.2" segment = "2.1.1" @@ -50,6 +45,7 @@ testcontainers-postgresql = { module = "org.testcontainers:postgresql", version. testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "testcontainers" } testcontainers-scylla = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } testcontainers-tidb = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-tidb-source = { module = "org.testcontainers:tidb", version.ref = "testcontainers" } datadog-trace-api = { module = "com.datadoghq:dd-trace-api", version.ref = "datadog-version" } datadog-trace-ot = { module = "com.datadoghq:dd-trace-ot", version.ref = "datadog-version" } fasterxml = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "fasterxml_version" } @@ -85,12 +81,13 @@ kotlinx-cli-jvm = { module = "org.jetbrains.kotlinx:kotlinx-cli-jvm", version = launchdarkly = { module = "com.launchdarkly:launchdarkly-java-server-sdk", version = "6.0.1" } log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" } -log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } +log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } +log4j-slf4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j-impl", version.ref = "log4j" } log4j-over-slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j" } log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j" } lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } micrometer-statsd = { module = "io.micrometer:micrometer-registry-statsd", version = "1.9.3" } -mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "4.6.1" } +mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.10.0" } mockk = { module = "io.mockk:mockk", version = "1.13.3" } mongo-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "4.10.2" } otel-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.14.0" } @@ -116,48 +113,14 @@ debezium-mongodb = { module = "io.debezium:debezium-connector-mongodb", version. debezium-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium"} debezium-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium"} -# Micronaut-related dependencies -h2-database = { module = "com.h2database:h2", version = "2.1.214" } -hibernate-types = { module = "com.vladmihalcea:hibernate-types-52", version = "2.16.3" } -jakarta-inject = { module = "jakarta.annotation:jakarta.annotation-api", version = "2.1.1" } -javax-transaction = { module = "javax.transaction:javax.transaction-api", version = "1.3" } -micronaut-bom = { module = "io.micronaut:micronaut-bom", version.ref = "micronaut" } -micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version = "3.5.0"} -micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version.ref = "micronaut-data" } -micronaut-data-tx = { module = "io.micronaut.data:micronaut-data-tx", version.ref = "micronaut-data" } -micronaut-flyway = { module = "io.micronaut.flyway:micronaut-flyway", version = "5.4.1" } -micronaut-inject = { module = "io.micronaut:micronaut-inject" } -micronaut-http = { module = "io.micronaut:micronaut-http", version.ref = "micronaut" } -micronaut-http-client = { module = "io.micronaut:micronaut-http-client" } -micronaut-http-server-netty = { module = "io.micronaut:micronaut-http-server-netty", version.ref = "micronaut" } -micronaut-inject-java = { module = "io.micronaut:micronaut-inject-java", version.ref = "micronaut" } -micronaut-jaxrs-processor = { module = "io.micronaut.jaxrs:micronaut-jaxrs-processor", version.ref = "micronaut-jaxrs" } -micronaut-jaxrs-server = { module = "io.micronaut.jaxrs:micronaut-jaxrs-server", version.ref = "micronaut-jaxrs" } -micronaut-jdbc = { module = "io.micronaut.sql:micronaut-jdbc", version = "4.7.2" } -micronaut-jdbc-hikari = { module = "io.micronaut.sql:micronaut-jdbc-hikari" } -micronaut-jooq = { module = "io.micronaut.sql:micronaut-jooq" } -micronaut-management = { module = "io.micronaut:micronaut-management" } -micronaut-runtime = { module = "io.micronaut:micronaut-runtime" } -micronaut-security = { module = "io.micronaut.security:micronaut-security", version.ref = "micronaut-security" } -micronaut-test-core = { module = "io.micronaut.test:micronaut-test-core", version.ref = "micronaut-test" } -micronaut-test-junit5 = { module = "io.micronaut.test:micronaut-test-junit5", version.ref = "micronaut-test" } -micronaut-validation = { module = "io.micronaut:micronaut-validation" } - [bundles] apache = ["apache-commons", "apache-commons-lang"] datadog = ["datadog-trace-api", "datadog-trace-ot"] jackson = ["jackson-databind", "jackson-annotations", "jackson-dataformat", "jackson-datatype"] junit = ["junit-jupiter-api", "junit-jupiter-params", "mockito-junit-jupiter"] -log4j = ["log4j-api", "log4j-core", "log4j-impl", "log4j-web"] -micronaut = ["jakarta-inject", "javax-transaction", "micronaut-http-server-netty", "micronaut-http-client", "micronaut-inject", "micronaut-validation", "micronaut-runtime", "micronaut-management", "micronaut-security", "micronaut-jaxrs-server", "micronaut-flyway", "micronaut-jdbc-hikari", "micronaut-jooq"] -micronaut-annotation = ["jakarta-inject", "micronaut-inject-java"] -micronaut-annotation-processor = ["micronaut-inject-java", "micronaut-management", "micronaut-validation", "micronaut-data-processor", "micronaut-jaxrs-processor"] -micronaut-server = ["micronaut-jaxrs-processor", "micronaut-jaxrs-server"] -micronaut-test = ["micronaut-test-core", "micronaut-test-junit5", "h2-database"] -micronaut-test-annotation-processor = ["micronaut-inject-java"] +log4j = ["log4j-api", "log4j-core", "log4j-slf4j-impl", "log4j-slf4j2-impl", "log4j-web"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] temporal = ["temporal-sdk", "temporal-serviceclient"] [plugins] kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } -kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } \ No newline at end of file diff --git a/docs/.gitbook/assets/add-a-destination/getting-started-destination-page.png b/docs/.gitbook/assets/add-a-destination/getting-started-destination-page.png deleted file mode 100644 index 16c15dadfe81..000000000000 Binary files a/docs/.gitbook/assets/add-a-destination/getting-started-destination-page.png and /dev/null differ diff --git a/docs/.gitbook/assets/add-a-source/getting-started-source-page.png b/docs/.gitbook/assets/add-a-source/getting-started-source-page.png deleted file mode 100644 index 32d8c65717c5..000000000000 Binary files a/docs/.gitbook/assets/add-a-source/getting-started-source-page.png and /dev/null differ diff --git a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-config.png b/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-config.png deleted file mode 100644 index 2f800ab71b84..000000000000 Binary files a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-config.png and /dev/null differ diff --git a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-streams.png b/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-streams.png deleted file mode 100644 index bde06fd88982..000000000000 Binary files a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-streams.png and /dev/null differ diff --git a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-success.png b/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-success.png deleted file mode 100644 index 7bf01c37f1f6..000000000000 Binary files a/docs/.gitbook/assets/set-up-a-connection/getting-started-connection-success.png and /dev/null differ diff --git a/docs/access-management/sso-providers/assets/okta-create-new-app-integration.png b/docs/access-management/sso-providers/assets/okta-create-new-app-integration.png new file mode 100644 index 000000000000..bff936656aad Binary files /dev/null and b/docs/access-management/sso-providers/assets/okta-create-new-app-integration.png differ diff --git a/docs/access-management/sso-providers/azure-entra-id.md b/docs/access-management/sso-providers/azure-entra-id.md new file mode 100644 index 000000000000..3b71e7c2ac7c --- /dev/null +++ b/docs/access-management/sso-providers/azure-entra-id.md @@ -0,0 +1,104 @@ +--- +sidebar_label: Microsoft Entra ID +products: cloud-teams, oss-enterprise +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Setup Single Sign-On via Microsoft Entra ID + +This page guides you through setting up [Single Sign-On](../sso.md) with Airbyte using **Microsoft Entra ID** (formerly known as **Azure ActiveDirectory**). + +Airbyte will communicate with your Entra ID using OpenID Connect (OIDC). + + + + +## Creating an Entra ID app for Airbyte + +:::info +The following steps need to be executed by an administrator of your company's Microsoft Entra ID account. +::: + +You'll require to know your **Company Identifier** to create your application. You receive this +from your contact at Airbyte. + +### Create application + +You will need to create a new Entra ID application for Airbyte. Log into the [Azure Portal](https://portal.azure.com/) and search for the Entra ID service. + +From the overview page of Entra ID, press **Add** > **App registration** on the top of the screen. + +Specify any name you want (e.g. "Airbyte") and configure a **Redirect URI** of type **Web** with the following value: + +``` +https://cloud.airbyte.com/auth/realms//broker/default/endpoint +``` + +Hit **Register** to create the application. + +### Create Client credentials + +To create Client credentials for Airbyte to talk to your application head to **Certificates & Secrets** on the detail screen of your application and select the **Client secrets** tab. + +Click **New client secret**, specify any Description you want and any expiry date you want. + +:::tip +We recommend to chose an expiry date of at least 12 months. You'll need to pass in the new client secret every time the old one expires to continue being able to log in via Entra ID. +::: + +Copy the **Value** (the Client Secret itself) immediately after creation. You won't be able to view this later on. + +### Setup information needed + +You'll need to pass your Airbyte contact the following information of the created application. + +* **Client Secret**: as copied above +* **Application (client) ID**: You'll find this in the **Essentials** section on the **Overview** page of the application you created +* **OpenID Connect metadata document**: You'll find this in the **Endpoints** panel, that you can open from the top bar on the **Overview** page + +Once we've received this information from you, We'll setup SSO for you and let you know once it's ready to be used. + + + + +## Creating an Entra ID app for Airbyte + +:::info +The following steps need to be executed by an administrator of your company's Azure Entra ID account. +::: + +### Create application + +You will need to create a new Entra ID application for Airbyte. Log into the [Azure Portal](https://portal.azure.com/) and search for the Entra ID service. + +From the overview page of Entra ID, press **Add** > **App registration** on the top of the screen. The name you select is your app integration name. Once chosen, configure a **Redirect URI** of type **Web** with the following value: + +``` +/auth/realms/airbyte/broker//endpoint +``` + +Hit **Register** to create the application. + +### Create client credentials + +To create client credentials for Airbyte to interface with your application, head to **Certificates & Secrets** on the detail screen of your application and select the **Client secrets** tab. Then: +1. Click **New client secret**, and enter the expiry date of your choosing. You'll need to pass in the new client secret every time the old one expires to continue being able to log in via Entra ID. +2. Copy the **Value** (the client secret itself) immediately after creation. You won't be able to view this later on. + +### Setup information needed + +Once your Microsoft Entra ID app is set up, you're ready to deploy Airbyte Self-Managed Enterprise with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: + + * OpenID Connect metadata document: You'll find this in the list of endpoints found in the **Endpoints** panel, which you can open from the top bar of the **Overview** page. This will be used to populate the `Domain` field in your `airbyte.yml`. + * App Integration Name: The name of the Entra ID application created in the first step. + * Client ID: You'll find this in the **Essentials** section on the **Overview** page of the application you created. + * Client Secret: The client secret you copied in the previous step. + +Use this information to configure the auth details of your `airbyte.yml` for your Self-Managed Enterprise deployment. To learn more on deploying Self-Managed Enterprise, see our [implementation guide](/enterprise-setup/implementation-guide). + + + + + diff --git a/docs/access-management/sso-providers/okta.md b/docs/access-management/sso-providers/okta.md new file mode 100644 index 000000000000..241c385cdf05 --- /dev/null +++ b/docs/access-management/sso-providers/okta.md @@ -0,0 +1,108 @@ +--- +sidebar_label: Okta +products: oss-enterprise, cloud-teams +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Setup Single Sign-On via Okta + +This page guides you through setting up Okta for [Single Sign-On](../sso.md) with Airbyte. + +Airbyte will communicate with your Okta account using OpenID Connect (OIDC). + +## Creating an Okta app for Airbyte + +:::info +The following steps need to be executed by an administrator of your company's Okta account. +::: + +You will need to create a new Okta OIDC App Integration for your Airbyte. Documentation on how to do this in Okta can be found [here](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_oidc.htm). + +You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: + +![Screenshot of Okta app integration creation modal](./assets/okta-create-new-app-integration.png) + +On the following screen you'll need to configure all parameters for your Okta application: + + + + You'll require to know your **Company Identifier** to fill in those values. You receive this + from your contact at Airbyte. + + Create the application with the following parameters: + +
      +
      **App integration name**
      +
      A human readable name for the application (e.g. **Airbyte Cloud**). This is only used for identification inside your Okta dashboard.
      +
      **Logo** (optional)
      +
      You can upload an Airbyte logo, which you can find at https://airbyte.com/company/press
      +
      **Grant type**
      +
      Only **Authorization Code** should be selected
      +
      **Sign-in redirect URIs**
      +
      + ``` + https://cloud.airbyte.com/auth/realms//broker/default/endpoint + ``` +
      +
      **Sign-out redirect URIs**
      +
      + ``` + https://cloud.airbyte.com/auth/realms//broker/default/endpoint/logout_response + ``` +
      +
      **Trusted Origins**
      +
      Leave empty
      +
      **Assignments > Controlled Access**
      +
      You can control whether everyone in your Okta organization should be able to access Airbyte using their Okta account or limit it only to a subset of your users by selecting specific groups who should get access.
      +
      + + You'll need to pass your Airbyte contact the following information of the created application. After that we'll setup SSO for you and let you know once it's ready. + + * Your **Okta domain** (it's not specific to this application, see [Find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) + * **Client ID** + * **Client Secret** +       + + Create the application with the following parameters: + +
      +
      **App integration name**
      +
      Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`. Screenshot of Okta app integration name Spaces or special characters in this field could result in invalid redirect URIs.
      +
      **Logo** (optional)
      +
      You can upload an Airbyte logo, which you can find at https://airbyte.com/company/press
      +
      **Grant type**
      +
      Only **Authorization Code** should be selected
      +
      **Sign-in redirect URIs**
      +
      + ``` + /auth/realms/airbyte/broker//endpoint + ``` + + `` refers to the domain you access your Airbyte instance at, e.g. `https://airbyte.internal.mycompany.com` + + `` refers to the value you entered in the **App integration name** field +
      +
      **Sign-out redirect URIs**
      +
      + ``` + /auth/realms/airbyte/broker//endpoint/logout_response + ``` +
      +
      **Trusted Origins**
      +
      Leave empty
      +
      **Assignments > Controlled Access**
      +
      You can control whether everyone in your Okta organization should be able to access Airbyte using their Okta account or limit it only to a subset of your users by selecting specific groups who should get access.
      +
      + + Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: + + * Okta domain ([How to find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) + * App Integration Name + * Client ID + * Client Secret + + Visit the [implementation guide](/enterprise-setup/implementation-guide.md) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. +       +       diff --git a/docs/access-management/sso.md b/docs/access-management/sso.md new file mode 100644 index 000000000000..065c7ed74e5b --- /dev/null +++ b/docs/access-management/sso.md @@ -0,0 +1,38 @@ +--- +products: oss-enterprise, cloud-teams +--- + +# Single Sign-On (SSO) + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +Single Sign-On (SSO) allows you to enable logging into Airbyte using your existing Identity Provider (IdP) like Okta or Active Directory. + +SSO is available in Airbyte Enterprise and on Cloud with the Teams add-on. [Talk to us](https://airbyte.com/company/talk-to-sales) if you are interested in setting up SSO for your organization. + +## Set up + +You can find setup explanations for all our supported Identity Providers on the following subpages: + +```mdx-code-block +import DocCardList from '@theme/DocCardList'; + + +``` + +## Logging in + + + + Once we inform you that you’re all set up, you can log into Airbyte using SSO by visiting [cloud.airbyte.com/sso](https://cloud.airbyte.com/sso) or select the **Continue with SSO** option on the login screen. + + Specify your _company identifier_ and hit “Continue with SSO”. You’ll be forwarded to your IdP's login page (e.g. Okta login page). Log into your work account and you’ll be forwarded back to Airbyte Cloud and be logged in. + + *Note:* you were already logged into your company’s Okta account you might not see any login screen and directly get forwarded back to Airbyte Cloud. + + + Accessing your self hosted Airbyte will automatically forward you to your IdP's login page (e.g. Okta login page). Log into your work account and you’ll be forwarded back to your Airbyte and be logged in. + + + diff --git a/docs/api-documentation.md b/docs/api-documentation.md index 448a63b2e952..53cf2c7845aa 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # API documentation Airbyte has two sets of APIs which are intended for different uses. The table below outlines their descriptions, use cases, availability and status. diff --git a/docs/assets/docs/airbyte-lib-high-level-architecture.svg b/docs/assets/docs/airbyte-lib-high-level-architecture.svg new file mode 100644 index 000000000000..832788f0cc8b --- /dev/null +++ b/docs/assets/docs/airbyte-lib-high-level-architecture.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/cli-documentation.md b/docs/cli-documentation.md deleted file mode 100644 index 17b13dac71bd..000000000000 --- a/docs/cli-documentation.md +++ /dev/null @@ -1,711 +0,0 @@ -# CLI documentation - -:::caution -The Octavia CLI is an alpha, unofficial CLI that won't be maintained. -::: - -:::tip Recommendation -We recommend all users leverage the official [Airbyte Terraform Provider](https://reference.airbyte.com/reference/using-the-terraform-provider), instead of this CLI. -::: - -## What is `octavia` CLI? - -Octavia CLI is a tool to manage Airbyte configurations in YAML. -It has the following features: - -- Scaffolding of a readable directory architecture that will host the YAML configs (`octavia init`). -- Auto-generation of YAML config file that matches the resources' schemas (`octavia generate`). -- Manage Airbyte resources with YAML config files. -- Safe resources update through diff display and validation (`octavia apply`). -- Simple secret management to avoid versioning credentials. - -## Why should I use `octavia` CLI? - -A CLI provides freedom to users to use the tool in whatever context and use case they have. -These are non-exhaustive use cases `octavia` can be convenient for: - -- Managing Airbyte configurations with a CLI instead of a web UI. -- Versioning Airbyte configurations in Git. -- Updating of Airbyte configurations in an automated deployment pipeline. -- Integrating the Airbyte configuration deployment in a dev ops tooling stack: Helm, Ansible etc. -- Streamlining the deployment of Airbyte configurations to multiple Airbyte instance. - -Readers can refer to our [opened GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) to check the ongoing work on this project. - -## Table of content - -- [Workflow](#workflow) -- [Secret management](#secret-management) -- [Install](#install) -- [Commands reference](#commands-reference) -- [Contributing](#contributing) -- [Telemetry](#telemetry) -- [Changelog](#changelog) - -## Workflow - -### 1. Generate local YAML files for sources or destinations - -1. Retrieve the _definition id_ of the connector you want to use using `octavia list` command. -2. Generate YAML configuration running `octavia generate source ` or `octavia generate destination `. - -### 2. Edit your local YAML configurations - -1. Edit the generated YAML configurations according to your need. -2. Use the [secret management feature](#secret-management) feature to avoid storing credentials in the YAML files. - -### 3. Create the declared sources or destinations on your Airbyte instance - -1. Run `octavia apply` to create the **sources** and **destinations** - -### 4. Generate connections - -1. Run `octavia octavia generate connection --source --destination ` to create a YAML configuration for a new connection. -2. Edit the created configuration file according to your need: change the scheduling or the replicated streams list. - -### 5. Create the declared connections - -1. Run `octavia apply` to create the newly declared connection on your Airbyte instance. - -### 6. Update your configurations - -Changes in your local configurations can be propagated to your Airbyte instance using `octavia apply`. You will be prompted for validation of changes. You can bypass the validation step using the `--force` flag. - -## Secret management - -Sources and destinations configurations have credential fields that you **do not want to store as plain text in your VCS**. -`octavia` offers secret management through environment variables expansion: - -```yaml -configuration: - password: ${MY_PASSWORD} -``` - -If you have set a `MY_PASSWORD` environment variable, `octavia apply` will load its value into the `password` field. - -## Install - -### Requirements - -We decided to package the CLI in a docker image with portability in mind. -**[Please install and run Docker if you are not](https://docs.docker.com/get-docker/)**. - -### As a command available in your bash profile - -```bash -curl -s -o- https://raw.githubusercontent.com/airbytehq/airbyte/master/octavia-cli/install.sh | bash -``` - -This script: - -1. Pulls the [octavia-cli image](https://hub.docker.com/r/airbyte/octavia-cli/tags) from our Docker registry. -2. Creates an `octavia` alias in your profile. -3. Creates a `~/.octavia` file whose values are mapped to the octavia container's environment variables. - -### Using `docker run` - -```bash -touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container -mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.32 -``` - -### Using `docker-compose` - -Using octavia in docker-compose could be convenient for automatic `apply` on start-up. - -Add another entry in the services key of your Airbyte `docker-compose.yml` - -```yaml -services: - # . . . - octavia-cli: - image: airbyte/octavia-cli:latest - command: apply --force - env_file: - - ~/.octavia # Use a local env file to store variables that will be mapped the octavia-cli container - volumes: - - :/home/octavia-project - depends_on: - - webapp -``` - -Other commands besides `apply` can be run like so: - -```bash -docker compose run octavia-cli ` -``` - -## Commands reference - -### `octavia` command flags - -| **Flag** | **Description** | **Env Variable** | **Default** | -| ---------------------------------------- | --------------------------------------------------------------------------------- |----------------------------| ------------------------------------------------------ | -| `--airbyte-url` | Airbyte instance URL. | `AIRBYTE_URL` | `http://localhost:8000` | -| `--airbyte-username` | Airbyte instance username (basic auth). | `AIRBYTE_USERNAME` | `airbyte` | -| `--airbyte-password` | Airbyte instance password (basic auth). | `AIRBYTE_PASSWORD` | `password` | -| `--workspace-id` | Airbyte workspace id. | `AIRBYTE_WORKSPACE_ID` | The first workspace id found on your Airbyte instance. | -| `--enable-telemetry/--disable-telemetry` | Enable or disable the sending of telemetry data. | `OCTAVIA_ENABLE_TELEMETRY` | True | -| `--api-http-header` | HTTP Header value pairs passed while calling Airbyte's API | None | None | -| `--api-http-headers-file-path` | Path to the YAML file that contains custom HTTP Headers to send to Airbyte's API. | None | None | - -#### Using custom HTTP headers - -You can set custom HTTP headers to send to Airbyte's API with options: - -```bash -octavia --api-http-header Header-Name Header-Value --api-http-header Header-Name-2 Header-Value-2 list connectors sources -``` - -You can also use a custom YAML file (one is already created on init in `api_http_headers.yaml`) to declare the HTTP headers to send to the API: - -```yaml -headers: - Authorization: Bearer my-secret-token - User-Agent: octavia-cli/0.0.0 -``` - -Environment variable expansion is available in this Yaml file - -```yaml -headers: - Authorization: Bearer ${MY_API_TOKEN} -``` - -**Options based headers are overriding file based headers if an header is declared in both.** - -### `octavia` subcommands - -| **Command** | **Usage** | -| ----------------------------------------- | ------------------------------------------------------------------------------------------ | -| **`octavia init`** | Initialize required directories for the project. | -| **`octavia list connectors sources`** | List all sources connectors available on the remote Airbyte instance. | -| **`octavia list connectors destination`** | List all destinations connectors available on the remote Airbyte instance. | -| **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | -| **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | -| **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | -| **`octavia get source`** | Get the JSON representation of an existing source in current the Airbyte workspace. | -| **`octavia get destination`** | Get the JSON representation of an existing destination in the current Airbyte workspace. | -| **`octavia get connection`** | Get the JSON representation of an existing connection in the current Airbyte workspace. | -| **`octavia import all`** | Import all existing sources, destinations and connections to manage them with octavia-cli. | -| **`octavia import source`** | Import an existing source to manage it with octavia-cli. | -| **`octavia import destination`** | Import an existing destination to manage it with octavia-cli. | -| **`octavia import connection`** | Import an existing connection to manage it with octavia-cli. | -| **`octavia generate source`** | Generate a local YAML configuration for a new source. | -| **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | -| **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | -| **`octavia apply`** | Create or update Airbyte remote resources according to local YAML configurations. | - -#### `octavia init` - -The `octavia init` commands scaffolds the required directory architecture for running `octavia generate` and `octavia apply` commands. - -**Example**: - -```bash -$ mkdir my_octavia_project && cd my_octavia_project -$ octavia init -🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace e1f46f7d-5354-4200-aed6-7816015ca54b. -🐙 - Project is not yet initialized. -🔨 - Initializing the project. -✅ - Created the following directories: sources, destinations, connections. -$ ls -connections destinations sources -``` - -#### `octavia list connectors sources` - -List all the source connectors currently available on your Airbyte instance. - -**Example**: - -```bash -$ octavia list connectors sources -NAME DOCKER REPOSITORY DOCKER IMAGE TAG SOURCE DEFINITION ID -Airtable airbyte/source-airtable 0.1.1 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212 -AWS CloudTrail airbyte/source-aws-cloudtrail 0.1.4 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 -Amazon Ads airbyte/source-amazon-ads 0.1.3 c6b0a29e-1da9-4512-9002-7bfd0cba2246 -Amazon Seller Partner airbyte/source-amazon-seller-partner 0.2.16 e55879a8-0ef8-4557-abcf-ab34c53ec460 -``` - -#### `octavia list connectors destinations` - -List all the destinations connectors currently available on your Airbyte instance. - -**Example**: - -```bash -$ octavia list connectors destinations -NAME DOCKER REPOSITORY DOCKER IMAGE TAG DESTINATION DEFINITION ID -Azure Blob Storage airbyte/destination-azure-blob-storage 0.1.3 b4c5d105-31fd-4817-96b6-cb923bfc04cb -Amazon SQS airbyte/destination-amazon-sqs 0.1.0 0eeee7fb-518f-4045-bacc-9619e31c43ea -BigQuery airbyte/destination-bigquery 0.6.11 22f6c74f-5699-40ff-833c-4a879ea40133 -BigQuery (denormalized typed struct) airbyte/destination-bigquery-denormalized 0.2.10 079d5540-f236-4294-ba7c-ade8fd918496 -``` - -#### `octavia list workspace sources` - -List all the sources existing on your targeted Airbyte instance. - -**Example**: - -```bash -$ octavia list workspace sources -NAME SOURCE NAME SOURCE ID -weather OpenWeather c4aa8550-2122-4a33-9a21-adbfaa638544 -``` - -#### `octavia list workspace destinations` - -List all the destinations existing on your targeted Airbyte instance. - -**Example**: - -```bash -$ octavia list workspace destinations -NAME DESTINATION NAME DESTINATION ID -my_db Postgres c0c977c2-48e7-46fe-9f57-576285c26d42 -``` - -#### `octavia list workspace connections` - -List all the connections existing on your targeted Airbyte instance. - -**Example**: - -```bash -$ octavia list workspace connections -NAME CONNECTION ID STATUS SOURCE ID DESTINATION ID -weather_to_pg a4491317-153e-436f-b646-0b39338f9aab active c4aa8550-2122-4a33-9a21-adbfaa638544 c0c977c2-48e7-46fe-9f57-576285c26d42 -``` - -#### `octavia get source or ` - -Get an existing source in current the Airbyte workspace. You can use a source ID or name. - -| **Argument** | **Description** | -| ------------- | ---------------- | -| `SOURCE_ID` | The source id. | -| `SOURCE_NAME` | The source name. | - -**Examples**: - -```bash -$ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 -{'connection_configuration': {'key': '**********', - 'start_date': '2010-01-01T00:00:00.000Z', - 'token': '**********'}, - 'name': 'Pokemon', - 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', - 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', - 'source_name': 'My Poke', - 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} -``` - -```bash -$ octavia get source "My Poke" -{'connection_configuration': {'key': '**********', - 'start_date': '2010-01-01T00:00:00.000Z', - 'token': '**********'}, - 'name': 'Pokemon', - 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', - 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', - 'source_name': 'My Poke', - 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} -``` - -#### `octavia get destination or ` - -Get an existing destination in current the Airbyte workspace. You can use a destination ID or name. - -| **Argument** | **Description** | -| ------------------ | --------------------- | -| `DESTINATION_ID` | The destination id. | -| `DESTINATION_NAME` | The destination name. | - -**Examples**: - -```bash -$ octavia get destination c0c977c2-48e7-46fe-9f57-576285c26d42 -{ - "destinationDefinitionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", - "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", - "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", - "connectionConfiguration": { - "user": "charles" - }, - "name": "pg", - "destinationName": "Postgres" -} -``` - -```bash -$ octavia get destination pg -{ - "destinationDefinitionId": "18102e7c-5160-4000-821f-4d7cfdf87201", - "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", - "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", - "connectionConfiguration": { - "user": "charles" - }, - "name": "string", - "destinationName": "string" -} -``` - -#### `octavia get connection or ` - -Get an existing connection in current the Airbyte workspace. You can use a connection ID or name. - -| **Argument** | **Description** | -| ----------------- | -------------------- | -| `CONNECTION_ID` | The connection id. | -| `CONNECTION_NAME` | The connection name. | - -**Example**: - -```bash -$ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 -{ - "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", - "name": "Poke To PG", - "namespaceDefinition": "source", - "namespaceFormat": "${SOURCE_NAMESPACE}", - "prefix": "string", - "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", - "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", - "operationIds": [ - "18102e7c-5340-4000-8ef0-f35c05a49a01" - ], - "syncCatalog": { - "streams": [ - { - "stream": { - "name": "string", - "jsonSchema": {}, - "supportedSyncModes": [ - "full_refresh" - ], - "sourceDefinedCursor": false, - "defaultCursorField": [ - "string" - ], - "sourceDefinedPrimaryKey": [ - [ - "string" - ] - ], - "namespace": "string" - }, - "config": { - "syncMode": "full_refresh", - "cursorField": [ - "string" - ], - "destinationSyncMode": "append", - "primaryKey": [ - [ - "string" - ] - ], - "aliasName": "string", - "selected": false - } - } - ] - }, - "schedule": { - "units": 0, - "timeUnit": "minutes" - }, - "status": "active", - "resourceRequirements": { - "cpu_request": "string", - "cpu_limit": "string", - "memory_request": "string", - "memory_limit": "string" - }, - "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" -} -``` - -```bash -$ octavia get connection "Poke To PG" -{ - "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", - "name": "Poke To PG", - "namespaceDefinition": "source", - "namespaceFormat": "${SOURCE_NAMESPACE}", - "prefix": "string", - "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", - "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", - "operationIds": [ - "18102e7c-5340-4000-8ef0-f35c05a49a01" - ], - "syncCatalog": { - "streams": [ - { - "stream": { - "name": "string", - "jsonSchema": {}, - "supportedSyncModes": [ - "full_refresh" - ], - "sourceDefinedCursor": false, - "defaultCursorField": [ - "string" - ], - "sourceDefinedPrimaryKey": [ - [ - "string" - ] - ], - "namespace": "string" - }, - "config": { - "syncMode": "full_refresh", - "cursorField": [ - "string" - ], - "destinationSyncMode": "append", - "primaryKey": [ - [ - "string" - ] - ], - "aliasName": "string", - "selected": false - } - } - ] - }, - "schedule": { - "units": 0, - "timeUnit": "minutes" - }, - "status": "active", - "resourceRequirements": { - "cpu_request": "string", - "cpu_limit": "string", - "memory_request": "string", - "memory_limit": "string" - }, - "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" -} -``` - -#### `octavia import all` - -Import all existing resources (sources, destinations, connections) on your Airbyte instance to manage them with octavia-cli. - -**Examples**: - -```bash -$ octavia import all -🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace b06c6fbb-cadd-4c5c-bdbb-710add7dedb9. -✅ - Imported source poke in sources/poke/configuration.yaml. State stored in sources/poke/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml -⚠️ - Please update any secrets stored in sources/poke/configuration.yaml -✅ - Imported destination Postgres in destinations/postgres/configuration.yaml. State stored in destinations/postgres/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml -⚠️ - Please update any secrets stored in destinations/postgres/configuration.yaml -✅ - Imported connection poke-to-pg in connections/poke_to_pg/configuration.yaml. State stored in connections/poke_to_pg/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml -``` - -You know have local configuration files for all Airbyte resources that were already existing. -You need to edit any secret values that exist in these configuration files as secrets are not imported. -You can edit the configuration files and run `octavia apply` to continue managing them with octavia-cli. - -#### `octavia import destination or ` - -Import an existing destination to manage it with octavia-cli. You can use a destination ID or name. - -| **Argument** | **Description** | -| ------------------ | --------------------- | -| `DESTINATION_ID` | The destination id. | -| `DESTINATION_NAME` | The destination name. | - -#### `octavia import source or ` - -Import an existing source to manage it with octavia-cli. You can use a source ID or name. - -| **Argument** | **Description** | -| ------------- | ---------------- | -| `SOURCE_ID` | The source id. | -| `SOURCE_NAME` | The source name. | - -**Examples**: - -```bash -$ octavia import source poke -🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. -✅ - Imported source poke in sources/poke/configuration.yaml. State stored in sources/poke/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml -⚠️ - Please update any secrets stored in sources/poke/configuration.yaml -``` - -You know have local configuration file for an Airbyte source that was already existing. -You need to edit any secret value that exist in this configuration as secrets are not imported. -You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. - -#### `octavia import destination or ` - -Import an existing destination to manage it with octavia-cli. You can use a destination ID or name. - -| **Argument** | **Description** | -| ------------------ | --------------------- | -| `DESTINATION_ID` | The destination id. | -| `DESTINATION_NAME` | The destination name. | - -**Examples**: - -```bash -$ octavia import destination pg -🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. -✅ - Imported destination pg in destinations/pg/configuration.yaml. State stored in destinations/pg/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml -⚠️ - Please update any secrets stored in destinations/pg/configuration.yaml -``` - -You know have local configuration file for an Airbyte destination that was already existing. -You need to edit any secret value that exist in this configuration as secrets are not imported. -You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. - -#### `octavia import connection or ` - -Import an existing connection to manage it with octavia-cli. You can use a connection ID or name. - -| **Argument** | **Description** | -| ----------------- | -------------------- | -| `CONNECTION_ID` | The connection id. | -| `CONNECTION_NAME` | The connection name. | - -**Examples**: - -```bash -$ octavia import connection poke-to-pg -🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. -✅ - Imported connection poke-to-pg in connections/poke-to-pg/configuration.yaml. State stored in connections/poke-to-pg/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml -⚠️ - Please update any secrets stored in connections/poke-to-pg/configuration.yaml -``` - -You know have local configuration file for an Airbyte connection that was already existing. -**N.B.: You first need to import the source and destination used by the connection.** -You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. - -#### `octavia generate source ` - -Generate a YAML configuration for a source. -The YAML file will be stored at `./sources//configuration.yaml`. - -| **Argument** | **Description** | -| --------------- | --------------------------------------------------------------------------------------------- | -| `DEFINITION_ID` | The source connector definition id. Can be retrieved using `octavia list connectors sources`. | -| `SOURCE_NAME` | The name you want to give to this source in Airbyte. | - -**Example**: - -```bash -$ octavia generate source d8540a80-6120-485d-b7d6-272bca477d9b weather -✅ - Created the source template for weather in ./sources/weather/configuration.yaml. -``` - -#### `octavia generate destination ` - -Generate a YAML configuration for a destination. -The YAML file will be stored at `./destinations//configuration.yaml`. - -| **Argument** | **Description** | -| ------------------ | ------------------------------------------------------------------------------------------------------- | -| `DEFINITION_ID` | The destination connector definition id. Can be retrieved using `octavia list connectors destinations`. | -| `DESTINATION_NAME` | The name you want to give to this destination in Airbyte. | - -**Example**: - -```bash -$ octavia generate destination 25c5221d-dce2-4163-ade9-739ef790f503 my_db -✅ - Created the destination template for my_db in ./destinations/my_db/configuration.yaml. -``` - -#### `octavia generate connection --source --destination ` - -Generate a YAML configuration for a connection. -The YAML file will be stored at `./connections//configuration.yaml`. - -| **Option** | **Required** | **Description** | -| --------------- | ------------ | ------------------------------------------------------------------------------------------ | -| `--source` | Yes | Path to the YAML configuration file of the source you want to create a connection from. | -| `--destination` | Yes | Path to the YAML configuration file of the destination you want to create a connection to. | - -| **Argument** | **Description** | -| ----------------- | -------------------------------------------------------- | -| `CONNECTION_NAME` | The name you want to give to this connection in Airbyte. | - -**Example**: - -```bash -$ octavia generate connection --source sources/weather/configuration.yaml --destination destinations/my_db/configuration.yaml weather_to_pg -✅ - Created the connection template for weather_to_pg in ./connections/weather_to_pg/configuration.yaml. -``` - -#### `octavia apply` - -Create or update the resource on your Airbyte instance according to local configurations found in your octavia project directory. -If the resource was not found on your Airbyte instance, **apply** will **create** the remote resource. -If the resource was found on your Airbyte instance, **apply** will prompt you for validation of the changes and will run an **update** of your resource. -Please note that if a secret field was updated on your configuration, **apply** will run this change without prompt. - -| **Option** | **Required** | **Description** | -| ---------- | ------------ | ------------------------------------------------------------------ | -| `--file` | No | Path to the YAML configuration files you want to create or update. | -| `--force` | No | Run update without prompting for changes validation. | - -**Example**: - -```bash -$ octavia apply -🐙 - weather exists on your Airbyte instance, let's check if we need to update it! -👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed): - E - Value of root['lat'] changed from "46.7603" to "45.7603". -❓ - Do you want to update weather? [y/N]: y -✍️ - Running update because a diff was detected between local and remote resource. -🎉 - Successfully updated weather on your Airbyte instance! -💾 - New state for weather stored at ./sources/weather/state_.yaml. -🐙 - my_db exists on your Airbyte instance, let's check if we need to update it! -😴 - Did not update because no change detected. -🐙 - weather_to_pg exists on your Airbyte instance, let's check if we need to update it! -👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed): - E - Value of root['schedule']['timeUnit'] changed from "days" to "hours". -❓ - Do you want to update weather_to_pg? [y/N]: y -✍️ - Running update because a diff was detected between local and remote resource. -🎉 - Successfully updated weather_to_pg on your Airbyte instance! -💾 - New state for weather_to_pg stored at ./connections/weather_to_pg/state_.yaml. -``` - -## Contributing - -1. Please sign up to [Airbyte's Slack workspace](https://slack.airbyte.io/) and join the `#octavia-cli`. We'll sync up community efforts in this channel. -2. Pick an existing [GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) or **open** a new one to explain what you'd like to implement. -3. Assign the GitHub issue to yourself. -4. Fork Airbyte's repo, code and test thoroughly. -5. Open a PR on our Airbyte repo from your fork. - -### Developing locally - -0. Build the project locally (from the root of Airbyte's repo): `SUB_BUILD=OCTAVIA_CLI ./gradlew build # from the root directory of the repo`. -1. Install Python 3.8.12. We suggest doing it through `pyenv`. -2. Create a virtualenv: `python -m venv .venv`. -3. Activate the virtualenv: `source .venv/bin/activate`. -4. Install dev dependencies: `pip install -e .\[tests\]`. -5. Install `pre-commit` hooks: `pre-commit install`. -6. Run the unittest suite: `pytest --cov=octavia_cli`. Note, a local version of airbyte needs to be running (e.g. `docker compose up` from the root directory of the project) -7. Make sure the build passes (step 0) before opening a PR. - -## Telemetry - -This CLI has some telemetry tooling to send Airbyte some data about the usage of this tool. -We will use this data to improve the CLI and measure its adoption. -The telemetry sends data about: - -- Which command was run (not the arguments or options used). -- Success or failure of the command run and the error type (not the error payload). -- The current Airbyte workspace id if the user has not set the _anonymous data collection_ on their Airbyte instance. - -You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment variable to `False` or using the `--disable-telemetry` flag. diff --git a/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png b/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png new file mode 100644 index 000000000000..a9df65156a02 Binary files /dev/null and b/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png differ diff --git a/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png b/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png new file mode 100644 index 000000000000..a382786e3804 Binary files /dev/null and b/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png differ diff --git a/docs/cloud/managing-airbyte-cloud/configuring-connections.md b/docs/cloud/managing-airbyte-cloud/configuring-connections.md index bc896004eb30..269b68723ce6 100644 --- a/docs/cloud/managing-airbyte-cloud/configuring-connections.md +++ b/docs/cloud/managing-airbyte-cloud/configuring-connections.md @@ -1,4 +1,8 @@ -# Configuring connections +--- +products: all +--- + +# Configuring Connections A connection links a source to a destination and defines how your data will sync. After you have created a connection, you can modify any of the configuration settings or stream settings. @@ -24,11 +28,12 @@ You can configure the following settings: | Setting | Description | |--------------------------------------|-------------------------------------------------------------------------------------| -| [Replication frequency](/using-airbyte/core-concepts/sync-schedules.md) | How often the data syncs | -| [Destination namespace](/using-airbyte/core-concepts/namespaces.md) | Where the replicated data is written | -| Destination stream prefix | How you identify streams from different connectors | -| [Detect and propagate schema changes](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How Airbyte handles syncs when it detects schema changes in the source | -| [Connection Data Residency](/cloud/managing-airbyte-cloud/manage-data-residency.md) | Where data will be processed | +| Connection Name | A custom name for your connection | +| [Replication frequency](/using-airbyte/core-concepts/sync-schedules.md) | How often data syncs (can be scheduled, cron, API-triggered or manual) | +| [Destination namespace](/using-airbyte/core-concepts/namespaces.md) | Where the replicated data is written to in the destination | +| Destination stream prefix | A prefix added to each table name in the destination | +| [Detect and propagate schema changes](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How Airbyte handles schema changes in the source | +| [Connection Data Residency](/cloud/managing-airbyte-cloud/manage-data-residency.md) | Where data will be processed (Cloud only) | ## Modify streams in your connection @@ -76,9 +81,9 @@ Source-defined cursors and primary keys are selected automatically and cannot be 7. The **Stream configuration changed** dialog displays. This gives you the option to reset streams when you save the changes. -:::caution +:::tip -Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. +When editing the stream configuration, Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. ::: diff --git a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md index 777433ec72e3..f5822a2d28ed 100644 --- a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md +++ b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md @@ -1,6 +1,8 @@ -# Use the dbt Cloud integration +--- +products: cloud +--- - +# Use the dbt Cloud integration By using the dbt Cloud integration, you can create and run dbt transformations during syncs in Airbyte Cloud. This allows you to transform raw data into a format that is suitable for analysis and reporting, including cleaning and enriching the data. @@ -27,7 +29,7 @@ To set up the dbt Cloud integration in Airbyte Cloud: 1. In the Airbyte UI, click **Settings**. -2. Click **dbt Cloud integration**. +2. Click **Integrations**. 3. Paste the service token from [Step 1](#step-1-generate-a-service-token) and click **Save changes**. diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md index 2b39a0bb1893..b03e0d24d6e9 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md +++ b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Manage notifications This page provides guidance on how to manage notifications for Airbyte, allowing you to stay up-to-date on the activities in your workspace. diff --git a/docs/cloud/managing-airbyte-cloud/manage-connection-state.md b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md index 23d25db6be99..a745288fbe8c 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-connection-state.md +++ b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md @@ -1,4 +1,8 @@ -# Manage the connection state +--- +products: all +--- + +# Modifying connection state The connection state provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next sync. diff --git a/docs/cloud/managing-airbyte-cloud/manage-credits.md b/docs/cloud/managing-airbyte-cloud/manage-credits.md index ed54d783d6ae..8f04f6ffa788 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-credits.md +++ b/docs/cloud/managing-airbyte-cloud/manage-credits.md @@ -1,20 +1,22 @@ +--- +products: cloud +--- + # Manage credits - +Airbyte [credits](https://airbyte.com/pricing) are used to pay for Airbyte resources when you run a sync. You can purchase credits on Airbyte Cloud to keep your data flowing without interruption. ## Buy credits -Airbyte [credits](https://airbyte.com/pricing) are used to pay for Airbyte resources when you run a sync. You can purchase credits on Airbyte Cloud to keep your data flowing without interruption. - -To buy credits: +To purchase credits directly through the UI, -1. In the Airbyte UI, click **Billing** in the navigation bar. +1. Click **Billing** in the left-hand sidebar. 2. If you are unsure of how many credits you need, use our [Cost Estimator](https://www.airbyte.com/pricing) or click **Talk to Sales** to find the right amount for your team. 3. Click **Buy credits**. -4. The Stripe payment page displays. If you want to change the amount of credits, click the **Qty 200** dropdown. The **Update quantity** dialog displays, and you can either type the amount or use minus (**–**) or plus (**+**) to change the quantity. Click **Update**. +4. Determine the quantity of credits you intend to purchase. Adjust the **credit quantity**. When you're ready, click **Checkout**. :::note @@ -26,9 +28,7 @@ To buy credits: To buy more credits or discuss a custom plan, reach out to [Sales](https://airbyte.com/talk-to-sales). -5. Fill out the payment information. - - After you enter your billing address, sales tax (if applicable) is calculated and added to the total. +5. You'll be renavigated to a Stripe payment page. If this is your first time purchasing, you'll be asked for payment details. After you enter your billing address, sales tax (if applicable) is calculated and added to the total. 6. Click **Pay**. diff --git a/docs/cloud/managing-airbyte-cloud/manage-data-residency.md b/docs/cloud/managing-airbyte-cloud/manage-data-residency.md index 384d18337bb5..478c6c5862b5 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-data-residency.md +++ b/docs/cloud/managing-airbyte-cloud/manage-data-residency.md @@ -1,12 +1,14 @@ -# Manage data residency +--- +products: cloud +--- - +# Setting data residency -In Airbyte Cloud, you can set the default data residency and choose the data residency for individual connections, which can help you comply with data localization requirements. +In Airbyte Cloud, you can set the default data residency for your workspace and also set the the data residency for individual connections, which can help you comply with data localization requirements. -## Choose your default data residency +## Choose your workspace default data residency -Default data residency allows you to choose where your data is processed. Set the default data residency before creating a new source or connection so workflows that rely on the default data residency, such as fetching the schema or testing the source or destination, can process data in the correct region. +Setting a default data residency allows you to choose where your data is processed. Set the default data residency **before** creating a new source or connection so that subsequent workflows that rely on the default data residency, such as fetching the schema or testing the source or destination, can process data in the correct region. :::note @@ -14,7 +16,7 @@ While the data is processed in a data plane of the chosen residency, the cursor ::: -When you set the default data residency, it applies to new connections only. If you do not set the default data residency, the [Airbyte Default](configuring-connections.md) region is used. If you want to change the data residency for a connection, you can do so in its [connection settings](configuring-connections.md). +When you set the default data residency, it applies your preference to new connections only. If you do not adjust the default data residency, the [Airbyte Default](configuring-connections.md) region is used (United States). If you want to change the data residency for an individual connection, you can do so in its [connection settings](configuring-connections.md). To choose your default data residency: @@ -33,9 +35,9 @@ Depending on your network configuration, you may need to add [IP addresses](/ope ::: ## Choose the data residency for a connection -You can choose the data residency for your connection in the connection settings. You can also choose data residency when creating a new connection, or you can set the default data residency for your workspace. +You can additionally choose the data residency for your connection in the connection settings. You can choose the data residency when creating a new connection, or you can set the default data residency for your workspace so that it applies for any new connections moving forward. -To choose the data residency for your connection: +To choose a custom data residency for your connection: 1. In the Airbyte UI, click **Connections** and then click the connection that you want to change. diff --git a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md index 50fddf411ae2..5865c43f8a1e 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md +++ b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md @@ -1,4 +1,8 @@ -# Manage schema changes +--- +products: all +--- + +# Schema Change Management You can specify for each connection how Airbyte should handle any change of schema in the source. This process helps ensure accurate and efficient data syncs, minimizing errors and saving you time and effort in managing your data pipelines. diff --git a/docs/cloud/managing-airbyte-cloud/review-connection-status.md b/docs/cloud/managing-airbyte-cloud/review-connection-status.md index 5970e3756f4b..c93a94d3bb1d 100644 --- a/docs/cloud/managing-airbyte-cloud/review-connection-status.md +++ b/docs/cloud/managing-airbyte-cloud/review-connection-status.md @@ -1,6 +1,12 @@ +--- +products: all +--- + # Review the connection status The connection status displays information about the connection and of each stream being synced. Reviewing this summary allows you to assess the connection's current status and understand when the next sync will be run. +![Connection Status](./assets/connection-status-page.png) + To review the connection status: 1. In the Airbyte UI, click **Connections**. diff --git a/docs/cloud/managing-airbyte-cloud/review-sync-history.md b/docs/cloud/managing-airbyte-cloud/review-sync-history.md index c0fd7abd4d82..dae49ab3c7ac 100644 --- a/docs/cloud/managing-airbyte-cloud/review-sync-history.md +++ b/docs/cloud/managing-airbyte-cloud/review-sync-history.md @@ -1,7 +1,13 @@ +--- +products: all +--- + # Review the sync history The job history displays information about synced data, such as the amount of data moved, the number of records read and committed, and the total sync time. Reviewing this summary can help you monitor the sync performance and identify any potential issues. - + +![Job History](./assets/connection-job-history.png) + To review the sync history, click a connection in the list to view its sync history. Sync History displays the sync status or [reset](/operator-guides/reset.md) status. The sync status is defined as: | Status | Description | diff --git a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md index ceea1f200282..f9e4b5467a84 100644 --- a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md +++ b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md @@ -1,3 +1,7 @@ +--- +products: cloud +--- + # Airbyte Cloud limits Understanding the following limitations will help you more effectively manage Airbyte Cloud. @@ -8,16 +12,5 @@ Understanding the following limitations will help you more effectively manage Ai * Max number of streams that can be returned by a source in a discover call: 1K * Max number of streams that can be configured to sync in a single connection: 1K * Size of a single record: 20MB - * A flag can be set in order to log the PKs of the record that are skipped because of a size limit. In order to do that, -the following entry need to be added to the file `flag.yml`: -```yaml - - name: platform.print-long-record-pks - serve: true -``` - * It is possible to not fail the syncs and instead skip the records by adding the following entry to the file `flag.yml` -```yaml - - name: platform.fail-sync-if-too-big - serve: false -``` *Limits on workspaces, sources, and destinations do not apply to customers of [Powered by Airbyte](https://airbyte.com/solutions/powered-by-airbyte). To learn more [contact us](https://airbyte.com/talk-to-sales)! diff --git a/docs/community/getting-support.md b/docs/community/getting-support.md index 03b1ff795560..339bd08399c4 100644 --- a/docs/community/getting-support.md +++ b/docs/community/getting-support.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Getting Support Hold up! Have you looked at [our docs](https://docs.airbyte.com/) yet? We recommend searching the wealth of knowledge in our documentation as many times the answer you are looking for is there! @@ -18,13 +22,6 @@ If you require personalized support, reach out to our sales team to inquire abou We are driving our community support from our [forum](https://github.com/airbytehq/airbyte/discussions) on GitHub. -### Office Hour - -Airbyte provides a [Daily Office Hour](https://airbyte.com/daily-office-hour) to discuss issues. -It is a 45 minute meeting, the first 20 minutes are reserved to a weekly topic presentation about Airbyte concepts and the others 25 minutes are for general questions. The schedule is: -* Monday, Wednesday and Fridays: 1 PM PST/PDT -* Tuesday and Thursday: 4 PM CEST - ## Airbyte Cloud Support diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index d33f9d148df5..34795a73856c 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -120,7 +120,7 @@ The steps for updating an existing connector are the same as for building a new _Coming soon._ -Typing and Deduplication is how Airbyte transforms the raw data which is transmitted during a sync into easy-to-use final tables for database and data warehouse destinations. For more information on how typing and deduplication works, see [this doc](/understanding-airbyte/typing-deduping). +Typing and Deduplication is how Airbyte transforms the raw data which is transmitted during a sync into easy-to-use final tables for database and data warehouse destinations. For more information on how typing and deduplication works, see [this doc](/using-airbyte/core-concepts/typing-deduping). ## Publishing a connector diff --git a/docs/connector-development/cdk-python/README.md b/docs/connector-development/cdk-python/README.md index f0eb2387c8b0..76872e0186c6 100644 --- a/docs/connector-development/cdk-python/README.md +++ b/docs/connector-development/cdk-python/README.md @@ -1,44 +1,54 @@ # Connector Development Kit :::info -Over the next few months, the project will only accept connector contributions that are made using the [Low-Code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview) or the [Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview). +Over the next few months, the project will only accept connector contributions that are made using the +[Low-Code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview) or the +[Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview). -Contributions made with the Python CDK will be closed, but we will inquire to understand why it wasn't done with Low-Code/Connector Builder so we can address missing features. -This decision is aimed at improving maintenance and providing a larger catalog with high-quality connectors. +New pull requests made with the Python CDK will be closed, but we will inquire to understand why it wasn't done with +Low-Code/Connector Builder so we can address missing features. This decision is aimed at improving maintenance and +providing a larger catalog with high-quality connectors. You can continue to use the Python CDK to build connectors to help your company or projects. ::: :::info -Developer updates will be announced via our #help-connector-development Slack channel. If you are using the CDK, please join to stay up to date on changes and issues. +Developer updates will be announced via +[#help-connector-development](https://airbytehq.slack.com/archives/C027KKE4BCZ) Slack channel. If you are using the +CDK, please join to stay up to date on changes and issues. ::: :::info -This section is for the Python CDK. See our [community-maintained CDKs section](../README.md#community-maintained-cdks) -if you want to write connectors in other languages. +This section is for the Python CDK. See our +[community-maintained CDKs section](../README.md#community-maintained-cdks) if you want to write connectors in other +languages. ::: -The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently offers helpers specific for creating Airbyte source connectors for: + +The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently +offers helpers specific for creating Airbyte source connectors for: - HTTP APIs \(REST APIs, GraphQL, etc..\) - Generic Python sources \(anything not covered by the above\) -- Singer Taps (Note: The CDK supports building Singer taps but Airbyte no longer access contributions of this type) - -The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate. -This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. +This document is a general introduction to the CDK. Readers should have basic familiarity with the +[Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. -If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to [the Connector Development section of our Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) to inquire further! +If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to +[the Connector Development section of our Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) to +inquire further! ## Getting Started -Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run +Generate an empty connector using the code generator. First clone the Airbyte repository, then from the repository +root run -```text +```bash cd airbyte-integrations/connector-templates/generator ./generate.sh ``` -then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. +Next, find all `TODO`s in the generated project directory. They're accompanied by comments explaining what you'll +need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. Additionally, you can follow [this tutorial](../tutorials/cdk-tutorial-python-http/getting-started.md) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. @@ -68,7 +78,6 @@ You can find a complete tutorial for implementing an HTTP source connector in [t **HTTP Connectors**: -- [Exchangerates API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py) - [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) - [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) @@ -81,11 +90,11 @@ You can find a complete tutorial for implementing an HTTP source connector in [t ### First time setup -We assume `python` points to python >=3.9. +We assume `python` points to Python 3.9 or higher. Setup a virtual env: -```text +```bash python -m venv .venv source .venv/bin/activate pip install -e ".[tests]" # [tests] installs test-only dependencies @@ -102,7 +111,7 @@ pip install -e ".[tests]" # [tests] installs test-only dependencies While developing your connector, you can print detailed debug information during a sync by specifying the `--debug` flag. This allows you to get a better picture of what is happening during each step of your sync. -```text +```bash python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --debug ``` @@ -120,11 +129,3 @@ All tests are located in the `unit_tests` directory. Run `pytest --cov=airbyte_c 1. Open a PR 2. Once it is approved and merge, an Airbyte member must run the `Publish CDK Manually` workflow using `release-type=major|manor|patch` and setting the changelog message. - -## Coming Soon - -- Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) -- Airbyte Java HTTP CDK -- CDK for Async HTTP endpoints \(request-poll-wait style endpoints\) -- CDK for other protocols -- Don't see a feature you need? [Create an issue and let us know how we can help!](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement&template=feature-request.md&title=) diff --git a/docs/connector-development/cdk-python/http-streams.md b/docs/connector-development/cdk-python/http-streams.md index 010fdeddc418..ac4af4efe632 100644 --- a/docs/connector-development/cdk-python/http-streams.md +++ b/docs/connector-development/cdk-python/http-streams.md @@ -35,7 +35,7 @@ Using either authenticator is as simple as passing the created authenticator int ## Pagination -Most APIs, when facing a large call, tend to return the results in pages. The CDK accommodates paging via the `next_page_token` function. This function is meant to extract the next page "token" from the latest response. The contents of a "token" are completely up to the developer: it can be an ID, a page number, a partial URL etc.. The CDK will continue making requests as long as the `next_page_token` function. The CDK will continue making requests as long as the `next_page_token` continues returning non-`None` results. This can then be used in the `request_params` and other methods in `HttpStream` to page through API responses. Here is an [example](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L34) from the Stripe API. +Most APIs, when facing a large call, tend to return the results in pages. The CDK accommodates paging via the `next_page_token` function. This function is meant to extract the next page "token" from the latest response. The contents of a "token" are completely up to the developer: it can be an ID, a page number, a partial URL etc.. The CDK will continue making requests as long as the `next_page_token` continues returning non-`None` results. This can then be used in the `request_params` and other methods in `HttpStream` to page through API responses. Here is an [example](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L34) from the Stripe API. ## Rate Limiting diff --git a/docs/connector-development/config-based/tutorial/0-getting-started.md b/docs/connector-development/config-based/tutorial/0-getting-started.md index 5a8a940c2973..5a264a66c4a6 100644 --- a/docs/connector-development/config-based/tutorial/0-getting-started.md +++ b/docs/connector-development/config-based/tutorial/0-getting-started.md @@ -42,8 +42,8 @@ This can be done by signing up for the Free tier plan on [Exchange Rates Data AP - An Exchange Rates API key - Python >= 3.9 +- [Poetry](https://python-poetry.org/) - Docker must be running -- NodeJS - [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1) CLI ## Next Steps diff --git a/docs/connector-development/config-based/tutorial/2-install-dependencies.md b/docs/connector-development/config-based/tutorial/2-install-dependencies.md index 06bc75dc38d6..55520557fc37 100644 --- a/docs/connector-development/config-based/tutorial/2-install-dependencies.md +++ b/docs/connector-development/config-based/tutorial/2-install-dependencies.md @@ -1,17 +1,9 @@ # Step 2: Install dependencies -Let's create a python virtual environment for our source. -You can do this by executing the following commands from the root of the Airbyte repository. - -The command below assume that `python` points to a version of python >=3.9.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. -If this is the case on your machine, substitute the `python` commands with `python3`. -The subsequent `python` invocations will use the virtual environment created for the connector. ```bash cd ../../connectors/source-exchange-rates-tutorial -python -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt +poetry install ``` These steps create an initial python environment, and install the dependencies required to run an API Source connector. @@ -19,7 +11,7 @@ These steps create an initial python environment, and install the dependencies r Let's verify everything works as expected by running the Airbyte `spec` operation: ```bash -python main.py spec +poetry run source-exchange-rates-tutorial spec ``` You should see an output similar to the one below: diff --git a/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md b/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md index 0afbd220633f..752ccee58efb 100644 --- a/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md +++ b/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md @@ -200,7 +200,7 @@ spec: We can now run the `check` operation, which verifies the connector can connect to the API source. ```bash -python main.py check --config secrets/config.json +poetry run source-exchange-rates-tutorial check --config secrets/config.json ``` which should now succeed with logs similar to: diff --git a/docs/connector-development/config-based/tutorial/4-reading-data.md b/docs/connector-development/config-based/tutorial/4-reading-data.md index a6d892af79e2..677f5af4d9e8 100644 --- a/docs/connector-development/config-based/tutorial/4-reading-data.md +++ b/docs/connector-development/config-based/tutorial/4-reading-data.md @@ -44,7 +44,7 @@ As an alternative to storing the stream's data schema to the `schemas/` director Reading from the source can be done by running the `read` operation ```bash -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-exchange-rates-tutorial read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` The logs should show that 1 record was read from the stream. @@ -57,7 +57,7 @@ The logs should show that 1 record was read from the stream. The `--debug` flag can be set to print out debug information, including the outgoing request and its associated response ```bash -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json --debug +poetry run source-exchange-rates-tutorial read --config secrets/config.json --catalog integration_tests/configured_catalog.json --debug ``` ## Next steps diff --git a/docs/connector-development/config-based/tutorial/5-incremental-reads.md b/docs/connector-development/config-based/tutorial/5-incremental-reads.md index 51d1e24bc326..9cf2aac0c86f 100644 --- a/docs/connector-development/config-based/tutorial/5-incremental-reads.md +++ b/docs/connector-development/config-based/tutorial/5-incremental-reads.md @@ -76,7 +76,7 @@ definitions: You can test these changes by executing the `read` operation: ```bash -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-exchange-rates-tutorial read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` By reading the output record, you should see that we read historical data instead of the latest exchange rate. @@ -240,7 +240,7 @@ spec: Running the `read` operation will now read all data for all days between start_date and now: ```bash -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +poetry run source-exchange-rates-tutorial read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` The operation should now output more than one record: @@ -295,7 +295,7 @@ We can simulate incremental syncs by creating a state file containing the last s Running the `read` operation will now only read data for dates later than the given state: ```bash -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json --state integration_tests/sample_state.json +poetry run source-exchange-rates-tutorial read --config secrets/config.json --catalog integration_tests/configured_catalog.json --state integration_tests/sample_state.json ``` There shouldn't be any data read if the state is today's date: diff --git a/docs/connector-development/config-based/tutorial/6-testing.md b/docs/connector-development/config-based/tutorial/6-testing.md index 4bbb90e8ed01..284dfacc5b8c 100644 --- a/docs/connector-development/config-based/tutorial/6-testing.md +++ b/docs/connector-development/config-based/tutorial/6-testing.md @@ -30,7 +30,7 @@ and `integration_tests/abnormal_state.json` with You can run the [acceptance tests](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md#L1) with the following commands using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1): ```bash -airbyte-ci connectors --use-remote-secrets=false --name source-exchange-rates-tutorial test +airbyte-ci connectors --use-remote-secrets=false --name source-exchange-rates-tutorial test --only-step=acceptance ``` ## Next steps: diff --git a/docs/connector-development/connector-builder-ui/incremental-sync.md b/docs/connector-development/connector-builder-ui/incremental-sync.md index 0a4db2bc7a54..4b05f8d48ba9 100644 --- a/docs/connector-development/connector-builder-ui/incremental-sync.md +++ b/docs/connector-development/connector-builder-ui/incremental-sync.md @@ -82,10 +82,14 @@ Then when a sync is triggered for the same connection the next day, the followin curl 'https://content.guardianapis.com/search?from-date=2023-04-15T07:30:58Z&to-date={``}' +:::info +If the last record read has a datetime earlier than the end time of the stream interval, the end time of the interval will be stored in the state. +::: + The `from-date` is set to the cutoff date of articles synced already and the `to-date` is set to the current date. :::info -In some cases, it's helpful to reference the start and end date of the interval that's currently synced, for example if it needs to be injected into the URL path of the current stream. In these cases it can be referenced using the `{{ stream_interval.start_date }}` and `{{ stream_interval.end_date }}` [placeholders](/connector-development/config-based/understanding-the-yaml-file/reference#variables). Check out [the tutorial](./tutorial.mdx#adding-incremental-reads) for such a case. +In some cases, it's helpful to reference the start and end date of the interval that's currently synced, for example if it needs to be injected into the URL path of the current stream. In these cases it can be referenced using the `{{ stream_interval.start_time }}` and `{{ stream_interval.end_time }}` [placeholders](/connector-development/config-based/understanding-the-yaml-file/reference#variables). Check out [the tutorial](./tutorial.mdx#adding-incremental-reads) for such a case. ::: ## Incremental sync without time filtering diff --git a/docs/connector-development/connector-builder-ui/overview.md b/docs/connector-development/connector-builder-ui/overview.md index 81141aba9f61..2acccfb717c6 100644 --- a/docs/connector-development/connector-builder-ui/overview.md +++ b/docs/connector-development/connector-builder-ui/overview.md @@ -13,7 +13,7 @@ Developer updates will be announced via our #help-connector-development Slack ch The connector builder is the right tool if the following points are met: * You want to integrate with a JSON-based HTTP API as a source of records -* The API you want to integrate with doesn't exist yet as a connector in the [connector catalog](/category/sources). +* The API you want to integrate with doesn't exist yet as a connector in the [connector catalog](/integrations/sources/). * The API is suitable for the connector builder as per the [compatibility guide](./connector-builder-compatibility.md). diff --git a/docs/connector-development/connector-metadata-file.md b/docs/connector-development/connector-metadata-file.md index a20ac939a74a..1b9fed5380b6 100644 --- a/docs/connector-development/connector-metadata-file.md +++ b/docs/connector-development/connector-metadata-file.md @@ -125,3 +125,45 @@ releases: message: "This version changes the connector’s authentication by removing ApiKey authentication, which is now deprecated by the [upstream source](upsteam-docs-url.com). Users currently using ApiKey auth will need to reauthenticate with OAuth after upgrading to continue syncing." upgradeDeadline: "2023-12-31" ``` + +#### `scopedImpact` +The optional `scopedImpact` property allows you to provide a list of scopes for which the change is breaking. +This allows you to reduce the scope of the change; it's assumed that any scopes not listed are unaffected by the breaking change. + +For example, consider the following `scopedImpact` definition: + +```yaml +releases: + breakingChanges: + 1.0.0: + message: "This version changes the cursor for the `Users` stream. After upgrading, please reset the stream." + upgradeDeadline: "2023-12-31" + scopedImpact: + - scopeType: stream + impactedScopes: ["users"] +``` + +This change only breaks the `users` stream - all other streams are unaffected. A user can safely ignore the breaking change +if they are not syncing the `users` stream. + +The supported scope types are listed below. + +| Scope Type | Value Type | Value Description | +|------------|------------|------------------| +| stream | `list[str]` | List of stream names | + +#### `remoteRegistries` +The optional `remoteRegistries` property allows you to configure how a connector should be published to registries like Pypi. + +**Important note**: Currently no automated publishing will occur. + +```yaml +remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-connector-name +``` + +The `packageName` property of the `pypi` section is the name of the installable package in the PyPi registry. + +If not specified, all remote registry configurations are disabled by default. diff --git a/docs/connector-development/testing-connectors/README.md b/docs/connector-development/testing-connectors/README.md index 90842bccbccb..06377781fd21 100644 --- a/docs/connector-development/testing-connectors/README.md +++ b/docs/connector-development/testing-connectors/README.md @@ -1,34 +1,32 @@ # Testing Connectors -## Our testing pyramid -Multiple tests suites compose the Airbyte connector testing pyramid: -Connector specific tests declared in the connector code directory: -* Unit tests -* Integration tests - -Tests common to all connectors: -* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py) -* [Connector Acceptance tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) +Multiple tests suites compose the Airbyte connector testing pyramid -## Running tests -Unit and integration tests can be run directly from the connector code. +## Common to all connectors +* [Connectors QA checks](https://docs.airbyte.com/contributing-to-airbyte/resources/qa-checks) +* [Connector Acceptance tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) -Using `pytest` for Python connectors: +## Connector specific tests +### 🐍 Python connectors +We use `pytest` to run unit and integration tests: ```bash -python -m pytest unit_tests/ -python -m pytest integration_tests/ +# From connector directory +poetry run pytest ``` -Using `gradle` for Java connectors: +### ☕ Java connectors +We run Java connector tests with gradle. ```bash +# Unit tests ./gradlew :airbyte-integrations:connectors:source-postgres:test +# Integration tests ./gradlew :airbyte-integrations:connectors:source-postgres:integrationTestJava ``` Please note that according to the test implementation you might have to provide connector configurations as a `config.json` file in a `.secrets` folder in the connector code directory. - +## 🤖 CI If you want to run the global test suite, exactly like what is run in CI, you should install [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) and use the following command: ```bash @@ -39,6 +37,5 @@ This will run all the tests for the connector, including the QA checks and the C Connector Acceptance tests require connector configuration to be provided as a `config.json` file in a `.secrets` folder in the connector code directory. -## Tests on pull requests Our CI infrastructure runs the connector tests with [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). Connectors tests are automatically and remotely triggered on your branch according to the changes made in your branch. **Passing tests are required to merge a connector pull request.** diff --git a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md index 458c96a3ca12..5c38d69cc276 100644 --- a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md @@ -144,12 +144,15 @@ Additional tests are validating the backward compatibility of the current specif These backward compatibility tests can be bypassed by changing the value of the `backward_compatibility_tests_config.disable_for_version` input in `acceptance-test-config.yml` (see below). One more test validates the specification against containing exposed secrets. This means fields that potentially could hold a secret value should be explicitly marked with `"airbyte_secret": true`. If an input field like `api_key` / `password` / `client_secret` / etc. is exposed, the test will fail. -| Input | Type | Default | Note | -| :--------------------------------------------------------------- | :----- | :------------------ | :-------------------------------------------------------------------------------------------------------------------- | -| `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | -| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | -| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | -| `timeout_seconds` | int | 10 | Test execution timeout in seconds | +| Input | Type | Default | Note | +|:-----------------------------------------------------------------|:--------|:--------------------|:----------------------------------------------------------------------------------------------------------------------| +| `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | +| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | +| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | +| `timeout_seconds` | int | 10 | Test execution timeout in seconds | +| `auth_default_method` | object | None | Ensure that OAuth is default method, if OAuth uses by source | +| `auth_default_method.oauth` | boolean | True | Validate that OAuth is default method if set to True | +| `auth_default_method.bypass_reason` | string | | Reason why OAuth is not default method | ## Test Connection @@ -180,26 +183,32 @@ These backward compatibility tests can be bypassed by changing the value of the Configuring all streams in the input catalog to full refresh mode verifies that a read operation produces some RECORD messages. Each stream should have some data, if you can't guarantee this for particular streams - add them to the `empty_streams` list. Set `validate_data_points=True` if possible. This validation is going to be enabled by default and won't be configurable in future releases. -| Input | Type | Default | Note | -| :---------------------------------------- | :--------------- | :------------------------------------------ | :------------------------------------------------------------------------------------------------------------ | -| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | -| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | -| `empty_streams` | array of objects | \[\] | List of streams that might be empty with a `bypass_reason` | -| `empty_streams[0].name` | string | | Name of the empty stream | -| `empty_streams[0].bypass_reason` | string | None | Reason why this stream is empty | -| `ignored_fields[stream][0].name` | string | | Name of the ignored field | -| `ignored_fields[stream][0].bypass_reason` | string | None | Reason why this field is ignored | -| `validate_schema` | boolean | True | Verify that structure and types of records matches the schema from discovery command | -| `fail_on_extra_columns` | boolean | True | Fail schema validation if undeclared columns are found in records. Only relevant when `validate_schema=True` | -| `validate_data_points` | boolean | False | Validate that all fields in all streams contained at least one data point | -| `timeout_seconds` | int | 5\*60 | Test execution timeout in seconds | -| `expect_trace_message_on_failure` | boolean | True | Ensure that a trace message is emitted when the connector crashes | -| `expect_records` | object | None | Compare produced records with expected records, see details below | -| `expect_records.path` | string | | File with expected records | -| `expect_records.bypass_reason` | string | | Explain why this test is bypassed | -| `expect_records.extra_fields` | boolean | False | Allow output records to have other fields i.e: expected records are a subset | -| `expect_records.exact_order` | boolean | False | Ensure that records produced in exact same order | -| `expect_records.extra_records` | boolean | True | Allow connector to produce extra records, but still enforce all records from the expected file to be produced | +| Input | Type | Default | Note | +|:------------------------------------------------|:-----------------|:--------------------------------------------|:--------------------------------------------------------------------------------------------------------------| +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | +| `empty_streams` | array of objects | \[\] | List of streams that might be empty with a `bypass_reason` | +| `empty_streams[0].name` | string | | Name of the empty stream | +| `empty_streams[0].bypass_reason` | string | None | Reason why this stream is empty | +| `ignored_fields[stream][0].name` | string | | Name of the ignored field | +| `ignored_fields[stream][0].bypass_reason` | string | None | Reason why this field is ignored | +| `validate_schema` | boolean | True | Verify that structure and types of records matches the schema from discovery command | +| `fail_on_extra_columns` | boolean | True | Fail schema validation if undeclared columns are found in records. Only relevant when `validate_schema=True` | +| `validate_data_points` | boolean | False | Validate that all fields in all streams contained at least one data point | +| `timeout_seconds` | int | 5\*60 | Test execution timeout in seconds | +| `expect_trace_message_on_failure` | boolean | True | Ensure that a trace message is emitted when the connector crashes | +| `expect_records` | object | None | Compare produced records with expected records, see details below | +| `expect_records.path` | string | | File with expected records | +| `expect_records.bypass_reason` | string | | Explain why this test is bypassed | +| `expect_records.extra_fields` | boolean | False | Allow output records to have other fields i.e: expected records are a subset | +| `expect_records.exact_order` | boolean | False | Ensure that records produced in exact same order | +| `expect_records.extra_records` | boolean | True | Allow connector to produce extra records, but still enforce all records from the expected file to be produced | +| `file_types` | object | None | Configure file-based connectors specific tests | +| `file_types.skip_test` | boolean | False | Skip file-based connectors specific tests for the current config with a `bypass_reason` | +| `file_types.bypass_reason` | string | None | Reason why file-based connectors specific tests are skipped | +| `file_types.unsupported_types` | array of objects | None | Configure file types which are not supported by a source | +| `file_types.unsupported_types[0].extension` | string | | File type in `.csv` format which cannot be added to a test account | +| `file_types.unsupported_types[0].bypass_reason` | string | None | Reason why this file type cannot be added to a test account | `expect_records` is a nested configuration, if omitted - the part of the test responsible for record matching will be skipped. Due to the fact that we can't identify records without primary keys, only the following flag combinations are supported: @@ -280,6 +289,30 @@ This test verifies that sync produces no records when run with the STATE with ab | `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | | | `bypass_reason` | string | None | Explain why this test is bypassed | | +## Test Connector Attributes + +Verifies that certain properties of the connector and its streams guarantee a higher level of usability standards for certified connectors. +Some examples of the types of tests covered are verification that streams define primary keys, correct OAuth spec configuration, or a connector emits the correct stream status during a read. + +| Input | Type | Default | Note | +|:------------------------------------------|:-----------------|:----------------------|:-----------------------------------------------------------------------| +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `streams_without_primary_key` | array of objects | None | List of streams that do not support a primary key like reports streams | +| `streams_without_primary_key.name` | string | None | Name of the stream missing the PK | +| `streams_without_primary_key.bypass_reason` | string | None | The reason the stream doesn't have the PK | +| `allowed_hosts.bypass_reason` | object with `bypass_reason` | None | Defines the `bypass_reason` description about why the `allowedHosts` check for the certified connector should be skipped | +| `suggested_streams.bypass_reason` | object with `bypass_reason` | None | Defines the `bypass_reason` description about why the `suggestedStreams` check for the certified connector should be skipped | + +## Test Connector Documentation + +Verifies that connectors documentation follows our standard template, does have correct order of headings, +does not have missing headings and all required fields in Prerequisites section. + +| Input | Type | Default | Note | +|:------------------|:-------|:----------------------|:-------------------------------------------------------------------| +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | + ## Strictness level To enforce maximal coverage of acceptances tests we expose a `test_strictness_level` field at the root of the `acceptance-test-config.yml` configuration. diff --git a/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md b/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md deleted file mode 100644 index d8c9fd4af67c..000000000000 --- a/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md +++ /dev/null @@ -1,36 +0,0 @@ -# Testing A Custom Registry - -## Purpose of this document -This document describes how to -1. Modify the connector catalog used by the platform -2. Use the newly modified catalog in the platform - -## Why you might need to -1. You've added/updated/deleted a generally available connector and want to test it in the platform UI -1. You've added/updated/deleted a generally available connector and want to test it in the platform API - -## Method 1: Edit the registry by hand (easiest) - -### 1. Download the current OSS Registry -Download the current registry from [here](https://connectors.airbyte.com/files/registries/v0/oss_registry.json) to somewhere on your local machine. - -### 2. Modify the registry -Modify the registry as you see fit. For example, you can add a new connector, update an existing connector, or delete a connector. - -### 3. Upload the modified registry to a public location -Upload the modified registry to a public location. For example, you can upload it to a public S3 bucket, or you can upload it to a public GitHub repo, or a service like file.io - -### 4. Point the platform to the modified registry -Run the platform with the following environment variable set: -``` -REMOTE_CONNECTOR_CATALOG_URL = -``` - -## Method 2: Use the registry generator (more involved) - -Follow the steps in the [Metadata Orchestrator Readme](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/metadata_service/orchestrator/README.md)) to setup the orchestrator. - -You can then use the public GCS url of the registry created by the orchestrator to point the platform to the modified registry. -``` -REMOTE_CONNECTOR_CATALOG_URL = -``` \ No newline at end of file diff --git a/docs/connector-development/tutorials/adding-incremental-sync.md b/docs/connector-development/tutorials/adding-incremental-sync.md index b463503a795b..8a454049a7dd 100644 --- a/docs/connector-development/tutorials/adding-incremental-sync.md +++ b/docs/connector-development/tutorials/adding-incremental-sync.md @@ -2,7 +2,7 @@ ## Overview -This tutorial will assume that you already have a working source. If you do not, feel free to refer to the [Building a Toy Connector](building-a-python-source.md) tutorial. This tutorial will build directly off the example from that article. We will also assume that you have a basic understanding of how Airbyte's Incremental-Append replication strategy works. We have a brief explanation of it [here](/using-airbyte/core-concepts/sync-modes/incremental-append.md). +This tutorial will assume that you already have a working source. If you do not, feel free to refer to the [Building a Toy Connector](build-a-connector-the-hard-way.md) tutorial. This tutorial will build directly off the example from that article. We will also assume that you have a basic understanding of how Airbyte's Incremental-Append replication strategy works. We have a brief explanation of it [here](/using-airbyte/core-concepts/sync-modes/incremental-append.md). ## Update Catalog in `discover` diff --git a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md index 9fb9a71aac70..5f9edd2d0d58 100644 --- a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md +++ b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md @@ -4,46 +4,38 @@ description: Building a source connector without using any helpers to learn the # Building a Source Connector: The Hard Way -This tutorial walks you through building a simple Airbyte source without using any helpers to demonstrate the following concepts in Action: +This tutorial walks you through building a simple Airbyte source without using any helpers to demonstrate the following concepts in action: - [The Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md) and the interface implemented by a source connector - [The AirbyteCatalog](../../understanding-airbyte/beginners-guide-to-catalog.md) - [Packaging your connector](https://docs.airbyte.com/connector-development#1.-implement-and-package-the-connector) - [Testing your connector](../testing-connectors/connector-acceptance-tests-reference.md) -At the end of this tutorial, you will have a working source that you will be able to use in the Airbyte UI. +:::warning +**This tutorial is meant for those interested in learning how the Airbyte Specification works in detail, +not for creating production connectors**. +If you're building a real source, you should start with using the [Connector Builder](../connector-builder-ui/overview), or +the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials). +::: -**This tutorial is meant for those interested in learning how the Airbyte Specification works in detail, not for creating production connectors**. We intentionally don't use helper libraries provided by Airbyte so that this tutorial is self-contained. If you were building a "real" source, you'll want to use the helper modules such as the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials). - -This tutorial can be done entirely on your local workstation. - -### Requirements +## Requirements To run this tutorial, you'll need: - Docker, Python, and Java with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md). - The `requests` Python package installed via `pip install requests` \(or `pip3` if `pip` is linked to a Python2 installation on your system\) -**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.9 or greater. Verify this by running - -```bash -$ python --version -Python 3.9.11 -``` - -On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3` . Otherwise, make sure to install Python 3 before beginning. - -You need also to install `requests` python library: - -```bash -pip install requests -``` - ## Our connector: a stock ticker API -Our connector will output the daily price of a stock since a given date. We'll leverage the free [Polygon.io API](https://polygon.io/pricing) for this. We'll use Python to implement the connector because its syntax is accessible to most programmers, but the process described here can be applied to any language. +The connector will output the daily price of a stock since a given date. +We'll leverage [Polygon.io API](https://polygon.io/) for this. + +:::info +We'll use Python to implement the connector, but you could build an Airbyte +connector in any language. +::: -Here's the outline of what we'll do to build our connector: +Here's the outline of what we'll do to build the connector: 1. Use the Airbyte connector template to bootstrap the connector package 2. Implement the methods required by the Airbyte Specification for our connector: @@ -52,30 +44,27 @@ Here's the outline of what we'll do to build our connector: 3. `discover`: declares the different streams of data that this connector can output 4. `read`: reads data from the underlying data source \(The stock ticker API\) 3. Package the connector in a Docker image -4. Test the connector using Airbyte's Standard Test Suite +4. Test the connector using Airbyte's Connector Acceptance Test Suite 5. Use the connector to create a new Connection and run a sync in Airbyte UI -Once we've completed the above steps, we will have built a functioning connector. Then, we'll add some optional functionality: +[Part 2 of this article](adding-incremental-sync.md) covers: -- Support [incremental sync](/using-airbyte/core-concepts/sync-modes/incremental-append.md) +- Support [incremental sync](../../using-airbyte/core-concepts/sync-modes/incremental-append.md) - Add custom integration tests +Let's get started! + +--- + ### 1. Bootstrap the connector package -We'll start the process from the Airbyte repository root: +Start the process from the Airbyte repository root: ```bash $ pwd /Users/sherifnada/code/airbyte ``` -First, let's create a new branch: - -```bash -$ git checkout -b $(whoami)/source-connector-tutorial -Switched to a new branch 'sherifnada/source-connector-tutorial' -``` - Airbyte provides a code generator which bootstraps the scaffolding for our connector. Let's use it by running: ```bash @@ -83,22 +72,23 @@ $ cd airbyte-integrations/connector-templates/generator $ ./generate.sh ``` -We'll select the `generic` template and call the connector `stock-ticker-api`: +Select the `Generic Source` template and call the connector `stock-ticker-api`: ![](../../.gitbook/assets/newsourcetutorial_plop.gif) -Note: The generic template is very bare. If you are planning on developing a Python source, we recommend using the `python` template. It provides some convenience code to help reduce boilerplate. This tutorial uses the bare-bones version because it makes it easier to see how all the pieces of a connector work together. You can find a walk through on how to build a Python connector here \(**coming soon**\). +:::info +This tutorial uses the bare-bones `Generic Source` template to illustrate how all the pieces of a connector +work together. For real connectors, the generator provides `Python` and `Python HTTP API` source templates, they use +[Airbyte CDK](../cdk-python/README.md). +::: -Head to the connector directory and we should see the following files have been generated: ```bash $ cd ../../connectors/source-stock-ticker-api $ ls -Dockerfile README.md acceptance-test-config.yml acceptance-test-docker.sh build.gradle +Dockerfile README.md acceptance-test-config.yml metadata.yaml ``` -We'll use each of these files later. But first, let's write some code! - ### 2. Implement the connector in line with the Airbyte Specification In the connector package directory, create a single Python file `source.py` that will hold our implementation: @@ -109,17 +99,18 @@ touch source.py #### Implement the spec operation -At this stage in the tutorial, we just want to implement the `spec` operation as described in the [Airbyte Protocol](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#spec). This involves a couple of steps: - -1. Decide which inputs we need from the user in order to connect to the stock ticker API \(i.e: the connector's specification\) and encode it as a JSON file. -2. Identify when the connector has been invoked with the `spec` operation and return the specification as an `AirbyteMessage` +The `spec` operation is described in the [Airbyte Protocol](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#spec). +It's a way for the connector to tell Airbyte what user inputs it needs in order to connecto to the source (the stock +ticker API in our case). Airbyte expects the command to output a connector specification in `AirbyteMessage` format. To contact the stock ticker API, we need two things: 1. Which stock ticker we're interested in 2. The API key to use when contacting the API \(you can obtain a free API token from [Polygon.io](https://polygon.io/dashboard/signup) free plan\) +:::info For reference, the API docs we'll be using [can be found here](https://polygon.io/docs/stocks/get_v2_aggs_ticker__stocksticker__range__multiplier___timespan___from___to). +::: Let's create a [JSONSchema](http://json-schema.org/) file `spec.json` encoding these two requirements: @@ -153,16 +144,14 @@ Let's create a [JSONSchema](http://json-schema.org/) file `spec.json` encoding t - `description` will be shown in the Airbyte UI under each field to help the user understand it - `airbyte_secret` used by Airbyte to determine if the field should be displayed as a password \(e.g: `********`\) in the UI and not readable from the API -We'll save this file in the root directory of our connector. Now we have the following files: ```bash $ ls -1 Dockerfile README.md acceptance-test-config.yml -acceptance-test-docker.sh -build.gradle source.py +metadata.yaml spec.json ``` @@ -174,6 +163,7 @@ import argparse # helps parse commandline arguments import json import sys import os +from datetime import datetime def read_json(filepath): @@ -186,6 +176,12 @@ def log(message): print(json.dumps(log_json)) +def log_error(error_message): + current_time_in_ms = int(datetime.now().timestamp()) * 1000 + log_json = {"type": "TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}} + print(json.dumps(log_json)) + + def spec(): # Read the file named spec.json from the module directory as a JSON file current_script_directory = os.path.dirname(os.path.realpath(__file__)) @@ -232,9 +228,13 @@ if __name__ == "__main__": Some notes on the above code: -1. As described in the [specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#key-takeaways), Airbyte connectors are CLIs which communicate via stdout, so the output of the command is simply a JSON string formatted according to the Airbyte Specification. So to "return" a value we use `print` to output the return value to stdout -2. All Airbyte commands can output log messages that take the form `{"type":"LOG", "log":"message"}`, so we create a helper method `log(message)` to allow logging -3. All Airbyte commands can output error messages that take the form `{"type":"TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}}}`, so we create a helper method `log_error(message)` to allow error messages +1. As described in the [specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#key-takeaways), + Airbyte connectors are CLIs which communicate via stdout, so the output of the command is simply a JSON string + formatted according to the Airbyte Specification. So to "return" a value we use `print` to output the return value to stdout. +2. All Airbyte commands can output log messages that take the form `{"type":"LOG", "log":"message"}`, so we create a helper method `log(message)` to allow logging. +3. All Airbyte commands can output error messages that take the form + `{"type":"TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}}}`, + so we create a helper method `log_error(message)` to allow error messages. Now if we run `python source.py spec` we should see the specification printed out: @@ -243,17 +243,19 @@ python source.py spec {"type": "SPEC", "spec": {"documentationUrl": "https://polygon.io/docs/stocks/get_v2_aggs_ticker__stocksticker__range__multiplier___timespan___from___to", "connectionSpecification": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "required": ["stock_ticker", "api_key"], "properties": {"stock_ticker": {"type": "string", "title": "Stock Ticker", "description": "The stock ticker to track", "examples": ["AAPL", "TSLA", "AMZN"]}, "api_key": {"type": "string", "description": "The Polygon.io Stocks API key to use to hit the API.", "airbyte_secret": true}}}}} ``` -We've implemented the first command! Three more and we'll have a working connector. - #### Implementing check connection -The second command to implement is the [check operation](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#check) `check --config `, which tells the user whether a config file they gave us is correct. In our case, "correct" means they input a valid stock ticker and a correct API key like we declare via the `spec` operation. +The second command to implement is the [check operation](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#check) `check --config `, +which tells the user whether a config file they gave us is correct. In our case, "correct" means they input a valid +stock ticker and a correct API key like we declare via the `spec` operation. To achieve this, we'll: -1. Create valid and invalid configuration files to test the success and failure cases with our connector. We'll place config files in the `secrets/` directory which is gitignored everywhere in the Airbyte monorepo by default to avoid accidentally checking in API keys -2. Add a `check` method which calls the Polygon.io API to verify if the provided token & stock ticker are correct and output the correct airbyte message -3. Extend the argument parser to recognize the `check --config ` command and call the `check` method when the `check` command is invoked +1. Create valid and invalid configuration files to test the success and failure cases with our connector. + We'll place config files in the `secrets/` directory which is gitignored everywhere in the Airbyte monorepo by + default to avoid accidentally checking in API keys. +2. Add a `check` method which calls the Polygon.io API to verify if the provided token & stock ticker are correct and output the correct airbyte message. +3. Extend the argument parser to recognize the `check --config ` command and call the `check` method when the `check` command is invoked. Let's first add the configuration files: @@ -265,7 +267,7 @@ $ echo '{"api_key": "not_a_real_key", "stock_ticker": "TSLA"}' > secrets/invalid Make sure to add your actual API key instead of the placeholder value `` when following the tutorial. -Then we'll add the `check_method`: +Then we'll add the `check` method: ```python import requests @@ -295,7 +297,8 @@ def check(config): print(json.dumps(output_message)) ``` -Lastly we'll extend the `run` method to accept the `check` command and call the `check` method. First we'll add a helper method for reading input: +In Airbyte, the contract for input files is that they will be available in the current working directory if they are not provided as an absolute path. +This method helps us achieve that: ```python def get_input_file_path(path): @@ -305,33 +308,7 @@ def get_input_file_path(path): return os.path.join(os.getcwd(), path) ``` -In Airbyte, the contract for input files is that they will be available in the current working directory if they are not provided as an absolute path. This method helps us achieve that. - -We also need to extend the arguments parser by adding the following two blocks to the `run` method: - -```python - # Accept the check command - check_parser = subparsers.add_parser("check", help="checks the config used to connect", parents=[parent_parser]) - required_check_parser = check_parser.add_argument_group("required named arguments") - required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") -``` - -and - -```python -elif command == "check": - config_file_path = get_input_file_path(parsed_args.config) - config = read_json(config_file_path) - check(config) -``` - -Then we need to update our list of available commands: - -```python - log("Invalid command. Allowable commands: [spec, check]") -``` - -This results in the following `run` method. +We'll then add the `check` command support to `run`: ```python def run(args): @@ -366,8 +343,6 @@ def run(args): sys.exit(0) ``` -and that should be it. - Let's test our new method: ```bash @@ -383,7 +358,8 @@ Our connector is able to detect valid and invalid configs correctly. Two methods The `discover` command outputs a Catalog, a struct that declares the Streams and Fields \(Airbyte's equivalents of tables and columns\) output by the connector. It also includes metadata around which features a connector supports \(e.g. which sync modes\). In other words it describes what data is available in the source. If you'd like to read a bit more about this concept check out our [Beginner's Guide to the Airbyte Catalog](../../understanding-airbyte/beginners-guide-to-catalog.md) or for a more detailed treatment read the [Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md). -The data output by this connector will be structured in a very simple way. This connector outputs records belonging to exactly one Stream \(table\). Each record contains three Fields \(columns\): `date`, `price`, and `stock_ticker`, corresponding to the price of a stock on a given day. +The stock ticker connector outputs records belonging to exactly one Stream \(table\). +Each record contains three Fields \(columns\): `date`, `price`, and `stock_ticker`, corresponding to the price of a stock on a given day. To implement `discover`, we'll: @@ -440,8 +416,9 @@ We need to update our list of available commands: ```python log("Invalid command. Allowable commands: [spec, check, discover]") ``` - +:::info You may be wondering why `config` is a required input to `discover` if it's not used. This is done for consistency: the Airbyte Specification requires `--config` as an input to `discover` because many sources require it \(e.g: to discover the tables available in a Postgres database, you must supply a password\). So instead of guessing whether the flag is required depending on the connector, we always assume it is required, and the connector can choose whether to use it. +::: The full run method is now below: @@ -504,10 +481,11 @@ python source.py read --config --catalog --use-remote-secrets=false test ``` -After tests have run, you should see a test summary like: - -```text -collecting ... - test_core.py ✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓ 95% █████████▌ - test_full_refresh.py ✓ 100% ██████████ - -================== short test summary info ================== -SKIPPED [1] connector_acceptance_test/plugin.py:56: Skipping TestIncremental.test_two_sequential_reads because not found in the config - -Results (8.91s): - 20 passed -``` +`airbyte-ci` will build and then test your connector, and provide a report on the test results. That's it! We've created a fully functioning connector. Now let's get to the exciting part: using it from the Airbyte UI. +--- + ### Use the connector in the Airbyte UI Let's recap what we've achieved so far: 1. Implemented a connector 2. Packaged it in a Docker image -3. Integrated it with the Airbyte Standard Test suite +3. Ran Connector Acceptance Tests for the connector with `airbyte-ci` To use it from the Airbyte UI, we need to: @@ -1056,7 +998,8 @@ To use it from the Airbyte UI, we need to: #### 1. Publish the Docker image -Since we're running this tutorial locally, Airbyte will have access to any Docker images available to your local `docker` daemon. So all we need to do is build & tag our connector. If you want your connector to be available to everyone using Airbyte, you'll need to publish it to `Dockerhub`. [Open a PR](https://github.com/airbytehq/airbyte) or visit our [Slack](https://slack.airbyte.io) for help with this. +Since we're running this tutorial locally, Airbyte will have access to any Docker images available to your local `docker` daemon. So all we need to do is build & tag our connector. +For real production connectors to be available on Airbyte Cloud, you'd need to publish them on DockerHub. Airbyte's build system builds and tags your connector's image correctly by default as part of the connector's standard `build` process. **From the Airbyte repo root**, run: @@ -1191,25 +1134,12 @@ $ cat /tmp/airbyte_local/tutorial_json/_airbyte_raw_stock_prices.jsonl Congratulations! We've successfully written a fully functioning Airbyte connector. You're an Airbyte contributor now ;\) -Armed with the knowledge you gained in this guide, here are some places you can go from here: - -1. Implement Incremental Sync for your connector \(described in the sections below\) -2. Implement another connector using the language specific helpers listed below -3. While not required, we love contributions! if you end up creating a new connector, we're here to help you make it available to everyone using Airbyte. Remember that you're never expected to maintain a connector by yourself if you merge it to Airbyte -- we're committed to supporting connectors if you can't do it yourself - -## Optional additions - -This section is not yet complete and will be completed soon. Please reach out to us on [Slack](https://slack.airbyte.io) or [Github](https://github.com/airbytehq/airbyte) if you need the information promised by these sections immediately. - -### Incremental sync - -Follow the [next tutorial](adding-incremental-sync.md) to implement incremental sync. - -### Connector Development Kit - -Like we mention at the beginning of the tutorial, this guide is meant more for understanding than as a blueprint for implementing production connectors. See the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials) for the frameworks you should use to build production-ready connectors. +1. Follow the [next tutorial](adding-incremental-sync.md) to implement incremental sync. +2. Implement another connector using the Low-code CDK, [Connector Builder](../connector-builder-ui/overview), or [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials) +3. We welcome low-code configuration based connector contributions! If you make a connector in the connector builder + and want to share it with everyone using Airbyte, pull requests are welcome! -### Language specific helpers +## Additional guides - [Building a Python Source](https://docs.airbyte.com/connector-development/tutorials/building-a-python-source) - [Building a Java Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-java-destination) diff --git a/docs/connector-development/tutorials/building-a-python-source.md b/docs/connector-development/tutorials/building-a-python-source.md index ce2a66cce0cc..49a12872363b 100644 --- a/docs/connector-development/tutorials/building-a-python-source.md +++ b/docs/connector-development/tutorials/building-a-python-source.md @@ -25,22 +25,18 @@ All the commands below assume that `python` points to a version of python >3. * Step 5: Implement `check` * Step 6: Implement `discover` * Step 7: Implement `read` -* Step 8: Set up Standard Tests +* Step 8: Set up Connector Acceptance Tests * Step 9: Write unit tests or integration tests * Step 10: Update the `README.md` \(If API credentials are required to run the integration, please document how they can be obtained or link to a how-to guide.\) * Step 11: Update the `metadata.yaml` file with accurate information about your connector. These metadata will be used to add the connector to Airbyte's connector registry. * Step 12: Add docs \(in `docs/integrations/sources/.md`\) :::info - Each step of the Creating a Source checklist is explained in more detail below. - ::: :::info - All `./gradlew` commands must be run from the root of the airbyte project. - ::: ### Submitting a Source to Airbyte @@ -52,9 +48,7 @@ All `./gradlew` commands must be run from the root of the airbyte project. * Edit the `airbyte/tools/bin/ci_credentials.sh` script to pull the script from the build environment and write it to `secrets/config.json` during the build. :::info - -If you have a question about a step the Submitting a Source to Airbyte checklist include it in your PR or ask it on [slack](https://slack.airbyte.io). - +If you have a question about a step the Submitting a Source to Airbyte checklist include it in your PR or ask it on [#help-connector-development channel on Slack](https://airbytehq.slack.com/archives/C027KKE4BCZ). ::: ## Explaining Each Step @@ -70,18 +64,16 @@ $ ./generate.sh Select the `python` template and then input the name of your connector. For this walk through we will refer to our source as `example-python` -### Step 2: Build the newly generated source +### Step 2: Install the newly generated source -Build the source by running: +Install the source by running: -```text +```bash cd airbyte-integrations/connectors/source- -python -m venv .venv # Create a virtual environment in the .venv directory -source .venv/bin/activate # enable the venv -pip install -r requirements.txt +poetry install ``` -This step sets up the initial python environment. **All** subsequent `python` or `pip` commands assume you have activated your virtual environment. +This step sets up the initial python environment. ### Step 3: Set up your Airbyte development environment @@ -105,9 +97,7 @@ The commands we ran above created a virtual environment for your source. If you Pretty much all it takes to create a source is to implement the `Source` interface. The template fills in a lot of information for you and has extensive docstrings describing what you need to do to implement each method. The next 4 steps are just implementing that interface. :::info - All logging should be done through the `logger` object passed into each method. Otherwise, logs will not be shown in the Airbyte UI. - ::: #### Iterating on your implementation @@ -118,18 +108,18 @@ Everyone develops differently but here are 3 ways that we recommend iterating on You'll notice in your source's directory that there is a python file called `main.py`. This file exists as convenience for development. You can call it from within the virtual environment mentioned above `. ./.venv/bin/activate` to test out that your source works. -```text +```bash # from airbyte-integrations/connectors/source- -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source- spec +poetry run source- check --config secrets/config.json +poetry run source- discover --config secrets/config.json +poetry run source- read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` The nice thing about this approach is that you can iterate completely within in python. The downside is that you are not quite running your source as it will actually be run by Airbyte. Specifically you're not running it from within the docker container that will house it. -** Build the source docker image** +**Build the source docker image** You have to build a docker image for your connector if you want to run your source exactly as it will be run by Airbyte. @@ -150,6 +140,7 @@ Once the command is done, you will find your connector image in your local docke If you don't want to rely on `airbyte-ci` to build your connector, you can build the docker image using your own Dockerfile. This method is not preferred, and is not supported for certified connectors. Create a `Dockerfile` in the root of your connector directory. The `Dockerfile` should look something like this: + ```Dockerfile FROM airbyte/python-connector-base:1.1.0 @@ -171,25 +162,29 @@ docker build . -t airbyte/source-example-python:dev **Run the source docker image** -``` +```bash docker run --rm airbyte/source-example-python:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-example-python:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` -Note: Each time you make a change to your implementation you need to re-build the connector image. This ensures the new python code is added into the docker container. +:::info +Each time you make a change to your implementation you need to re-build the connector image. This ensures the new python code is added into the docker container. +::: The nice thing about this approach is that you are running your source exactly as it will be run by Airbyte. The tradeoff is that iteration is slightly slower, because you need to re-build the connector between each change. **Detailed Debug Messages** During development of your connector, you can enable the printing of detailed debug information during a sync by specifying the `--debug` flag. This will allow you to get a better picture of what is happening during each step of your sync. -```text -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --debug + +```bash +poetry run source- read --config secrets/config.json --catalog sample_files/configured_catalog.json --debug ``` In addition to the preset CDK debug statements, you can also emit custom debug information from your connector by introducing your own debug statements: + ```python self.logger.debug( "your debug message here", @@ -200,9 +195,9 @@ self.logger.debug( ) ``` -**TDD using standard tests** +**TDD using acceptance tests & integration tests** -Airbyte provides a standard test suite that is run against every source. The objective of these tests is to provide some "free" tests that can sanity check that the basic functionality of the source works. One approach to developing your connector is to simply run the tests between each change and use the feedback from them to guide your development. +Airbyte provides an acceptance test suite that is run against every source. The objective of these tests is to provide some "free" tests that can sanity check that the basic functionality of the source works. One approach to developing your connector is to simply run the tests between each change and use the feedback from them to guide your development. If you want to try out this approach, check out Step 8 which describes what you need to do to set up the standard tests for your source. @@ -232,33 +227,32 @@ For a brief overview on the catalog check out [Beginner's Guide to the Airbyte C As described in the template code, this method takes in the same config object as the previous methods. It also takes in a "configured catalog". This object wraps the catalog emitted by the `discover` step and includes configuration on how the data should be replicated. For a brief overview on the configured catalog check out [Beginner's Guide to the Airbyte Catalog](../../understanding-airbyte/beginners-guide-to-catalog.md). It then returns a generator which returns each record in the stream. -### Step 8: Set up Standard Tests +### Step 8: Set up Connector Acceptance Tests (CATs) -The Standard Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../testing-connectors/connector-acceptance-tests-reference.md) explains Standard Tests and how to run them. +The Connector Acceptance Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../testing-connectors/connector-acceptance-tests-reference.md) explains Connector Acceptance Tests and how to run them. -You can run the tests using `./gradlew :airbyte-integrations:connectors:source-:integrationTest`. Make sure to run this command from the Airbyte repository root. +You can run the tests using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +`airbyte-ci connectors --name source- test --only-step=acceptance` :::info - In some rare cases we make exceptions and allow a source to not need to pass all the standard tests. If for some reason you think your source cannot reasonably pass one of the tests cases, reach out to us on github or slack, and we can determine whether there's a change we can make so that the test will pass or if we should skip that test for your source. - ::: ### Step 9: Write unit tests and/or integration tests -The Standard Tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. In case you need to test additional functionality of your source, write unit or integration tests. +The connector acceptance tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. In case you need to test additional functionality of your source, write unit or integration tests. #### Unit Tests -Add any relevant unit tests to the `unit_tests` directory. Unit tests should _not_ depend on any secrets. +Add any relevant unit tests to the `tests/unit_tests` directory. Unit tests should _not_ depend on any secrets. -You can run the tests using `python -m pytest -s unit_tests` +You can run the tests using `poetry run pytest tests/unit_tests` #### Integration Tests Place any integration tests in the `integration_tests` directory such that they can be [discovered by pytest](https://docs.pytest.org/en/6.2.x/goodpractices.html#conventions-for-python-test-discovery). -Run integration tests using `python -m pytest -s integration_tests`. +You can run the tests using `poetry run pytest tests/integration_tests` ### Step 10: Update the `README.md` @@ -272,7 +266,7 @@ If you are self hosting Airbyte (OSS) you are able to use the Custom Connector f If you are using Airbyte Cloud (or OSS), you can submit a PR to add your connector to the Airbyte repository. Once the PR is merged, the connector will be available to all Airbyte Cloud users. You can read more about it [here](https://docs.airbyte.com/contributing-to-airbyte/submit-new-connector). Note that when submitting an Airbyte connector, you will need to ensure that -1. The connector passes the standard test suite. See [Set up Standard Tests](#step-8-set-up-standard-tests). +1. The connector passes the CAT suite. See [Set up Connector Acceptance Tests](#step-8-set-up-connector-acceptance-tests-\(cats\)). 2. The metadata.yaml file (created by our generator) is filed out and valid. See [Connector Metadata File](https://docs.airbyte.com/connector-development/connector-metadata-file). 3. You have created appropriate documentation for the connector. See [Add docs](#step-12-add-docs). diff --git a/docs/connector-development/tutorials/cdk-speedrun.md b/docs/connector-development/tutorials/cdk-speedrun.md index d6caac36974f..d9fc6bc82ffd 100644 --- a/docs/connector-development/tutorials/cdk-speedrun.md +++ b/docs/connector-development/tutorials/cdk-speedrun.md @@ -11,6 +11,7 @@ If you are a visual learner and want to see a video version of this guide going ## Dependencies 1. Python >= 3.9 +2. [Poetry](https://python-poetry.org/) 2. Docker 3. NodeJS @@ -30,9 +31,7 @@ Select the `Python HTTP API Source` and name it `python-http-example`. ```bash cd ../../connectors/source-python-http-example -python -m venv .venv # Create a virtual environment in the .venv directory -source .venv/bin/activate -pip install -r requirements.txt +poetry install ``` ### Define Connector Inputs @@ -118,17 +117,17 @@ cd .. mkdir sample_files echo '{"pokemon_name": "pikachu"}' > sample_files/config.json echo '{"pokemon_name": "chikapu"}' > sample_files/invalid_config.json -python main.py check --config sample_files/config.json -python main.py check --config sample_files/invalid_config.json +poetry run source-python-http-example check --config sample_files/config.json +poetry run source-python-http-example check --config sample_files/invalid_config.json ``` Expected output: -```text -> python main.py check --config sample_files/config.json +```bash +> poetry run source-python-http-example check --config sample_files/config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "SUCCEEDED"}} -> python main.py check --config sample_files/invalid_config.json +> poetry run source-python-http-example check --config sample_files/invalid_config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "FAILED", "message": "'Input Pokemon chikapu is invalid. Please check your spelling our input a valid Pokemon.'"}} ``` @@ -169,7 +168,7 @@ This file defines your output schema for every endpoint that you want to impleme Test your discover function. You should receive a fairly large JSON object in return. ```bash -python main.py discover --config sample_files/config.json +poetry run source-python-http-example discover --config sample_files/config.json ``` Note that our discover function is using the `pokemon_name` config variable passed in from the `Pokemon` stream when we set it in the `__init__` function. @@ -226,7 +225,7 @@ We now need a catalog that defines all of our streams. We only have one stream: Let's read some data. ```bash -python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json +poetry run source-python-http-example read --config sample_files/config.json --catalog sample_files/configured_catalog.json ``` If all goes well, containerize it so you can use it in the UI: diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/connection-checking.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/connection-checking.md index 7acf2b50020a..2e34eb1adf30 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/connection-checking.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/connection-checking.md @@ -42,21 +42,21 @@ Note: in a real implementation you should write code to connect to the API to va Let's test out this implementation by creating two objects: a valid and an invalid config and attempt to give them as input to the connector. For this section, you will need to take the API access key generated earlier and add it to both configs. Because these configs contain secrets, we recommend storing configs which contain secrets in `secrets/config.json` because the `secrets` directory is gitignored by default. -```text +```bash mkdir sample_files echo '{"start_date": "2022-04-01", "base": "USD", "apikey": }' > secrets/config.json echo '{"start_date": "2022-04-01", "base": "BTC", "apikey": }' > secrets/invalid_config.json -python main.py check --config secrets/config.json -python main.py check --config secrets/invalid_config.json +poetry run source-python-http-example check --config secrets/config.json +poetry run source-python-http-example check --config secrets/invalid_config.json ``` You should see output like the following: -```text -> python main.py check --config secrets/config.json +```bash +> poetry run source-python-http-example check --config secrets/config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "SUCCEEDED"}} -> python main.py check --config secrets/invalid_config.json +> poetry run source-python-http-example check --config secrets/invalid_config.json {"type": "CONNECTION_STATUS", "connectionStatus": {"status": "FAILED", "message": "Input currency BTC is invalid. Please input one of the following currencies: {'DKK', 'USD', 'CZK', 'BGN', 'JPY'}"}} ``` diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/declare-schema.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/declare-schema.md index 2ba36afbcbf3..b97aeb1b587b 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/declare-schema.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/declare-schema.md @@ -63,7 +63,7 @@ Having created this stream in code, we'll put a file `exchange_rates.json` in th With `.json` schema file in place, let's see if the connector can now find this schema and produce a valid catalog: ```text -python main.py discover --config secrets/config.json # this is not a mistake, the schema file is found by naming snake_case naming convention as specified above +poetry run source-python-http-example discover --config secrets/config.json # this is not a mistake, the schema file is found by naming snake_case naming convention as specified above ``` you should see some output like: diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md index dce0f253bbec..57b2fb4624f9 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md @@ -7,8 +7,8 @@ This is a step-by-step guide for how to create an Airbyte source in Python to re ## Requirements * Python >= 3.9 +* [Poetry](https://python-poetry.org/) * Docker -* NodeJS \(only used to generate the connector\). We'll remove the NodeJS dependency soon. All the commands below assume that `python` points to a version of python >=3.9.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`. diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/install-dependencies.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/install-dependencies.md index 16c58757969f..3d7e50e22377 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/install-dependencies.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/install-dependencies.md @@ -2,19 +2,16 @@ Now that you've generated the module, let's navigate to its directory and install dependencies: -```text +```bash cd ../../connectors/source- -python -m venv .venv # Create a virtual environment in the .venv directory -source .venv/bin/activate # enable the venv -pip install -r requirements.txt +poetry install ``` -This step sets up the initial python environment. **All** subsequent `python` or `pip` commands assume you have activated your virtual environment. Let's verify everything is working as intended. Run: -```text -python main.py spec +```bash +poetry run source- spec ``` You should see some output: @@ -25,7 +22,6 @@ You should see some output: We just ran Airbyte Protocol's `spec` command! We'll talk more about this later, but this is a simple sanity check to make sure everything is wired up correctly. -Note that the `main.py` file is a simple script that makes it easy to run your connector. Its invocation format is `python main.py [args]`. See the module's generated `README.md` for the commands it supports. ## Notes on iteration cycle @@ -47,12 +43,12 @@ There are two ways we recommend iterating on a source. Consider using whichever You'll notice in your source's directory that there is a python file called `main.py`. This file exists as convenience for development. You run it to test that your source works: -```text +```bash # from airbyte-integrations/connectors/source- -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source- spec +poetry run source- check --config secrets/config.json +poetry run source- discover --config secrets/config.json +poetry run source- read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` The nice thing about this approach is that you can iterate completely within python. The downside is that you are not quite running your source as it will actually be run by Airbyte. Specifically, you're not running it from within the docker container that will house it. @@ -61,7 +57,7 @@ The nice thing about this approach is that you can iterate completely within pyt If you want to run your source exactly as it will be run by Airbyte \(i.e. within a docker container\), you can use the following commands from the connector module directory \(`airbyte-integrations/connectors/source-python-http-example`\): -```text +```bash # First build the container docker build . -t airbyte/source-:dev diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md index 8cdee893e5ab..0417bcdbde25 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md @@ -107,8 +107,8 @@ We're now ready to query the API! To do this, we'll need a [ConfiguredCatalog](../../../understanding-airbyte/beginners-guide-to-catalog.md). We've prepared one [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/tutorials/cdk-tutorial-python-http/configured_catalog.json) -- download this and place it in `sample_files/configured_catalog.json`. Then run: -```text - python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +```bash + poetry run source- --config secrets/config.json --catalog sample_files/configured_catalog.json ``` you should see some output lines, one of which is a record from the API: @@ -239,18 +239,18 @@ We should now have a working implementation of incremental sync! Let's try it out: -```text -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +```bash +poetry run source- --config secrets/config.json --catalog sample_files/configured_catalog.json ``` You should see a bunch of `RECORD` messages and `STATE` messages. To verify that incremental sync is working, pass the input state back to the connector and run it again: -```text +```bash # Save the latest state to sample_files/state.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json | grep STATE | tail -n 1 | jq .state.data > sample_files/state.json +poetry run source- --config secrets/config.json --catalog sample_files/configured_catalog.json | grep STATE | tail -n 1 | jq .state.data > sample_files/state.json # Run a read operation with the latest state message -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --state sample_files/state.json +poetry run source- --config secrets/config.json --catalog sample_files/configured_catalog.json --state sample_files/state.json ``` You should see that only the record from the last date is being synced! This is acceptable behavior, since Airbyte requires at-least-once delivery of records, so repeating the last record twice is OK. diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/test-your-connector.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/test-your-connector.md index f5fbcc07b4f0..521d8b05821f 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/test-your-connector.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/test-your-connector.md @@ -2,15 +2,15 @@ ## Unit Tests -Add any relevant unit tests to the `unit_tests` directory. Unit tests should **not** depend on any secrets. +Add any relevant unit tests to the `tests/unit_tests` directory. Unit tests should **not** depend on any secrets. -You can run the tests using `python -m pytest -s unit_tests`. +You can run the tests using `poetry run pytest tests/unit_tests`. ## Integration Tests Place any integration tests in the `integration_tests` directory such that they can be [discovered by pytest](https://docs.pytest.org/en/6.2.x/goodpractices.html#conventions-for-python-test-discovery). -Run integration tests using `python -m pytest -s integration_tests`. +You can run the tests using `poetry run pytest tests/integration_tests`. More information on integration testing can be found on [the Testing Connectors doc](https://docs.airbyte.com/connector-development/testing-connectors/#running-integration-tests). diff --git a/docs/contributing-to-airbyte/change-cdk-connector.md b/docs/contributing-to-airbyte/change-cdk-connector.md index f4becce2492f..81fcd8a291c0 100644 --- a/docs/contributing-to-airbyte/change-cdk-connector.md +++ b/docs/contributing-to-airbyte/change-cdk-connector.md @@ -21,10 +21,10 @@ Make sure you're working on an issue had been already triaged to not have your c 3. Code the change 4. Write a unit test for each custom function you added or changed 5. Ensure all tests, including connector acceptance tests, pass -6. Update the `metadata.yaml` and `Dockerfile` version following the [guidelines](./resources/pull-requests-handbook.md#semantic-versioning-for-connectors) +6. Update the `metadata.yaml` following the [guidelines](./resources/pull-requests-handbook.md#semantic-versioning-for-connectors) 7. Update the changelog entry in documentation in `docs/integrations/.md` +8. Make sure your contribution passes our [QA checks](./resources/qa-checks.md) -A comment will automatically be added to your PR with a checklist containing the necessary steps to complete your contribution and get it merged. :::info There is a README file inside each connector folder containing instructions to run that connector's tests locally. @@ -55,7 +55,7 @@ When we review, we look at: ## Breaking Changes to Connectors -Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require a +Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require: 1. A **Major Version** increase  2. A [`breakingChanges` entry](https://docs.airbyte.com/connector-development/connector-metadata-file/) in the `releases` section of the `metadata.yaml` file @@ -66,7 +66,12 @@ Often times, changes to connectors can be made without impacting the user experi A breaking change is any change that will require users to take action before they can continue to sync data. The following are examples of breaking changes: - **Spec Change** - The configuration required by users of this connector have been changed and syncs will fail until users reconfigure or re-authenticate.  This change is not possible via a Config Migration  -- **Schema Change** - The type of a property previously present within a record has changed +- **Schema Change** - The type of property previously present within a record has changed - **Stream or Property Removal** - Data that was previously being synced is no longer going to be synced. - **Destination Format / Normalization Change** - The way the destination writes the final data or how normalization cleans that data is changing in a way that requires a full-refresh. -- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced \ No newline at end of file +- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced + +### Limiting the Impact of Breaking Changes +Some of the changes listed above may not impact all users of the connector. For example, a change to the schema of a specific stream only impacts users who are syncing that stream. + +The breaking change metadata allows you to specify narrowed scopes that are specifically affected by a breaking change. See the [`breakingChanges` entry](https://docs.airbyte.com/connector-development/connector-metadata-file/) documentation for supported scopes. diff --git a/docs/contributing-to-airbyte/resources/developing-locally.md b/docs/contributing-to-airbyte/resources/developing-locally.md index 3ca71c9dbd4f..3ddcb437a14b 100644 --- a/docs/contributing-to-airbyte/resources/developing-locally.md +++ b/docs/contributing-to-airbyte/resources/developing-locally.md @@ -2,7 +2,7 @@ The following technologies are required to build Airbyte locally. -1. [`Java 17`](https://jdk.java.net/archive/) +1. [`Java 21`](https://jdk.java.net/archive/) 2. `Node 16` 3. `Python 3.9` 4. `Docker` @@ -204,11 +204,10 @@ cd airbyte-webapp nvm install ``` -- Install the `pnpm` package manager in the required version: +- Install the `pnpm` package manager in the required version. You can use Node's [corepack](https://nodejs.org/api/corepack.html) for that: ```bash -# must be the exact version from airbyte-webapp/package.json > engines.pnpm -npm install -g pnpm@ +corepack enable && corepack install ``` - Start up the react app. diff --git a/docs/contributing-to-airbyte/resources/qa-checks.md b/docs/contributing-to-airbyte/resources/qa-checks.md new file mode 100644 index 000000000000..640b3d458a1a --- /dev/null +++ b/docs/contributing-to-airbyte/resources/qa-checks.md @@ -0,0 +1,90 @@ +# Airbyte connectors QA checks + +This document is listing all the static-analysis checks that are performed on the Airbyte connectors. +These checks are running in our CI/CD pipeline and are used to ensure a connector is following the best practices and is respecting the Airbyte standards. +Meeting these standards means that the connector will be able to be safely integrated into the Airbyte platform and released to registries (DockerHub, Pypi etc.). +You can consider these checks as a set of guidelines to follow when developing a connector. +They are by no mean replacing the need for a manual review of the connector codebase and the implementation of good test suites. + + +## 📄 Documentation + +### Breaking changes must be accompanied by a migration guide +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +When a breaking change is introduced, we check that a migration guide is available. It should be stored under `./docs/integrations/s/-migrations.md`. +This document should contain a section for each breaking change, in order of the version descending. It must explain users which action to take to migrate to the new version. +### Connectors must have user facing documentation +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +The user facing connector documentation should be stored under `./docs/integrations/s/.md`. +### Connectors must have a changelog entry for each version +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Each new version of a connector must have a changelog entry defined in the user facing documentation in `./docs/integrations/s/.md`. + +## 📝 Metadata + +### Connectors must have valid metadata.yaml file +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Connectors must have a `metadata.yaml` file at the root of their directory. This file is used to build our connector registry. Its structure must follow our metadata schema. Field values are also validated. This is to ensure that all connectors have the required metadata fields and that the metadata is valid. More details in this [documentation](https://docs.airbyte.com/connector-development/connector-metadata-file). + +## 📦 Packaging + +### Connectors must use Poetry for dependency management +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: python, low-code* + +Connectors must use [Poetry](https://python-poetry.org/) for dependency management. This is to ensure that all connectors use a dependency management tool which locks dependencies and ensures reproducible installs. +### Connectors must be licensed under MIT or Elv2 +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Connectors must be licensed under the MIT or Elv2 license. This is to ensure that all connectors are licensed under a permissive license. More details in our [License FAQ](https://docs.airbyte.com/developer-guides/licenses/license-faq). +### Connector license in metadata.yaml and pyproject.toml file must match +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: python, low-code* + +Connectors license in metadata.yaml and pyproject.toml file must match. This is to ensure that all connectors are consistently licensed. +### Connector version must follow Semantic Versioning +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Connector version must follow the Semantic Versioning scheme. This is to ensure that all connectors follow a consistent versioning scheme. Refer to our [Semantic Versioning for Connectors](https://docs.airbyte.com/contributing-to-airbyte/#semantic-versioning-for-connectors) for more details. +### Connector version in metadata.yaml and pyproject.toml file must match +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: python, low-code* + +Connector version in metadata.yaml and pyproject.toml file must match. This is to ensure that connector release is consistent. +### Python connectors must have PyPi publishing enabled +*Applies to the following connector types: source* +*Applies to the following connector languages: python, low-code* + +Python connectors must have [PyPi](https://pypi.org/) publishing enabled in their `metadata.yaml` file. This is declared by setting `remoteRegistries.pypi.enabled` to `true` in metadata.yaml. This is to ensure that all connectors can be published to PyPi and can be used in `airbyte-lib`. + +## 💼 Assets + +### Connectors must have an icon +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Each connector must have an icon available in at the root of the connector code directory. It must be an SVG file named `icon.svg` and must be a square. + +## 🔒 Security + +### Connectors must use HTTPS only +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: java, low-code, python* + +Connectors must use HTTPS only when making requests to external services. +### Python connectors must not use a Dockerfile and must declare their base image in metadata.yaml file +*Applies to the following connector types: source, destination* +*Applies to the following connector languages: python, low-code* + +Connectors must use our Python connector base image (`docker.io/airbyte/python-connector-base`), declared through the `connectorBuildOptions.baseImage` in their `metadata.yaml`. +This is to ensure that all connectors use a base image which is maintained and has security updates. diff --git a/docs/contributing-to-airbyte/submit-new-connector.md b/docs/contributing-to-airbyte/submit-new-connector.md index 82cbb25fbf66..664064d4cd13 100644 --- a/docs/contributing-to-airbyte/submit-new-connector.md +++ b/docs/contributing-to-airbyte/submit-new-connector.md @@ -19,6 +19,7 @@ This will enable our team to make sure your contribution does not overlap with e 3. Code the change 4. Ensure all tests pass. For connectors, this includes acceptance tests as well. 5. Update documentation in `docs/integrations/.md` +6. Make sure your contribution passes our [QA checks](./resources/qa-checks.md) #### Open a pull request diff --git a/docs/contributing-to-airbyte/writing-docs.md b/docs/contributing-to-airbyte/writing-docs.md index 62c27b72779d..75e6efd9a4e6 100644 --- a/docs/contributing-to-airbyte/writing-docs.md +++ b/docs/contributing-to-airbyte/writing-docs.md @@ -58,13 +58,13 @@ To make complex changes or edit multiple files, edit the files on your local mac ```bash cd docusaurus - yarn install + pnpm install ``` To see changes as you make them, run: ```bash - yarn start + pnpm start ``` Then navigate to [http://localhost:3005/](http://localhost:3005/). Whenever you make and save changes, you will see them reflected in the server. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal. @@ -72,8 +72,8 @@ To make complex changes or edit multiple files, edit the files on your local mac You can also build the docs locally and see the resulting changes. This is useful if you introduce changes that need to be run at build-time (e.g. adding a docs plug-in). To do so, run: ```bash - yarn build - yarn serve + pnpm build + pnpm serve ``` Then navigate to [http://localhost:3000/](http://localhost:3000/) to see your changes. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal. @@ -324,6 +324,26 @@ Back to ordinary markdown content. ``` Eagle-eyed readers may note that _all_ markdown should support this feature since it's part of the html spec. However, it's worth special mention since these dropdowns have been styled to be a graceful visual fit within our rendered documentation in all environments. +#### Documenting airbyte-lib usage + +airbyte-lib is a Python library that allows to run syncs within a Python script for a subset of connectors. Documentation around airbyte-lib connectors is automatically generated from the connector's JSON schema spec. +There are a few approaches to combine full control over the documentation with automatic generation for common cases: +* If a connector is airbyte-lib enabled (`remoteRegistries.pypi.enabled` set in the `metadata.yaml` file of the connector) and there is no second-level heading `Usage with airbyte-lib` in the documentation, the documentation will be automatically generated and placed above the `Changelog` section. +* By manually specifying a `Usage with airbyte-lib` section, this automatism is disabled. The following is a good starting point for this section: +```md + + +## Usage with airbyte-lib + + + + + + +``` + +The `AirbyteLibExample` component will generate a code example that can be run with airbyte-lib, excluding an auto-generated sample configuration based on the configuration schema. The `SpecSchema` component will generate a reference table with the connector's JSON schema spec, like a non-interactive version of the connector form in the UI. It can be used on any docs page. + ## Additional guidelines - If you're updating a connector doc, follow the [Connector documentation template](https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw) diff --git a/docs/deploying-airbyte/on-aws-ec2.md b/docs/deploying-airbyte/on-aws-ec2.md index cd45e33d1f65..3b352ce59015 100644 --- a/docs/deploying-airbyte/on-aws-ec2.md +++ b/docs/deploying-airbyte/on-aws-ec2.md @@ -91,7 +91,7 @@ ssh -i $SSH_KEY -L 8000:localhost:8000 -N -f ec2-user@$INSTANCE_IP ## Get Airbyte logs in CloudWatch -Follow this [guide](https://aws.amazon.com/pt/premiumsupport/knowledge-center/cloudwatch-docker-container-logs-proxy/) to get your logs from your Airbyte Docker containers in CloudWatch. +Follow this [guide](https://aws.amazon.com/en/premiumsupport/knowledge-center/cloudwatch-docker-container-logs-proxy/) to get your logs from your Airbyte Docker containers in CloudWatch. ## Troubleshooting diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md index 818dec3f78f5..d4f974eb8030 100644 --- a/docs/deploying-airbyte/on-kubernetes-via-helm.md +++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md @@ -2,7 +2,7 @@ ## Overview -Airbyte allows scaling sync workloads horizontally using Kubernetes. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. +Airbyte allows scaling sync workloads horizontally using Kubernetes. The core components \(api server, worker, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. ## Quickstart @@ -10,6 +10,15 @@ If you don't want to configure your own Kubernetes cluster and Airbyte instance, Alternatively, you can deploy Airbyte on [Restack](https://www.restack.io) to provision your Kubernetes cluster on AWS. Follow [this guide](on-restack.md) to get started. +:::note +Airbyte running on Self-Hosted Kubernetes doesn't support DBT Transformations. Please refer to [#5901](https://github.com/airbytehq/airbyte/issues/5091) +::: + +:::note +Airbyte Kubernetes Community Edition does not support basic auth by default. +To enable basic auth, consider adding a reverse proxy in front of Airbyte. +::: + ## Getting Started ### Cluster Setup @@ -158,40 +167,273 @@ Before upgrading the chart update values.yaml as stated above and then run: - Perform upgrade of chart by running `helm upgrade %release_name% airbyte/airbyte --set auth.rootPassword=$ROOT_PASSWORD` - If you get an error about setting the auth.rootPassword, then you forgot to update the `values.yaml` file -### Custom logging and jobs configuration +### External Logs with S3 + +::info +S3 logging was tested on [Airbyte Helm Chart Version 0.50.13](https://artifacthub.io/packages/helm/airbyte/airbyte/0.50.13) +::: + +Create a file called `airbyte-logs-secrets.yaml` to store the AWS Keys and other informations: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: airbyte-logs-secrets +type: Opaque +stringData: + AWS_KEY: + AWS_SECRET_KEY: + S3_LOG_BUCKET: + S3_LOG_BUCKET_REGION: +``` +Run `kubectl apply -f airbyte-logs-secrets.yaml -n ` to create the secret in the namespace you're using Airbyte. +This file contains more than just the keys but it needs for now. Future updates will make the configuration easier. -Starting from `0.39.37-alpha` if you've configured logging yourself using `logging or jobs` section of `values.yaml` file, you need to update your configuration so you can continue to use your custom logging and jobs configuration. +Change the global section to use `S3` external logs. +```yaml +global: + # <...> + state: + # -- Determines which state storage will be utilized; "MINIO", "S3", or "GCS" + storage: + type: "S3" + # <...> + logs: + accessKey: + password: "" + existingSecret: "airbyte-logs-secrets" + existingSecretKey: "AWS_KEY" + secretKey: + password: "" + existingSecret: "airbyte-logs-secrets" + existingSecretKey: "AWS_SECRET_KEY" + # <...> + storage: + type: "S3" + + minio: + # Change from true to false + enabled: false + nodeSelector: {} + tolerations: [] + affinity: {} +``` +GCS Logging information is below but you can try to use `External Minio` as well but it was not tested yet. Feel free to run tests and update the documentation. + +Add extra env variables to the following blocks: +```yaml +worker: + extraEnv: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_KEY + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_SECRET_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_SECRET_KEY + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: S3_LOG_BUCKET + - name: STATE_STORAGE_S3_REGION + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: S3_LOG_BUCKET_REGION +``` + +and also edit the server block: + +```yaml +server: + extraEnv: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_KEY + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_SECRET_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: AWS_SECRET_KEY + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: S3_LOG_BUCKET + - name: STATE_STORAGE_S3_REGION + valueFrom: + secretKeyRef: + name: airbyte-logs-secrets + key: S3_LOG_BUCKET_REGION +``` -Simply declare global value in `values.yaml` file and move everything related to logging and jobs under that section like in the example bellow: +Than run: +`helm upgrade --install %RELEASE_NAME% airbyte/airbyte -n --values /path/to/values.yaml --version 0.50.13` -```text +### External Logs with GCS + + +:::Info +GCS Logging is similar to the approach taken for S3 above, with a few small differences +GCS logging was tested on [Airbyte Helm Chart Version 0.53.178](https://artifacthub.io/packages/helm/airbyte/airbyte/0.53.178) +::: + +#### Create Google Cloud Storage Bucket + +1. **Access Google Cloud Console**: Go to the Google Cloud Console and select or create a project where you want to create the bucket. +2. **Open Cloud Storage**: Navigate to "Storage" > "Browser" in the left-side menu. +3. **Create Bucket**: Click on "Create bucket". Give your bucket a unique name, select a region for the bucket, and configure other settings such as storage class and access control according to your requirements. Finally, click "Create". + +#### Create Google Cloud Service Account + +1. **Open IAM & Admin**: In the Cloud Console, navigate to "IAM & Admin" > "Service Accounts". +2. **Create Service Account**: Click "Create Service Account", enter a name, description, and then click "Create". +3. **Grant Permissions**: Assign the role of "Storage Object Admin" to the service account by selecting it from the role list. +4. **Create Key**: After creating the service account, click on it, go to the "Keys" tab, and then click "Add Key" > "Create new key". Choose JSON as the key type and click "Create". The key file will be downloaded automatically to your computer. + +#### Create a Kubernetes Secret + +- Use the **`kubectl create secret`** command to create a Kubernetes secret from the JSON key file. Replace **``** with the desired name for your secret, **``** with the path to the JSON key file you downloaded, and **``** with the namespace where your deployment will be running. + +```kubectl create secret generic --from-file=gcp.json= --namespace=``` + +#### Create an extra Volume where the GCSFS secret will be added in the values.yaml inside of the worker section +``` +worker: + extraVolumes: + - name: gcsfs-creds + secret: + secretName: + extraVolumeMounts: + - name: gcsfs-creds + mountPath: "/etc/secrets" + readOnly: true +``` + +#### Update the values.yaml with the GCS Logging Information below +Update the following Environment Variables in the global section: +``` global: - logging: - %your_logging_options_here% - jobs: - %your_jobs_options_here% + state: + storage: + type: "GCS" + + logs: + storage: + type: "GCS" + gcs: + bucket: "" + credentials: "/etc/secrets/gcp.json" + + extraEnv: + - name: STATE_STORAGE_GCS_BUCKET_NAME + value: + - name: STATE_STORAGE_GCS_APPLICATION_CREDENTIALS + value: /etc/secrets/gcp.json + - name: CONTAINER_ORCHESTRATOR_SECRET_NAME + value: + - name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + value: /etc/secrets/ ``` -After updating `values.yaml` simply upgrade your chart by running command: +Than run: +`helm upgrade --install %RELEASE_NAME% airbyte/airbyte -n --values /path/to/values.yaml --version 0.53.178` -```shell -helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte +### External Airbyte Database + + + +:::info +This was tested using [Airbyte Helm Chart Version 0.50.13](https://artifacthub.io/packages/helm/airbyte/airbyte/0.50.13). +Previous or newer version can change how the external database can be configured. +::: + + +The Airbyte Database only works with Postgres 13. +Make sure the database is accessible inside the cluster using `busy-box` service using `telnet` or `ping` command. + +:::warning +If you're using the external database for the first time you must ensure the database you're going to use exists. The default database Airbyte will try to use is `airbyte` but you can modified it in the `values.yaml`. +::: + +:::warning +You can use only one database to a one Airbyte Helm deployment. If you try to use the same database for a different deployment it will have conflict with Temporal internal databases. +::: + +Create a Kubernetes secret to store the database password. +Save the file as `db-secrets.yaml`. +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: db-secrets +type: Opaque +stringData: + DATABASE_PASSWORD: ``` -### Database external secrets +Run `kubectl apply -f db-secrets.yaml -n ` to create the secret in the namespace you're using Airbyte. -If you're using external DB secrets, then provide them in `values.yaml` under global.database section in the following format: +Afterward, modify the following blocks in the Helm Chart `values.yaml` file: +```yaml +postgresql: + # Change the value from true to false. + enabled: false +``` +Then: +```yaml +externalDatabase: + # Add the host, username and database name you're using. + host: + user: + database: + password: "" + existingSecret: "db-secrets" + existingSecretPasswordKey: "DATABASE_PASSWORD" + port: 5432 + jdbcUrl: "" +``` +Keep password empty as the Chart will use the `db-secrets` value. +Edit only the host, username, and database name. If your database is using a differnet `port` or need an special `jdbcUrl` you can edit here. +This wasn't fully tested yet. -```text +Next, reference the secret in the global section: +```yaml +global: database: - secretName: "myOctaviaSecret" - secretValue: "postgresql-password" - host: "example.com" - port: "5432" + secretName: "db-secrets" + secretValue: "DATABASE_PASSWORD" ``` -And upgrade the chart by running: +Unfortunately, the `airbyte-bootloader` configuration uses this variable. Future improvements are planned. +Upgrade the chart by running: ```shell -helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte +helm upgrade --install %RELEASE_NAME% airbyte/airbyte -n --values /path/to/values.yaml --version 0.50.13 ``` diff --git a/docs/enterprise-setup/README.md b/docs/enterprise-setup/README.md index 9bb1a95450fa..c46542f240be 100644 --- a/docs/enterprise-setup/README.md +++ b/docs/enterprise-setup/README.md @@ -1,17 +1,21 @@ -# Airbyte Enterprise +--- +products: oss-enterprise +--- -[Airbyte Enterprise](https://airbyte.com/product/airbyte-enterprise) is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. +# Airbyte Self-Managed Enterprise -A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. +[Airbyte Self-Managed Enterprise](https://airbyte.com/product/airbyte-enterprise) is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. + +A valid license key is required to get started with Airbyte Self-Managed Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. The following pages outline how to: 1. [Deploy Airbyte Enterprise using Kubernetes](./implementation-guide.md) -2. [Configure Okta for Single Sign-On (SSO) with Airbyte Enterprise](./sso.md) +2. [Configure Okta for Single Sign-On (SSO) with Airbyte Self-Managed Self-Managed Enterprise](/access-management/sso.md) | Feature | Description | |---------------------------|--------------------------------------------------------------------------------------------------------------| | Premium Support | [Priority assistance](https://docs.airbyte.com/operator-guides/contact-support/#airbyte-enterprise-self-hosted-support) with deploying, managing and upgrading Airbyte or troubleshooting any connection issues. | -| User Management | [Okta SSO](./sso.md) to extend each Airbyte workspace to multiple users | +| User Management | [Okta SSO](/access-management/sso.md) to extend each Airbyte workspace to multiple users | | Multiple Workspaces | Ability to create + manage multiple workspaces on one Airbyte instance | | Role-Based Access Control | Isolate workspaces from one another with users roles scoped to individual workspaces | diff --git a/docs/enterprise-setup/assets/okta-app-integration-name.png b/docs/enterprise-setup/assets/okta-app-integration-name.png deleted file mode 100644 index 87a9b89d77e4..000000000000 Binary files a/docs/enterprise-setup/assets/okta-app-integration-name.png and /dev/null differ diff --git a/docs/enterprise-setup/assets/okta-login-redirect-uris.png b/docs/enterprise-setup/assets/okta-login-redirect-uris.png deleted file mode 100644 index 40463b1acfd4..000000000000 Binary files a/docs/enterprise-setup/assets/okta-login-redirect-uris.png and /dev/null differ diff --git a/docs/enterprise-setup/assets/self-managed-enterprise-aws.png b/docs/enterprise-setup/assets/self-managed-enterprise-aws.png new file mode 100644 index 000000000000..5bb6c7d92ac2 Binary files /dev/null and b/docs/enterprise-setup/assets/self-managed-enterprise-aws.png differ diff --git a/docs/enterprise-setup/implementation-guide.md b/docs/enterprise-setup/implementation-guide.md index 4223b4a807e5..cde0f05d7349 100644 --- a/docs/enterprise-setup/implementation-guide.md +++ b/docs/enterprise-setup/implementation-guide.md @@ -1,41 +1,62 @@ +--- +products: oss-enterprise +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; # Implementation Guide -[Airbyte Enterprise](./README.md) is in an early access stage for select priority users. Once you [are qualified for an Airbyte Enterprise license key](https://airbyte.com/company/talk-to-sales), you can deploy Airbyte with the following instructions. +[Airbyte Self-Managed Enterprise](./README.md) is in an early access stage for select priority users. Once you [are qualified for a Self-Managed Enterprise license key](https://airbyte.com/company/talk-to-sales), you can deploy Airbyte with the following instructions. -Airbyte Enterprise must be deployed using Kubernetes. This is to enable Airbyte's best performance and scale. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. +Airbyte Self-Managed Enterprise must be deployed using Kubernetes. This is to enable Airbyte's best performance and scale. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. ## Prerequisites -There are three prerequisites to deploying Enterprise: installing [helm](https://helm.sh/docs/intro/install/), a Kubernetes cluster, and having configured `kubectl` to connect to the cluster. +For a production-ready deployment of Self-Managed Enterprise, various infrastructure components are required. We recommend deploying to Amazon EKS or Google Kubernetes Engine. The following diagram illustrates a typical Airbyte deployment running on AWS: + +![AWS Architecture Diagram](./assets/self-managed-enterprise-aws.png) + +Prior to deploying Self-Managed Enterprise, we recommend having each of the following infrastructure components ready to go. When possible, it's easiest to have all components running in the same [VPC](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). The provided recommendations are for customers deploying to AWS: -For production, we recommend deploying to EKS, GKE or AKS. If you are doing some local testing, follow the cluster setup instructions outlined [here](/deploying-airbyte/on-kubernetes-via-helm.md#cluster-setup). +| Component | Recommendation | +|--------------------------|-----------------------------------------------------------------------------| +| Kubernetes Cluster | Amazon EKS cluster running in [2 or more availability zones](https://docs.aws.amazon.com/eks/latest/userguide/disaster-recovery-resiliency.html) on a minimum of 6 nodes. | +| Ingress | [Amazon ALB](#configuring-ingress) and a URL for users to access the Airbyte UI or make API requests. | +| Object Storage | [Amazon S3 bucket](#configuring-external-logging) with two directories for log and state storage. | +| Dedicated Database | [Amazon RDS Postgres](#configuring-the-airbyte-database) with at least one read replica. | +| External Secrets Manager | [Amazon Secrets Manager](/operator-guides/configuring-airbyte#secrets) for storing connector secrets. | -To install `kubectl`, please follow [these instructions](https://kubernetes.io/docs/tasks/tools/). To configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`, see the following: + +We also require you to install and configure the following Kubernetes tooling: +1. Install `helm` by following [these instructions](https://helm.sh/docs/intro/install/) +2. Install `kubectl` by following [these instructions](https://kubernetes.io/docs/tasks/tools/). +3. Configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`:
      - Configure kubectl to connect to your cluster - - -
        -
      1. Configure gcloud with gcloud auth login.
        2. -
      2. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME
        3. -
      3. Use kubectl config get-contexts to show the contexts available.
        4. -
      4. Run kubectl config use-context $GKE_CONTEXT to access the cluster from kubectl.
        5. -
      -       - -
        -
      1. Configure your AWS CLI to connect to your project.
        2. -
      2. Install eksctl.
        3. -
      3. Run eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME to make the context available to kubectl.
        4. -
      4. Use kubectl config get-contexts to show the contexts available.
        5. -
      5. Run kubectl config use-context $EKS_CONTEXT to access the cluster with kubectl.
        6. -
      -       -       +Configure kubectl to connect to your cluster + + + + +1. Configure your [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to connect to your project. +2. Install [eksctl](https://eksctl.io/introduction/). +3. Run `eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME` to make the context available to kubectl. +4. Use `kubectl config get-contexts` to show the available contexts. +5. Run `kubectl config use-context $EKS_CONTEXT` to access the cluster with kubectl. + + + + + +1. Configure `gcloud` with `gcloud auth login`. +2. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: `gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME`. +3. Use `kubectl config get-contexts` to show the available contexts. +4. Run `kubectl config use-context $EKS_CONTEXT` to access the cluster with kubectl. + + + +
      ## Deploy Airbyte Enterprise @@ -57,13 +78,16 @@ Follow these instructions to add the Airbyte helm repository: cp configs/airbyte.sample.yml configs/airbyte.yml ``` -3. Add your Airbyte Enterprise license key to your `airbyte.yml`. +3. Add your Airbyte Self-Managed Enterprise license key to your `airbyte.yml`. -4. Add your [auth details](/enterprise-setup/sso) to your `airbyte.yml`. Auth configurations aren't easy to modify after Airbyte is installed, so please double check them to make sure they're accurate before proceeding. +4. Add your [auth details](/access-management/sso) to your `airbyte.yml`.
      Configuring auth in your airbyte.yml file + + + To configure SSO with Okta, add the following at the end of your `airbyte.yml` file: ```yaml @@ -76,14 +100,42 @@ auth: client-secret: $OKTA_CLIENT_SECRET ``` +See the [following guide](/access-management/sso-providers/okta) on how to collect this information for Okta. + + + + +To configure SSO with any identity provider via [OpenID Connect (OIDC)](https://openid.net/developers/how-connect-works/), such as Azure Entra ID (formerly ActiveDirectory), add the following at the end of your `airbyte.yml` file: + +```yaml +auth: + identity-providers: + - type: oidc + domain: $DOMAIN + app-name: $APP_INTEGRATION_NAME + client-id: $CLIENT_ID + client-secret: $CLIENT_SECRET +``` + +See the [following guide](/access-management/sso-providers/azure-entra-id) on how to collect this information for Azure Entra ID (formerly ActiveDirectory). + + + + To configure basic auth (deploy without SSO), remove the entire `auth:` section from your airbyte.yml config file. You will authenticate with the instance admin user and password included in the your `airbyte.yml`. +To modify auth configurations after Airbyte is installed, you will need to redeploy Airbyte with the additional environment variable `KEYCLOAK_RESET_REALM=TRUE`. As this also resets the list of Airbyte users and permissions, please use this with caution. +
      #### Configuring the Airbyte Database For Self-Managed Enterprise deployments, we recommend using a dedicated database instance for better reliability, and backups (such as AWS RDS or GCP Cloud SQL) instead of the default internal Postgres database (`airbyte/db`) that Airbyte spins up within the Kubernetes cluster. +:::info +Currently, Airbyte requires connection to a Postgres 13 instance. +::: + We assume in the following that you've already configured a Postgres instance:
      @@ -139,11 +191,13 @@ minio: - + ```yaml global: ... + log4jConfig: "log4j2-no-minio.xml" + logs: storage: type: "S3" @@ -169,6 +223,37 @@ global: For each of `accessKey` and `secretKey`, the `password` and `existingSecret` fields are mutually exclusive. +3. Ensure your access key is tied to an IAM user with the [following policies](https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-policies-s3.html#iam-policy-ex0), allowing the user access to S3 storage: + +```yaml +{ + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action": "s3:ListAllMyBuckets", + "Resource":"*" + }, + { + "Effect":"Allow", + "Action":["s3:ListBucket","s3:GetBucketLocation"], + "Resource":"arn:aws:s3:::YOUR-S3-BUCKET-NAME" + }, + { + "Effect":"Allow", + "Action":[ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:GetObject", + "s3:GetObjectAcl", + "s3:DeleteObject" + ], + "Resource":"arn:aws:s3:::YOUR-S3-BUCKET-NAME/*" + } + ] +} +``` + @@ -176,6 +261,8 @@ For each of `accessKey` and `secretKey`, the `password` and `existingSecret` fie ```yaml global: ... + log4jConfig: "log4j2-no-minio.xml" + logs: storage: type: "GCS" @@ -185,7 +272,7 @@ global: gcs: bucket: airbyte-dev-logs # GCS bucket name that you've created. - credentials: "" ## ??? + credentials: "" credentialsJson: "" ## Base64 encoded json GCP credentials file contents ``` @@ -200,11 +287,18 @@ Note that the `credentials` and `credentialsJson` fields are mutually exclusive. To access the Airbyte UI, you will need to manually attach an ingress configuration to your deployment. The following is a skimmed down definition of an ingress resource you could use for Self-Managed Enterprise: +
      +Ingress configuration setup steps + + + ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: enterprise-demo + name: # ingress name, example: enterprise-demo + annotations: + ingress.kubernetes.io/ssl-redirect: "false" spec: rules: - host: # host, example: enterprise-demo.airbyte.com @@ -226,8 +320,77 @@ spec: number: # service port, example: 8180 path: /auth pathType: Prefix + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-api-server-svc + name: airbyte-pro-airbyte-api-server-svc + port: + number: # service port, example: 8180 + path: /v1 + pathType: Prefix ``` + + + +If you are intending on using Amazon Application Load Balancer (ALB) for ingress, this ingress definition will be close to what's needed to get up and running: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: + annotations: + # Specifies that the Ingress should use an AWS ALB. + kubernetes.io/ingress.class: "alb" + # Redirects HTTP traffic to HTTPS. + ingress.kubernetes.io/ssl-redirect: "true" + # Creates an internal ALB, which is only accessible within your VPC or through a VPN. + alb.ingress.kubernetes.io/scheme: internal + # Specifies the ARN of the SSL certificate managed by AWS ACM, essential for HTTPS. + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-x:xxxxxxxxx:certificate/xxxxxxxxx-xxxxx-xxxx-xxxx-xxxxxxxxxxx + # Sets the idle timeout value for the ALB. + alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=30 + # [If Applicable] Specifies the VPC subnets and security groups for the ALB + # alb.ingress.kubernetes.io/subnets: '' e.g. 'subnet-12345, subnet-67890' + # alb.ingress.kubernetes.io/security-groups: +spec: + rules: + - host: e.g. enterprise-demo.airbyte.com + http: + paths: + - backend: + service: + name: airbyte-pro-airbyte-webapp-svc + port: + number: 80 + path: / + pathType: Prefix + - backend: + service: + name: airbyte-pro-airbyte-keycloak-svc + port: + number: 8180 + path: /auth + pathType: Prefix + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-api-server-svc + name: airbyte-pro-airbyte-api-server-svc + port: + number: # service port, example: 8180 + path: /v1 + pathType: Prefix +``` + +The ALB controller will use a `ServiceAccount` that requires the [following IAM policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) to be attached. + + + +
      + +Once this is complete, ensure that the value of the `webapp-url` field in your `airbyte.yml` is configured to match the ingress URL. + You may configure ingress using a load balancer or an API Gateway. We do not currently support most service meshes (such as Istio). If you are having networking issues after fully deploying Airbyte, please verify that firewalls or lacking permissions are not interfering with pod-pod communication. Please also verify that deployed pods have the right permissions to make requests to your external database. ### Install Airbyte Enterprise diff --git a/docs/enterprise-setup/sso.md b/docs/enterprise-setup/sso.md deleted file mode 100644 index 8aede3304284..000000000000 --- a/docs/enterprise-setup/sso.md +++ /dev/null @@ -1,58 +0,0 @@ -# Using Single Sign-On (SSO) - -Leverage your existing identity provider to enable employees to access your Airbyte instance using their corporate credentials, simplifying user provisioning. Enabling Single Sign-On extends Airbyte Self Managed to support multiple users, and multiple teams all on one instance. - -Airbyte Self Managed currently supports SSO via OIDC with [Okta](https://www.okta.com/) as an IdP. Support for Azure Active Directory and connecting via SAML are both coming soon. Please talk to us to learn more about upcoming [enterprise features](https://airbyte.com/company/talk-to-sales). - -The following instructions walk you through: -1. [Setting up the Okta OIDC App Integration to be used by your Airbyte instance](#setting-up-okta-for-sso) -2. [Configuring Airbyte Enterprise to use SSO](#deploying-airbyte-enterprise-with-okta) - -### Setting up Okta for SSO - -You will need to create a new Okta OIDC App Integration for your Airbyte instance. Documentation on how to do this in Okta can be found [here](https://help.okta.com/en-us/Content/Topics/Apps/Apps_App_Integration_Wizard_OIDC.htm). - -You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: - -![Screenshot of Okta app integration creation modal](./assets/okta-create-new-app-integration.png) - -#### App integration name - -Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`: - -![Screenshot of Okta app integration name](./assets/okta-app-integration-name.png) - -Spaces or special characters in this field could result in invalid redirect URIs. - -#### Redirect URIs - -In the **Login** section, set the following fields, substituting `` and `` for your own values: - -Sign-in redirect URIs: - -``` -/auth/realms/airbyte/broker//endpoint -``` - -Sign-out redirect URIs - -``` -/auth/realms/airbyte/broker//endpoint/logout_response -``` - -![Okta app integration name screenshot](./assets/okta-login-redirect-uris.png) - -_Example values_ - -`` should point to where your Airbyte instance will be available, including the http/https protocol. - -## Deploying Airbyte Enterprise with Okta - -Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: - -- Okta domain ([how to find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) -- App integration name -- Client ID -- Client Secret - -Visit the [implementation guide](./implementation-guide.md) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. diff --git a/docs/enterprise-setup/upgrading-from-community.md b/docs/enterprise-setup/upgrading-from-community.md new file mode 100644 index 000000000000..15217913cc17 --- /dev/null +++ b/docs/enterprise-setup/upgrading-from-community.md @@ -0,0 +1,105 @@ +--- +products: oss-enterprise +--- + +# Existing Instance Upgrades + +This page supplements the [Self-Managed Enterprise implementation guide](./implementation-guide.md). It highlights the steps to take if you are currently using Airbyte Self-Managed Community, our free open source offering, and are ready to upgrade to [Airbyte Self-Managed Enterprise](./README.md). + +A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. + +These instructions are for you if: +* You want your Self-Managed Enterprise instance to inherit state from your existing deployment. +* You are currently deploying Airbyte on Kubernetes. +* You are comfortable with an in-place upgrade. This guide does not dual-write to a new Airbyte deployment. + +### Step 1: Update Airbyte Open Source + +You must first update to the latest Open Source community release. We assume you are running the following steps from the root of the `airbytehq/airbyte-platform` cloned repo. + +1. Determine your current helm release name by running `helm list`. This will now be referred to as `[RELEASE_NAME]` for the rest of this guide. +2. Upgrade to the latest Open Source community release. The output will now be refered to as `[RELEASE_VERSION]` for the rest of this guide: + +```sh +helm upgrade [RELEASE_NAME] airbyte/airbyte +``` + +### Step 2: Configure Self-Managed Enterprise + +At this step, please create and fill out the `airbyte.yml` as explained in the [Self-Managed Enterprise implementation guide](./implementation-guide.md#clone--configure-airbyte) in the `configs` directory. You should avoid making any changes to your Airbyte database or log storage at this time. When complete, you should have a completed file matching the following skeleton: + +
      +Configuring your airbyte.yml file + +```yml +webapp-url: # example: localhost:8080 + +initial-user: + email: + first-name: + last-name: + username: # your existing Airbyte instance username + password: # your existing Airbyte instance password + +license-key: + +auth: + identity-providers: + - type: okta + domain: + app-name: + client-id: + client-secret: +``` + +
      + +### Step 3: Deploy Self-Managed Enterprise + +1. You can now run the following command to upgrade your instance to Self-Managed Enterprise. If you previously included additional `values` files on your existing deployment, be sure to add these here as well: + +```sh +helm upgrade [RELEASE_NAME] airbyte/airbyte \ +--version [RELEASE_VERSION] \ +--set-file airbyteYml=./configs/airbyte.yml \ +--values ./charts/airbyte/airbyte-pro-values.yaml [... additional --values] +``` + +2. Once this is complete, you will need to upgrade your ingress to include the new `/auth` path. The following is a skimmed down definition of an ingress resource you could use for Self-Managed Enterprise: + +
      +Configuring your Airbyte ingress + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: # ingress name, example: enterprise-demo + annotations: + ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - host: # host, example: enterprise-demo.airbyte.com + http: + paths: + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-webapp-svc + name: airbyte-pro-airbyte-webapp-svc + port: + number: # service port, example: 8080 + path: / + pathType: Prefix + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-keycloak-svc + name: airbyte-pro-airbyte-keycloak-svc + port: + number: # service port, example: 8180 + path: /auth + pathType: Prefix +``` + +
      + +All set! When you log in, you should expect all connections, sources and destinations to be present, and configured as prior. \ No newline at end of file diff --git a/docs/integrations/connector-support-levels.md b/docs/integrations/connector-support-levels.md index e684c1292b7c..f32a1619f663 100644 --- a/docs/integrations/connector-support-levels.md +++ b/docs/integrations/connector-support-levels.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Connector Support Levels The following table describes the support levels of Airbyte connectors. @@ -8,32 +12,57 @@ The following table describes the support levels of Airbyte connectors. | **Who builds them?** | Either the community or the Airbyte team. | Typically they are built by the community. The Airbyte team may upgrade them to Certified at any time. | Anyone can build custom connectors. We recommend using our [Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview) or [Low-code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). | | **Who maintains them?** | The Airbyte team | Users | Users | | **Production Readiness** | Guaranteed by Airbyte | Not guaranteed | Not guaranteed | -| **Support: Cloud** | Supported* | No Support | Supported** | -| **Support: Powered by Airbyte** | Supported* | No Support | Supported** | -| **Support: Self-Managed Enterprise** | Supported* | No Support | Supported** | +| **Support: Cloud** | Supported\* | No Support | Supported\*\* | +| **Support: Powered by Airbyte** | Supported\* | No Support | Supported\*\* | +| **Support: Self-Managed Enterprise** | Supported\* | No Support | Supported\*\* | | **Support: Community (OSS)** | Slack Support only | No Support | Slack Support only | -\*For Certified connectors, Official Support SLAs are only available to customers with Premium Support included in their contract. Otherwise, please use our support portal and we will address your issues as soon as possible. +\*For Certified connectors, Official Support SLAs are only available to customers with Premium +Support included in their contract. Otherwise, please use our support portal and we will address +your issues as soon as possible. -\*\*For Custom connectors, Official Support SLAs are only available to customers with Premium Support included in their contract. This support is provided with best efforts, and maintenance/upgrades are owned by the customer. +\*\*For Custom connectors, Official Support SLAs are only available to customers with Premium +Support included in their contract. This support is provided with best efforts, and +maintenance/upgrades are owned by the customer. ## Certified -A **Certified** connector is actively maintained and supported by the Airbyte team and maintains a high quality bar. It is production ready. +A **Certified** connector is actively maintained and supported by the Airbyte team and maintains a +high quality bar. It is production ready. ### What you should know about Certified connectors: - Certified connectors are available to all users. - These connectors have been tested and vetted in order to be certified and are production ready. -- Certified connectors should go through minimal breaking change but in the event an upgrade is needed users will be given an adequate upgrade window. +- Certified connectors should go through minimal breaking change but in the event an upgrade is + needed users will be given an adequate upgrade window. ## Community -A **Community** connector is maintained by the Airbyte community until it becomes Certified. Airbyte has over 800 code contributors and 15,000 people in the Slack community to help. The Airbyte team is continually certifying Community connectors as usage grows. As these connectors are not maintained by Airbyte, we do not offer support SLAs around them, and we encourage caution when using them in production. +A **Community** connector is maintained by the Airbyte community until it becomes Certified. Airbyte +has over 800 code contributors and 15,000 people in the Slack community to help. The Airbyte team is +continually certifying Community connectors as usage grows. As these connectors are not maintained +by Airbyte, we do not offer support SLAs around them, and we encourage caution when using them in +production. ### What you should know about Community connectors: - Community connectors are available to all users. -- Community connectors may be upgraded to Certified at any time, and we will notify users of these upgrades via our Slack Community and in our Connector Catalog. -- Community connectors might not be feature-complete (features planned for release are under development or not prioritized) and may include backward-incompatible/breaking API changes with no or short notice. +- Community connectors may be upgraded to Certified at any time, and we will notify users of these + upgrades via our Slack Community and in our Connector Catalog. +- Community connectors might not be feature-complete (features planned for release are under + development or not prioritized) and may include backward-incompatible/breaking API changes with no + or short notice. - Community connectors have no Support SLAs. + +## Archived + +From time to time, Airbyte will remove a connector from the Connector Catalog. This is typically due +extremely low usage and/or if the connector is no longer maintained by the community. This is +necessary to ensure that the Connector Catalog maintains a minimum level of quality. + +Archived connectors will not receive any further updates or support from the Airbyte team. Archived +connectors remain source-available in the +[`airbytehq/connector-archive`](https://github.com/airbytehq/connector-archive) repository on +GitHub. If you wish to take over the maintenance of an archived connector, please open a Github +Discussion. diff --git a/docs/integrations/custom-connectors.md b/docs/integrations/custom-connectors.md index bdd14e56a251..aef3132e0be7 100644 --- a/docs/integrations/custom-connectors.md +++ b/docs/integrations/custom-connectors.md @@ -4,7 +4,7 @@ description: Missing a connector? # Custom or New Connector -If you'd like to **ask for a new connector,** you can request it directly [here](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fintegration%2C+new-integration&template=new-integration-request.md&title=). +If you'd like to **ask for a new connector,** you can request it directly [here](https://github.com/airbytehq/airbyte/discussions/new?category=new-connector-request). If you'd like to build new connectors and **make them part of the pool of pre-built connectors on Airbyte,** first a big thank you. We invite you to check our [contributing guide on building connectors](../contributing-to-airbyte/README.md). @@ -12,6 +12,10 @@ If you'd like to build new connectors, or update existing ones, **for your own u ## Developing your own connector +:::info +Custom connectors are currently exclusive to Airbyte Open-Source deployments. However, there are plans for their release on Airbyte Cloud, scheduled for January 2024. You can track the progress on this development [here](https://github.com/orgs/airbytehq/projects/37?pane=issue&itemId=45471174). +::: + It's easy to code your own connectors on Airbyte. Here is a link to instruct on how to code new sources and destinations: [building new connectors](../connector-development/README.md) While the guides in the link above are specific to the languages used most frequently to write integrations, **Airbyte connectors can be written in any language**. Please reach out to us if you'd like help developing connectors in other languages. diff --git a/docs/integrations/destinations/README.md b/docs/integrations/destinations/README.md new file mode 100644 index 000000000000..84df3b620ea9 --- /dev/null +++ b/docs/integrations/destinations/README.md @@ -0,0 +1,14 @@ +import ConnectorRegistry from '@site/src/components/ConnectorRegistry'; + +# Destinations + +A destination is a data warehouse, data lake, database, or an analytics tool where you want to load your ingested data. + +Read more about our [Connector Support Levels](/integrations/connector-support-levels) to understand what to expect from a connector. + + +## Destinations + + + +_[View the connector registry in full](/integrations)_ diff --git a/docs/integrations/destinations/amazon-sqs.md b/docs/integrations/destinations/amazon-sqs.md index 6178d690e0f0..1cf727ff6a34 100644 --- a/docs/integrations/destinations/amazon-sqs.md +++ b/docs/integrations/destinations/amazon-sqs.md @@ -2,7 +2,8 @@ ## Overview -The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently supports sending all streams to a single Queue. +The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently supports sending all +streams to a single Queue. ### Sync overview @@ -10,63 +11,73 @@ The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently All streams will be output into a single SQS Queue. -Amazon SQS messages can only contain JSON, XML or text, and this connector supports writing messages in all three formats. See the **Writing Text or XML messages** section for more info. +Amazon SQS messages can only contain JSON, XML or text, and this connector supports writing messages +in all three formats. See the **Writing Text or XML messages** section for more info. #### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | No | | -| Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | No | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | No | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | No | | +| Namespaces | No | | ## Getting started ### Requirements -* AWS IAM Access Key -* AWS IAM Secret Key -* AWS SQS Queue +- AWS IAM Access Key +- AWS IAM Secret Key +- AWS SQS Queue #### Permissions If the target SQS Queue is not public, you will need the following permissions on the Queue: -* `sqs:SendMessage` +- `sqs:SendMessage` ### Properties Required properties are 'Queue URL' and 'AWS Region' as noted in **bold** below. -* **Queue URL** (STRING) - * The full AWS endpoint URL of the queue e.g.`https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` -* **AWS Region** (STRING) - * The region code for the SQS Queue e.g. eu-west-1 -* Message Delay (INT) - * Time in seconds that this message should be hidden from consumers. - * See the [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-timers.html) for more detail. -* AWS IAM Access Key ID (STRING) - * The Access Key for the IAM User with permissions on this Queue - * Permission `sqs:SendMessage` is required -* AWS IAM Secret Key (STRING) - * The Secret Key for the IAM User with permissions on this Queue -* Message Body Key (STRING) - * Rather than sending the entire Record as the Message Body, use this property to reference a Key in the Record to use as the message body. The value of this property should be the Key name in the input Record. The key must be at the top level of the Record, nested Keys are not supported. -* Message Group Id (STRING) - * When using a FIFO queue, this property is **required**. - * See the [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) for more detail. +- **Queue URL** (STRING) + - The full AWS endpoint URL of the queue + e.g.`https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` +- **AWS Region** (STRING) + - The region code for the SQS Queue e.g. eu-west-1 +- Message Delay (INT) + - Time in seconds that this message should be hidden from consumers. + - See the + [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-timers.html) + for more detail. +- AWS IAM Access Key ID (STRING) + - The Access Key for the IAM User with permissions on this Queue + - Permission `sqs:SendMessage` is required +- AWS IAM Secret Key (STRING) + - The Secret Key for the IAM User with permissions on this Queue +- Message Body Key (STRING) + - Rather than sending the entire Record as the Message Body, use this property to reference a Key + in the Record to use as the message body. The value of this property should be the Key name in + the input Record. The key must be at the top level of the Record, nested Keys are not supported. +- Message Group Id (STRING) + - When using a FIFO queue, this property is **required**. + - See the + [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) + for more detail. ### Setup guide -* [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) -* [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) +- [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) +- [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) #### Using the Message Body Key -This property allows you to reference a Key within the input Record as using that properties Value as the SQS Message Body. +This property allows you to reference a Key within the input Record as using that properties Value +as the SQS Message Body. For example, with the input Record: + ``` { "parent_with_child": { @@ -76,7 +87,9 @@ For example, with the input Record: } ``` -To send *only* the `parent_with_child` object, we can set `Message Body Key` to `parent_with_child`. Giving an output SQS Message of: +To send _only_ the `parent_with_child` object, we can set `Message Body Key` to `parent_with_child`. +Giving an output SQS Message of: + ``` { "child": "child_value" @@ -85,9 +98,11 @@ To send *only* the `parent_with_child` object, we can set `Message Body Key` to #### Writing Text or XML messages -To output Text or XML, the data must be contained within a String field in the input data, and then referenced by setting the `Message Body Key` property. +To output Text or XML, the data must be contained within a String field in the input data, and then +referenced by setting the `Message Body Key` property. For example, with an input Record as: + ``` { "my_xml_field": "value" @@ -102,9 +117,9 @@ The output SQS message would contain: value ``` - ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.0` | 2021-10-27 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | `Initial version` | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------- | +| 0.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.0 | 2021-10-27 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | Initial version | diff --git a/docs/integrations/destinations/astra.md b/docs/integrations/destinations/astra.md new file mode 100644 index 000000000000..b17eb8c29e8c --- /dev/null +++ b/docs/integrations/destinations/astra.md @@ -0,0 +1,42 @@ +# Astra DB Destination + +This page contains the setup guide and reference information for the destination-astra connector. + +## Pre-Requisites + +- An OpenAI, AzureOpenAI, Cohere, etc. API Key + +## Setup Guide + +#### Set Up an Astra Database + +- Create an Astra account [here](https://astra.datastax.com/signup) +- In the Astra Portal, select Databases in the main navigation. +- Click Create Database. +- In the Create Database dialog, select the Serverless (Vector) deployment type. +- In the Configuration section, enter a name for the new database in the Database name field. +-- Because database names can’t be changed later, it’s best to name your database something meaningful. Database names must start and end with an alphanumeric character, and may contain the following special characters: & + - _ ( ) < > . , @. +- Select your preferred Provider and Region. +-- You can select from a limited number of regions if you’re on the Free plan. Regions with a lock icon require that you upgrade to a Pay As You Go plan. +- Click Create Database. +-- You are redirected to your new database’s Overview screen. Your database starts in Pending status before transitioning to Initializing. You’ll receive a notification once your database is initialized. + +#### Gathering other credentials + +- Go back to the Overview tab on the Astra UI +- Copy the Endpoint under Database Details and load into Airbyte under the name astra_db_endpoint +- Click generate token, copy the application token and load under astra_db_app_token + +## Supported Sync Modes + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | | + +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.1 | 2024-01-26 | | DS Branding Update | +| 0.1.0 | 2024-01-08 | | Initial Release | diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index f8ca4dfc07e1..d0323ea35dd8 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -1,78 +1,98 @@ # AWS Datalake -This page contains the setup guide and reference information for the AWS Datalake destination connector. +This page contains the setup guide and reference information for the AWS Datalake destination +connector. -The AWS Datalake destination connector allows you to sync data to AWS. It will write data as JSON files in S3 and -will make it available through a [Lake Formation Governed Table](https://docs.aws.amazon.com/lake-formation/latest/dg/governed-tables.html) in the Glue Data Catalog so that the data is available throughout other AWS services such as Athena, Glue jobs, EMR, Redshift, etc. +The AWS Datalake destination connector allows you to sync data to AWS. It will write data as JSON +files in S3 and will make it available through a +[Lake Formation Governed Table](https://docs.aws.amazon.com/lake-formation/latest/dg/governed-tables.html) +in the Glue Data Catalog so that the data is available throughout other AWS services such as Athena, +Glue jobs, EMR, Redshift, etc. ## Prerequisites To use this destination connector, you will need: -* An AWS account -* An S3 bucket where the data will be written -* An AWS Lake Formation database where tables will be created (one per stream) -* AWS credentials in the form of either the pair Access key ID / Secret key ID or a role with the following permissions: - * Writing objects in the S3 bucket - * Updating of the Lake Formation database +- An AWS account +- An S3 bucket where the data will be written +- An AWS Lake Formation database where tables will be created (one per stream) +- AWS credentials in the form of either the pair Access key ID / Secret key ID or a role with the + following permissions: + + - Writing objects in the S3 bucket + - Updating of the Lake Formation database Please check the Setup guide below if you need guidance creating those. ## Setup guide -You should now have all the requirements needed to configure AWS Datalake as a destination in the UI. You'll need the -following information to configure the destination: +You should now have all the requirements needed to configure AWS Datalake as a destination in the +UI. You'll need the following information to configure the destination: -- Aws Account Id : The account ID of your AWS account. You will find the instructions to setup a new AWS account [here](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). +- Aws Account Id : The account ID of your AWS account. You will find the instructions to setup a new + AWS account + [here](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). - Aws Region : The region in which your resources are deployed -- Authentication mode : The AWS Datalake connector lets you authenticate with either a user or a role. In both case, you will have to make sure -that appropriate policies are in place. Select "ROLE" if you are using a role, "USER" if using a user with Access key / Secret Access key. -- Target Role Arn : The name of the role, if "Authentication mode" was "ROLE". You will find the instructions to create a new role [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-service.html). -- Access Key Id : The Access Key ID of the user if "Authentication mode" was "USER". You will find the instructions to create a new user [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). Make sure to select "Programmatic Access" so that you get secret access keys. +- Authentication mode : The AWS Datalake connector lets you authenticate with either a user or a + role. In both case, you will have to make sure that appropriate policies are in place. Select + "ROLE" if you are using a role, "USER" if using a user with Access key / Secret Access key. +- Target Role Arn : The name of the role, if "Authentication mode" was "ROLE". You will find the + instructions to create a new role + [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-service.html). +- Access Key Id : The Access Key ID of the user if "Authentication mode" was "USER". You will find + the instructions to create a new user + [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). Make sure to select + "Programmatic Access" so that you get secret access keys. - Secret Access Key : The Secret Access Key ID of the user if "Authentication mode" was "USER" -- S3 Bucket Name : The bucket in which the data will be written. You will find the instructions to create a new S3 bucket [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html). +- S3 Bucket Name : The bucket in which the data will be written. You will find the instructions to + create a new S3 bucket + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html). - Target S3 Bucket Prefix : A prefix to prepend to the file name when writing to the bucket -- Database : The database in which the tables will be created. You will find the instructions to create a new Lakeformation Database [here](https://docs.aws.amazon.com/lake-formation/latest/dg/creating-database.html). +- Database : The database in which the tables will be created. You will find the instructions to + create a new Lakeformation Database + [here](https://docs.aws.amazon.com/lake-formation/latest/dg/creating-database.html). **Assigning proper permissions** The policy used by the user or the role must have access to the following services: -* AWS Lake Formation -* AWS Glue -* AWS S3 +- AWS Lake Formation +- AWS Glue +- AWS S3 -You can use [the AWS policy generator](https://awspolicygen.s3.amazonaws.com/policygen.html) to help you generate an appropriate policy. +You can use [the AWS policy generator](https://awspolicygen.s3.amazonaws.com/policygen.html) to help +you generate an appropriate policy. -Please also make sure that the role or user you will use has appropriate permissions on the database in AWS Lakeformation. You will find more information about Lake Formation permissions in the [AWS Lake Formation Developer Guide](https://docs.aws.amazon.com/lake-formation/latest/dg/lake-formation-permissions.html). +Please also make sure that the role or user you will use has appropriate permissions on the database +in AWS Lakeformation. You will find more information about Lake Formation permissions in the +[AWS Lake Formation Developer Guide](https://docs.aws.amazon.com/lake-formation/latest/dg/lake-formation-permissions.html). ## Supported sync modes -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Namespaces | No | | - +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | ## Data type map -The Glue tables will be created with schema information provided by the source, i.e : You will find the same columns -and types in the destination table as in the source except for the following types which will be translated for compatibility with the Glue Data Catalog: - -|Type in the source| Type in the destination| -| :--- | :--- | -| number | float | -| integer | int | - +The Glue tables will be created with schema information provided by the source, i.e : You will find +the same columns and types in the destination table as in the source except for the following types +which will be translated for compatibility with the Glue Data Catalog: +| Type in the source | Type in the destination | +| :----------------- | :---------------------- | +| number | float | +| integer | int | ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.4 | 2023-10-25 | [\#29221](https://github.com/airbytehq/airbyte/pull/29221) | Upgrade AWSWrangler | -| 0.1.3 | 2023-03-28 | [\#24642](https://github.com/airbytehq/airbyte/pull/24642) | Prefer airbyte type for complex types when available | -| 0.1.2 | 2022-09-26 | [\#17193](https://github.com/airbytehq/airbyte/pull/17193) | Fix schema keyerror and add parquet support | -| 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | -| 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------- | +| 0.1.5 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.4 | 2023-10-25 | [\#29221](https://github.com/airbytehq/airbyte/pull/29221) | Upgrade AWSWrangler | +| 0.1.3 | 2023-03-28 | [\#24642](https://github.com/airbytehq/airbyte/pull/24642) | Prefer airbyte type for complex types when available | +| 0.1.2 | 2022-09-26 | [\#17193](https://github.com/airbytehq/airbyte/pull/17193) | Fix schema keyerror and add parquet support | +| 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | +| 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | diff --git a/docs/integrations/destinations/bigquery-migrations.md b/docs/integrations/destinations/bigquery-migrations.md index 7f62d4880b4c..059044e8cec9 100644 --- a/docs/integrations/destinations/bigquery-migrations.md +++ b/docs/integrations/destinations/bigquery-migrations.md @@ -11,4 +11,4 @@ Worthy of specific mention, this version includes: - Removal of sub-tables for nested properties - Removal of SCD tables -Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). \ No newline at end of file +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). \ No newline at end of file diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 32470ef4502c..7f475376d592 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -1,68 +1,115 @@ # BigQuery -Setting up the BigQuery destination connector involves setting up the data loading method (BigQuery Standard method and Google Cloud Storage bucket) and configuring the BigQuery destination connector using the Airbyte UI. +Setting up the BigQuery destination connector involves setting up the data loading method (BigQuery +Standard method and Google Cloud Storage bucket) and configuring the BigQuery destination connector +using the Airbyte UI. This page guides you through setting up the BigQuery destination connector. ## Prerequisites -- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your BigQuery connector to version `1.1.14` or newer +- For Airbyte Open Source users using the + [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, + [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to + version `v0.40.0-alpha` or newer and upgrade your BigQuery connector to version `1.1.14` or newer - [A Google Cloud project with BigQuery enabled](https://cloud.google.com/bigquery/docs/quickstarts/query-public-dataset-console) -- [A BigQuery dataset](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui#create_a_dataset) to sync data to. +- [A BigQuery dataset](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui#create_a_dataset) + to sync data to. - **Note:** Queries written in BigQuery can only reference datasets in the same physical location. If you plan on combining the data that Airbyte syncs with data from other datasets in your queries, create the datasets in the same location on Google Cloud. For more information, read [Introduction to Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) + **Note:** Queries written in BigQuery can only reference datasets in the same physical location. + If you plan on combining the data that Airbyte syncs with data from other datasets in your + queries, create the datasets in the same location on Google Cloud. For more information, read + [Introduction to Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) -- (Required for Airbyte Cloud; Optional for Airbyte Open Source) A Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts) with the [`BigQuery User`](https://cloud.google.com/bigquery/docs/access-control#bigquery) and [`BigQuery Data Editor`](https://cloud.google.com/bigquery/docs/access-control#bigquery) roles and the [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). +- (Required for Airbyte Cloud; Optional for Airbyte Open Source) A Google Cloud + [Service Account](https://cloud.google.com/iam/docs/service-accounts) with the + [`BigQuery User`](https://cloud.google.com/bigquery/docs/access-control#bigquery) and + [`BigQuery Data Editor`](https://cloud.google.com/bigquery/docs/access-control#bigquery) roles and + the + [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). ## Setup guide ### Step 1: Set up a data loading method -Although you can load data using BigQuery's [`INSERTS`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax), we highly recommend using a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/introduction) not only for performance and cost but reliability since larger datasets are prone to more failures when using standard inserts. +Although you can load data using BigQuery's +[`INSERTS`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax), we highly +recommend using a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/introduction) +not only for performance and cost but reliability since larger datasets are prone to more failures +when using standard inserts. #### (Recommended) Using a Google Cloud Storage bucket To use a Google Cloud Storage bucket: -1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) with the Protection Tools set to `none` or `Object versioning`. Make sure the bucket does not have a [retention policy](https://cloud.google.com/storage/docs/samples/storage-set-retention-policy). +1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) with the + Protection Tools set to `none` or `Object versioning`. Make sure the bucket does not have a + [retention policy](https://cloud.google.com/storage/docs/samples/storage-set-retention-policy). 2. [Create an HMAC key and access ID](https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create). -3. Grant the [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). This must be the same service account as the one you configure for BigQuery access in the [BigQuery connector setup step](#step-2-set-up-the-bigquery-connector). -4. Make sure your Cloud Storage bucket is accessible from the machine running Airbyte. The easiest way to verify if Airbyte is able to connect to your bucket is via the check connection tool in the UI. - -Your bucket must be encrypted using a Google-managed encryption key (this is the default setting when creating a new bucket). We currently do not support buckets using customer-managed encryption keys (CMEK). You can view this setting under the "Configuration" tab of your GCS bucket, in the `Encryption type` row. +3. Grant the + [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) + to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). This + must be the same service account as the one you configure for BigQuery access in the + [BigQuery connector setup step](#step-2-set-up-the-bigquery-connector). +4. Make sure your Cloud Storage bucket is accessible from the machine running Airbyte. The easiest + way to verify if Airbyte is able to connect to your bucket is via the check connection tool in + the UI. + +Your bucket must be encrypted using a Google-managed encryption key (this is the default setting +when creating a new bucket). We currently do not support buckets using customer-managed encryption +keys (CMEK). You can view this setting under the "Configuration" tab of your GCS bucket, in the +`Encryption type` row. #### Using `INSERT` -You can use BigQuery's [`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax) statement to upload data directly from your source to BigQuery. While this is faster to set up initially, we strongly recommend not using this option for anything other than a quick demo. Due to the Google BigQuery SDK client limitations, using `INSERT` is 10x slower than using a Google Cloud Storage bucket, and you may see some failures for big datasets and slow sources (For example, if reading from a source takes more than 10-12 hours). For more details, refer to https://github.com/airbytehq/airbyte/issues/3549 +You can use BigQuery's +[`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax) statement to +upload data directly from your source to BigQuery. While this is faster to set up initially, we +strongly recommend not using this option for anything other than a quick demo. Due to the Google +BigQuery SDK client limitations, using `INSERT` is 10x slower than using a Google Cloud Storage +bucket, and you may see some failures for big datasets and slow sources (For example, if reading +from a source takes more than 10-12 hours). For more details, refer to +https://github.com/airbytehq/airbyte/issues/3549 ### Step 2: Set up the BigQuery connector -1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source + account. 2. Click **Destinations** and then click **+ New destination**. 3. On the Set up the destination page, select **BigQuery** from the **Destination type** dropdown. 4. Enter the name for the BigQuery connector. -5. For **Project ID**, enter your [Google Cloud project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). -6. For **Dataset Location**, select the location of your BigQuery dataset. - :::warning - You cannot change the location later. - ::: -7. For **Default Dataset ID**, enter the BigQuery [Dataset ID](https://cloud.google.com/bigquery/docs/datasets#create-dataset). -8. For **Loading Method**, select [Standard Inserts](#using-insert) or [GCS Staging](#recommended-using-a-google-cloud-storage-bucket). - :::tip - We recommend using the GCS Staging option. - ::: -9. For **Service Account Key JSON (Required for cloud, optional for open-source)**, enter the Google Cloud [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). -10. For **Transformation Query Run Type (Optional)**, select **interactive** to have [BigQuery run interactive query jobs](https://cloud.google.com/bigquery/docs/running-queries#queries) or **batch** to have [BigQuery run batch queries](https://cloud.google.com/bigquery/docs/running-queries#batch). - - :::note - Interactive queries are executed as soon as possible and count towards daily concurrent quotas and limits, while batch queries are executed as soon as idle resources are available in the BigQuery shared resource pool. If BigQuery hasn't started the query within 24 hours, BigQuery changes the job priority to interactive. Batch queries don't count towards your concurrent rate limit, making it easier to start many queries at once. - ::: - -11. For **Google BigQuery Client Chunk Size (Optional)**, use the default value of 15 MiB. Later, if you see networking or memory management problems with the sync (specifically on the destination), try decreasing the chunk size. In that case, the sync will be slower but more likely to succeed. +5. For **Project ID**, enter your + [Google Cloud project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). +6. For **Dataset Location**, select the location of your BigQuery dataset. :::warning You cannot + change the location later. ::: +7. For **Default Dataset ID**, enter the BigQuery + [Dataset ID](https://cloud.google.com/bigquery/docs/datasets#create-dataset). +8. For **Loading Method**, select [Standard Inserts](#using-insert) or + [GCS Staging](#recommended-using-a-google-cloud-storage-bucket). :::tip We recommend using the + GCS Staging option. ::: +9. For **Service Account Key JSON (Required for cloud, optional for open-source)**, enter the Google + Cloud + [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). +10. For **Transformation Query Run Type (Optional)**, select **interactive** to have + [BigQuery run interactive query jobs](https://cloud.google.com/bigquery/docs/running-queries#queries) + or **batch** to have + [BigQuery run batch queries](https://cloud.google.com/bigquery/docs/running-queries#batch). + + :::note Interactive queries are executed as soon as possible and count towards daily concurrent + quotas and limits, while batch queries are executed as soon as idle resources are available in + the BigQuery shared resource pool. If BigQuery hasn't started the query within 24 hours, + BigQuery changes the job priority to interactive. Batch queries don't count towards your + concurrent rate limit, making it easier to start many queries at once. ::: + +11. For **Google BigQuery Client Chunk Size (Optional)**, use the default value of 15 MiB. Later, if + you see networking or memory management problems with the sync (specifically on the + destination), try decreasing the chunk size. In that case, the sync will be slower but more + likely to succeed. ## Supported sync modes -The BigQuery destination connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +The BigQuery destination connector supports the following +[sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - Full Refresh Sync - Incremental - Append Sync @@ -70,7 +117,9 @@ The BigQuery destination connector supports the following [sync modes](https://d ## Output schema -Airbyte outputs each stream into its own raw table in `airbyte_internal` dataset by default (can be overriden by user) and a final table with Typed columns. Contents in raw table are _NOT_ deduplicated. +Airbyte outputs each stream into its own raw table in `airbyte_internal` dataset by default (can be +overriden by user) and a final table with Typed columns. Contents in raw table are _NOT_ +deduplicated. ### Raw Table schema @@ -81,55 +130,76 @@ Airbyte outputs each stream into its own raw table in `airbyte_internal` dataset | \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP | | \_airbyte_data | A JSON blob with the event data. | STRING | -**Note:** Although the contents of the `_airbyte_data` are fairly stable, schema of the raw table could be subject to change in future versions. +**Note:** Although the contents of the `_airbyte_data` are fairly stable, schema of the raw table +could be subject to change in future versions. ### Final Table schema -- `airbyte_raw_id`: A UUID assigned by Airbyte to each event that is processed. The column type in BigQuery is `String`. -- `airbyte_extracted_at`: A timestamp representing when the event was pulled from the data source. The column type in BigQuery is `Timestamp`. -- `_airbyte_meta`: A JSON blob representing typing errors. You can query these results to audit misformatted or unexpected data. The column type in BigQuery is `JSON`. - ... and a column of the proper data type for each of the top-level properties from your source's schema. Arrays and Objects will remain as JSON columns in BigQuery. Learn more about Typing and Deduping [here](/understanding-airbyte/typing-deduping) - -The output tables in BigQuery are partitioned by the Time-unit column `airbyte_extracted_at` at a daily granularity and clustered by `airbyte_extracted_at` and the table Primary Keys. Partitions boundaries are based on UTC time. -This is useful to limit the number of partitions scanned when querying these partitioned tables, by using a predicate filter (a `WHERE` clause). Filters on the partitioning column are used to prune the partitions and reduce the query cost. (The parameter **Require partition filter** is not enabled by Airbyte, but you may toggle it by updating the produced tables.) +- `airbyte_raw_id`: A UUID assigned by Airbyte to each event that is processed. The column type in + BigQuery is `String`. +- `airbyte_extracted_at`: A timestamp representing when the event was pulled from the data source. + The column type in BigQuery is `Timestamp`. +- `_airbyte_meta`: A JSON blob representing typing errors. You can query these results to audit + misformatted or unexpected data. The column type in BigQuery is `JSON`. ... and a column of the + proper data type for each of the top-level properties from your source's schema. Arrays and + Objects will remain as JSON columns in BigQuery. Learn more about Typing and Deduping + [here](/using-airbyte/core-concepts/typing-deduping) + +The output tables in BigQuery are partitioned by the Time-unit column `airbyte_extracted_at` at a +daily granularity and clustered by `airbyte_extracted_at` and the table Primary Keys. Partitions +boundaries are based on UTC time. This is useful to limit the number of partitions scanned when +querying these partitioned tables, by using a predicate filter (a `WHERE` clause). Filters on the +partitioning column are used to prune the partitions and reduce the query cost. (The parameter +**Require partition filter** is not enabled by Airbyte, but you may toggle it by updating the +produced tables.) ## BigQuery Naming Conventions -Follow [BigQuery Datasets Naming conventions](https://cloud.google.com/bigquery/docs/datasets#dataset-naming). +Follow +[BigQuery Datasets Naming conventions](https://cloud.google.com/bigquery/docs/datasets#dataset-naming). -Airbyte converts any invalid characters into `_` characters when writing data. However, since datasets that begin with `_` are hidden on the BigQuery Explorer panel, Airbyte prepends the namespace with `n` for converted namespaces. +Airbyte converts any invalid characters into `_` characters when writing data. However, since +datasets that begin with `_` are hidden on the BigQuery Explorer panel, Airbyte prepends the +namespace with `n` for converted namespaces. ## Data type map | Airbyte type | BigQuery type | | :---------------------------------- | :------------ | -| DATE | DATE | -| STRING (BASE64) | STRING | -| NUMBER | FLOAT | -| OBJECT | STRING | | STRING | STRING | -| BOOLEAN | BOOLEAN | -| INTEGER | INTEGER | +| STRING (BASE64) | STRING | | STRING (BIG_NUMBER) | STRING | | STRING (BIG_INTEGER) | STRING | -| ARRAY | REPEATED | +| NUMBER | NUMERIC | +| INTEGER | INT64 | +| BOOLEAN | BOOL | | STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMP | -| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | DATETIME | +| STRING (TIME_WITH_TIMEZONE) | STRING | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | JSON | +| ARRAY | JSON | ## Troubleshooting permission issues The service account does not have the proper permissions. -- Make sure the BigQuery service account has `BigQuery User` and `BigQuery Data Editor` roles or equivalent permissions as those two roles. -- If the GCS staging mode is selected, ensure the BigQuery service account has the right permissions to the GCS bucket and path or the `Cloud Storage Admin` role, which includes a superset of the required permissions. +- Make sure the BigQuery service account has `BigQuery User` and `BigQuery Data Editor` roles or + equivalent permissions as those two roles. +- If the GCS staging mode is selected, ensure the BigQuery service account has the right permissions + to the GCS bucket and path or the `Cloud Storage Admin` role, which includes a superset of the + required permissions. The HMAC key is wrong. -- Make sure the HMAC key is created for the BigQuery service account, and the service account has permission to access the GCS bucket and path. +- Make sure the HMAC key is created for the BigQuery service account, and the service account has + permission to access the GCS bucket and path. ## Tutorials -Now that you have set up the BigQuery destination connector, check out the following BigQuery tutorials: +Now that you have set up the BigQuery destination connector, check out the following BigQuery +tutorials: - [Export Google Analytics data to BigQuery](https://airbyte.com/tutorials/export-google-analytics-to-bigquery) - [Load data from Facebook Ads to BigQuery](https://airbyte.com/tutorials/facebook-ads-to-bigquery) @@ -140,6 +210,29 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.4.11 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 2.4.10 | 2024-02-15 | [35240](https://github.com/airbytehq/airbyte/pull/35240) | Adopt CDK 0.20.9 | +| 2.4.9 | 2024-02-15 | [35285](https://github.com/airbytehq/airbyte/pull/35285) | Adopt CDK 0.20.8 | +| 2.4.8 | 2024-02-12 | [35144](https://github.com/airbytehq/airbyte/pull/35144) | Adopt CDK 0.20.2 | +| 2.4.7 | 2024-02-12 | [35111](https://github.com/airbytehq/airbyte/pull/35111) | Adopt CDK 0.20.1 | +| 2.4.6 | 2024-02-09 | [34575](https://github.com/airbytehq/airbyte/pull/34575) | Adopt CDK 0.20.0 | +| 2.4.5 | 2024-02-08 | [34745](https://github.com/airbytehq/airbyte/pull/34745) | Adopt CDK 0.19.0 | +| 2.4.4 | 2024-02-08 | [35027](https://github.com/airbytehq/airbyte/pull/35027) | Upgrade CDK to 0.17.1 | +| 2.4.3 | 2024-02-01 | [34728](https://github.com/airbytehq/airbyte/pull/34728) | Upgrade CDK to 0.16.4; Notable changes from 0.14.2, 0.15.1 and 0.16.3 | +| 2.4.2 | 2024-01-24 | [34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 2.4.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 2.4.0 | 2024-01-24 | [34468](https://github.com/airbytehq/airbyte/pull/34468) | Upgrade CDK to 0.14.0 | +| 2.3.31 | 2024-01-22 | [\#34023](https://github.com/airbytehq/airbyte/pull/34023) | Combine DDL operations into a single execution | +| 2.3.30 | 2024-01-12 | [\#34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies | +| 2.3.29 | 2024-01-09 | [\#34003](https://github.com/airbytehq/airbyte/pull/34003) | Fix loading credentials from GCP Env | +| 2.3.28 | 2024-01-08 | [\#34021](https://github.com/airbytehq/airbyte/pull/34021) | Add idempotency ids in dummy insert for check call | +| 2.3.27 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | +| 2.3.26 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 2.3.25 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Update to java CDK 0.10.0 (no changes) | +| 2.3.24 | 2023-12-20 | [\#33697](https://github.com/airbytehq/airbyte/pull/33697) | Stop creating unnecessary tmp tables | +| 2.3.23 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | +| 2.3.22 | 2023-12-14 | [\#33451](https://github.com/airbytehq/airbyte/pull/33451) | Remove old spec option | +| 2.3.21 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Only run typing+deduping for a stream if the stream had any records | | 2.3.20 | 2023-12-08 | [\#33263](https://github.com/airbytehq/airbyte/pull/33263) | Adopt java CDK version 0.7.0 | | 2.3.19 | 2023-12-07 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Update common T&D interfaces | | 2.3.18 | 2023-12-04 | [\#33084](https://github.com/airbytehq/airbyte/pull/33084) | T&D SQL statements moved to debug log level | diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md index af1cb3499f0c..10d65af7170a 100644 --- a/docs/integrations/destinations/chroma.md +++ b/docs/integrations/destinations/chroma.md @@ -76,6 +76,7 @@ You should now have all the requirements needed to configure Chroma as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.9 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.0.8 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | | 0.0.7 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | | 0.0.6 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | diff --git a/docs/integrations/destinations/clickhouse-migrations.md b/docs/integrations/destinations/clickhouse-migrations.md new file mode 100644 index 000000000000..df8590b36a56 --- /dev/null +++ b/docs/integrations/destinations/clickhouse-migrations.md @@ -0,0 +1,66 @@ +# Clickhouse Migration Guide + +## Upgrading to 1.0.0 + +This version removes the option to use "normalization" with clickhouse. It also changes +the schema and database of Airbyte's "raw" tables to be compatible with the new +[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) +format. These changes will likely require updates to downstream dbt / SQL models. After this update, +Airbyte will only produce the ‘raw’ v2 tables, which store all content in JSON. These changes remove +the ability to do deduplicated syncs with Clickhouse. (Clickhouse has an overview)[[https://clickhouse.com/docs/en/integrations/dbt]] +for integrating with dbt If you are interested in the Clickhouse destination gaining the full features +of Destinations V2 (including final tables), click [[https://github.com/airbytehq/airbyte/discussions/35339]] +to register your interest. + +This upgrade will ignore any existing raw tables and will not migrate any data to the new schema. +For each stream, you could perform the following query to migrate the data from the old raw table +to the new raw table: + +```sql +-- assumes your database was 'default' +-- replace `{{stream_name}}` with replace your stream name + +CREATE TABLE airbyte_internal.default_raw__stream_{{stream_name}} +( + `_airbyte_raw_id` String, + `_airbyte_extracted_at` DateTime64(3, 'GMT') DEFAULT now(), + `_airbyte_loaded_at` DateTime64(3, 'GMT') NULL, + `_airbyte_data` String, + PRIMARY KEY(`_airbyte_raw_id`) +) +ENGINE = MergeTree; + +INSERT INTO `airbyte_internal`.`default_raw__stream_{{stream_name}}` + SELECT + `_airbyte_ab_id` AS "_airbyte_raw_id", + `_airbyte_emitted_at` AS "_airbyte_extracted_at", + NULL AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM default._airbyte_raw_{{stream_name}}; +``` + +Airbyte will not delete any of your v1 data. + +### Database/Schema and the Internal Schema +We have split the raw and final tables into their own schemas, +which in clickhouse is analogous to a `database`. For the Clickhouse destination, this means that +we will only write into the raw table which will live in the `airbyte_internal` database. +The tables written into this schema will be prefixed with either the default database provided in +the `DB Name` field when configuring clickhouse (but can also be overridden in the connection). You can +change the "raw" database from the default `airbyte_internal` by supplying a value for +`Raw Table Schema Name`. + +For Example: + + - DB Name: `default` + - Stream Name: `my_stream` + +Writes to `airbyte_intneral.default_raw__stream_my_stream` + +where as: + + - DB Name: `default` + - Stream Name: `my_stream` + - Raw Table Schema Name: `raw_data` + +Writes to: `raw_data.default_raw__stream_my_stream` diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 02446ba825f6..4495cb79e3da 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -44,6 +44,17 @@ You can create such a user by running: ``` GRANT CREATE ON * TO airbyte_user; +GRANT CREATE ON default * TO airbyte_user; +GRANT DROP ON * TO airbyte_user; +GRANT TRUNCATE ON * TO airbyte_user; +GRANT INSERT ON * TO airbyte_user; +GRANT SELECT ON * TO airbyte_user; +GRANT CREATE DATABASE ON airbyte_internal.* TO airbyte_user; +GRANT CREATE TABLE ON airbyte_internal.* TO airbyte_user; +GRANT DROP ON airbyte_internal.* TO airbyte_user; +GRANT TRUNCATE ON airbyte_internal.* TO airbyte_user; +GRANT INSERT ON airbyte_internal.* TO airbyte_user; +GRANT SELECT ON airbyte_internal.* TO airbyte_user; ``` You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte. @@ -78,7 +89,8 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +|:--------|:-----------| :--------------------------------------------------------- |:----------------------------------------------------------------------------------------------| +| 1.0.0 | 2024-02-07 | [\#34637](https://github.com/airbytehq/airbyte/pull/34637) | Update the raw table schema | | 0.2.5 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | | 0.2.4 | 2023-06-05 | [\#27036](https://github.com/airbytehq/airbyte/pull/27036) | Internal code change for future development (install normalization packages inside connector) | | 0.2.3 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md index e5bcd6e8cdc0..d39e64084c44 100644 --- a/docs/integrations/destinations/databricks.md +++ b/docs/integrations/destinations/databricks.md @@ -2,11 +2,20 @@ ## Overview -This destination syncs data to Delta Lake on Databricks Lakehouse. Each stream is written to its own [delta-table](https://delta.io/). +This destination syncs data to Delta Lake on Databricks Lakehouse. Each stream is written to its own +[delta-table](https://delta.io/). -This connector requires a JDBC driver to connect to the Databricks cluster. By using the driver and the connector, you must agree to the [JDBC ODBC driver license](https://databricks.com/jdbc-odbc-driver-license). This means that you can only use this connector to connect third party applications to Apache Spark SQL within a Databricks offering using the ODBC and/or JDBC protocols. +This connector requires a JDBC driver to connect to the Databricks cluster. By using the driver and +the connector, you must agree to the +[JDBC ODBC driver license](https://databricks.com/jdbc-odbc-driver-license). This means that you can +only use this connector to connect third party applications to Apache Spark SQL within a Databricks +offering using the ODBC and/or JDBC protocols. -Currently, this connector requires 30+MB of memory for each stream. When syncing multiple streams, it may run into an out-of-memory error if the allocated memory is too small. This performance bottleneck is tracked in [this issue](https://github.com/airbytehq/airbyte/issues/11424). Once this issue is resolved, the connector should be able to sync an almost infinite number of streams with less than 500MB of memory. +Currently, this connector requires 30+MB of memory for each stream. When syncing multiple streams, +it may run into an out-of-memory error if the allocated memory is too small. This performance +bottleneck is tracked in [this issue](https://github.com/airbytehq/airbyte/issues/11424). Once this +issue is resolved, the connector should be able to sync an almost infinite number of streams with +less than 500MB of memory. ## Getting started @@ -14,23 +23,31 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing ### 1. Create a Databricks Workspace -- Follow Databricks guide [Create a workspace using the account console](https://docs.databricks.com/administration-guide/workspace/create-workspace.html#create-a-workspace-using-the-account-console). - > **_IMPORTANT:_** Don't forget to create a [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) for workspaces +- Follow Databricks guide + [Create a workspace using the account console](https://docs.databricks.com/administration-guide/workspace/create-workspace.html#create-a-workspace-using-the-account-console). + > **_IMPORTANT:_** Don't forget to create a + > [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) + > for workspaces > **_TIP:_** Alternatively use Databricks quickstart for new workspace > ![](../../.gitbook/assets/destination/databricks/databricks_workspace_quciksetup.png) ### 2. Create a metastore and attach it to workspace -> **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier. +> **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to +> access the data. Make sure that this matches the region of the cloud storage bucket you created +> earlier. #### Setup storage bucket and IAM role in AWS -Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) to setup AWS bucket with necessary permissions. +Follow +[Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) +to setup AWS bucket with necessary permissions. #### Create metastore -- Login into Databricks [account console](https://accounts.cloud.databricks.com/login) with admin permissions. +- Login into Databricks [account console](https://accounts.cloud.databricks.com/login) with admin + permissions. - Go to Data tab and hit Create metastore button: ![](../../.gitbook/assets/destination/databricks/databricks_new_metastore.png) @@ -41,8 +58,11 @@ Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks. - `Name` - `Region` The metastore should be in same region as the workspace. - - `S3 bucket path` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. - - `IAM role ARN` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. Example: `arn:aws:iam:::role/` + - `S3 bucket path` created at + [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. + - `IAM role ARN` created at + [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. + Example: `arn:aws:iam:::role/` - Select the workspaces in `Assign to workspaces` tab and click Assign. @@ -134,16 +154,24 @@ Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks. ![](../../.gitbook/assets/destination/databricks/databricks_new_external_location.png) -> **_TIP:_** The new `Storage credential` can be added in the `Storage Credentials` tab or use same as for Metastore. +> **_TIP:_** The new `Storage credential` can be added in the `Storage Credentials` tab or use same +> as for Metastore. ## Airbyte Setup ### Databricks fields -- `Agree to the Databricks JDBC Driver Terms & Conditions` - [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license). -- `Server Hostname` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. -- `HTTP Path` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. -- `Port` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `Agree to the Databricks JDBC Driver Terms & Conditions` - + [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license). +- `Server Hostname` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `HTTP Path` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `Port` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. - `Access Token` - can be taken from [7. Create Databricks Token](#7-create-databricks-token) step. ### Data Source @@ -156,25 +184,39 @@ You could choose a data source type #### Managed tables data source type -Please check Databricks documentation about [What is managed tables](https://docs.databricks.com/lakehouse/data-objects.html#what-is-a-managed-table) +Please check Databricks documentation about +[What is managed tables](https://docs.databricks.com/lakehouse/data-objects.html#what-is-a-managed-table) > **_TIP:_** There is no addition setup should be done for this type. #### Amazon S3 data source type (External storage) -> **_IMPORTANT:_** Make sure the `External Locations` has been added to the workspace. Check [Adding External Locations](#8-adding-external-locations-optional) step. +> **_IMPORTANT:_** Make sure the `External Locations` has been added to the workspace. Check +> [Adding External Locations](#8-adding-external-locations-optional) step. Provide your Amazon S3 data: - `S3 Bucket Name` - The bucket name - `S3 Bucket Path` - Subdirectory under the above bucket to sync the data into -- `S3 Bucket Region` - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. - > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier. +- `S3 Bucket Region` - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. + > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to + > access the data. Make sure that this matches the region of the cloud storage bucket you created + > earlier. - `S3 Access Key ID` - Corresponding key to the above key id - `S3 Secret Access Key` - - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. -- `S3 Filename pattern` - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't be recognized + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. +- `S3 Filename pattern` - The pattern allows you to set the file-name format for the S3 staging + file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, + `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't be recognized #### Azure Blob Storage data source type (External storage) @@ -191,34 +233,39 @@ Provide your Amazon S3 data: ## Configuration -| Category | Parameter | Type | Notes | -| :------------------ | :-------------------- | :-----: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Databricks | Server Hostname | string | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster. | -| | HTTP Path | string | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | -| | Port | string | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | -| | Personal Access Token | string | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html). | -| General | Databricks catalog | string | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used. | -| | Database schema | string | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used. | -| | Schema evolution | boolean | Optional. The connector enables automatic schema evolution in the destination tables. | -| | Purge Staging Data | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes. | -| Data Source - S3 | Bucket Name | string | Name of the bucket to sync data into. | -| | Bucket Path | string | Subdirectory under the above bucket to sync the data into. | -| | Region | string | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. | -| | Access Key ID | string | AWS/Minio credential. | -| | Secret Access Key | string | AWS/Minio credential. | +| Category | Parameter | Type | Notes | +| :------------------ | :-------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Databricks | Server Hostname | string | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster. | +| | HTTP Path | string | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | +| | Port | string | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | +| | Personal Access Token | string | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html). | +| General | Databricks catalog | string | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used. | +| | Database schema | string | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used. | +| | Schema evolution | boolean | Optional. The connector enables automatic schema evolution in the destination tables. | +| | Purge Staging Data | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes. | +| Data Source - S3 | Bucket Name | string | Name of the bucket to sync data into. | +| | Bucket Path | string | Subdirectory under the above bucket to sync the data into. | +| | Region | string | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. | +| | Access Key ID | string | AWS/Minio credential. | +| | Secret Access Key | string | AWS/Minio credential. | | | S3 Filename pattern | string | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. | -| Data Source - Azure | Account Name | string | Name of the account to sync data into. | -| | Container Name | string | Container under the above account to sync the data into. | -| | SAS token | string | Shared-access signature token for the above account. | -| | Endpoint domain name | string | Usually blob.core.windows.net. | +| Data Source - Azure | Account Name | string | Name of the account to sync data into. | +| | Container Name | string | Container under the above account to sync the data into. | +| | SAS token | string | Shared-access signature token for the above account. | +| | Endpoint domain name | string | Usually blob.core.windows.net. | -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you provision a dedicated S3 or Azure resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be +wiped out before each sync. We recommend you provision a dedicated S3 or Azure resource for this +sync to prevent unexpected data deletion from misconfiguration. ⚠️ ## Staging Files (Delta Format) ### S3 -Data streams are first written as staging delta-table ([Parquet](https://parquet.apache.org/) + [Transaction Log](https://databricks.com/blog/2019/08/21/diving-into-delta-lake-unpacking-the-transaction-log.html)) files on S3, and then loaded into Databricks delta-tables. All the staging files will be deleted after the sync is done. For debugging purposes, here is the full path for a staging file: +Data streams are first written as staging delta-table ([Parquet](https://parquet.apache.org/) + +[Transaction Log](https://databricks.com/blog/2019/08/21/diving-into-delta-lake-unpacking-the-transaction-log.html)) +files on S3, and then loaded into Databricks delta-tables. All the staging files will be deleted +after the sync is done. For debugging purposes, here is the full path for a staging file: ```text s3:///// @@ -238,11 +285,15 @@ s3://testing_bucket/data_output_path/98c450be-5b1c-422d-b8b5-6ca9903727d9/users/ ### Azure -Similarly, streams are first written to a staging location, but the Azure option uses CSV format. A staging table is created from the CSV files. +Similarly, streams are first written to a staging location, but the Azure option uses CSV format. A +staging table is created from the CSV files. ## Unmanaged Spark SQL Table -Currently, all streams are synced into unmanaged Spark SQL tables. See [documentation](https://docs.databricks.com/data/tables.html#managed-and-unmanaged-tables) for details. In summary, you have full control of the location of the data underlying an unmanaged table. In S3, the full path of each data stream is: +Currently, all streams are synced into unmanaged Spark SQL tables. See +[documentation](https://docs.databricks.com/data/tables.html#managed-and-unmanaged-tables) for +details. In summary, you have full control of the location of the data underlying an unmanaged +table. In S3, the full path of each data stream is: ```text s3:///// @@ -265,7 +316,8 @@ In Azure, the full path of each data stream is: abfss://@.dfs.core.windows.net// ``` -Please keep these data directories on S3/Azure. Otherwise, the corresponding tables will have no data in Databricks. +Please keep these data directories on S3/Azure. Otherwise, the corresponding tables will have no +data in Databricks. ## Output Schema @@ -277,16 +329,23 @@ Each table will have the following columns: | `_airbyte_emitted_at` | timestamp | Data emission timestamp. | | `_airbyte_data` | JSON | The data from your source will be in this column | -Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the Json to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the +Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet +format. Because the data stream can come from any data source, the Json to Avro conversion process +has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the +current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). ## Related tutorial -Suppose you are interested in learning more about the Databricks connector or details on how the Delta Lake tables are created. You may want to consult the tutorial on [How to Load Data into Delta Lake on Databricks Lakehouse](https://airbyte.com/tutorials/load-data-into-delta-lake-on-databricks-lakehouse). +Suppose you are interested in learning more about the Databricks connector or details on how the +Delta Lake tables are created. You may want to consult the tutorial on +[How to Load Data into Delta Lake on Databricks Lakehouse](https://airbyte.com/tutorials/load-data-into-delta-lake-on-databricks-lakehouse). ## CHANGELOG | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------- | +| 1.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | (incompatible with CDK, do not use) Add new ap-southeast-3 AWS region | | 1.1.0 | 2023-06-02 | [\#26942](https://github.com/airbytehq/airbyte/pull/26942) | Support schema evolution | | 1.0.2 | 2023-04-20 | [\#25366](https://github.com/airbytehq/airbyte/pull/25366) | Fix default catalog to be `hive_metastore` | | 1.0.1 | 2023-03-30 | [\#24657](https://github.com/airbytehq/airbyte/pull/24657) | Fix support for external tables on S3 | diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index 08000797ba53..5143a177f287 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -2,17 +2,22 @@ This destination writes data to AWS DynamoDB. -The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to its own table under the DynamoDB. +The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to +its own table under the DynamoDB. ## Prerequisites -- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer +- For Airbyte Open Source users using the + [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, + [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to + version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer ## Sync overview ### Output schema -Each stream will be output into its own DynamoDB table. Each table will a collections of `json` objects containing 4 fields: +Each stream will be output into its own DynamoDB table. Each table will a collections of `json` +objects containing 4 fields: - `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. @@ -30,13 +35,15 @@ Each stream will be output into its own DynamoDB table. Each table will a collec ### Performance considerations -This connector by default uses 10 capacity units for both Read and Write in DynamoDB tables. Please provision more capacity units in the DynamoDB console when there are performance constraints. +This connector by default uses 10 capacity units for both Read and Write in DynamoDB tables. Please +provision more capacity units in the DynamoDB console when there are performance constraints. ## Getting started ### Requirements -1. Allow connections from Airbyte server to your AWS DynamoDB tables \(if they exist in separate VPCs\). +1. Allow connections from Airbyte server to your AWS DynamoDB tables \(if they exist in separate + VPCs\). 2. The credentials for AWS DynamoDB \(for the COPY strategy\). ### Setup guide @@ -49,19 +56,27 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna - **DynamoDB Region** - The region of the DynamoDB. - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) to the DynamoDB table. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) + to the DynamoDB table. - **Secret Access Key** - Corresponding key to the above key id. - Make sure your DynamoDB tables are accessible from the machine running Airbyte. - This depends on your networking setup. - - You can check AWS DynamoDB documentation with a tutorial on how to properly configure your DynamoDB's access [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html). - - The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check connection tool in the UI. + - You can check AWS DynamoDB documentation with a tutorial on how to properly configure your + DynamoDB's access + [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html). + - The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check + connection tool in the UI. ## CHANGELOG | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | | 0.1.7 | 2022-11-03 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Added strict-encrypt cloud runner | | 0.1.6 | 2022-11-01 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Enforce to use ssl connection | | 0.1.5 | 2022-08-05 | [\#15350](https://github.com/airbytehq/airbyte/pull/15350) | Added per-stream handling | diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 6844931c94c9..2bcc50112ebb 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -46,6 +46,7 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | |:--------|:-----------| :------------------------------------------------------- |:----------------------------------------------------------| +| 0.3.1 | 2024-02-14 | [35278](https://github.com/airbytehq/airbyte/pull/35278) | Adopt CDK 0.20.6 | | 0.3.0 | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Standardize spec and change property field to non-keyword | | 0.2.4 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index f272b77a9d6c..e18228bba87d 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -237,6 +237,8 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | +| 0.4.6 | 2024-02-15 | [35285](https://github.com/airbytehq/airbyte/pull/35285) | Adopt CDK 0.20.8 | +| 0.4.5 | 2024-02-08 | [34745](https://github.com/airbytehq/airbyte/pull/34745) | Adopt CDK 0.19.0 | | 0.4.4 | 2023-07-14 | [#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | | 0.4.3 | 2023-07-05 | [#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal code update | | 0.4.2 | 2023-06-30 | [#27891](https://github.com/airbytehq/airbyte/pull/27891) | Internal code update | diff --git a/docs/integrations/destinations/iceberg.md b/docs/integrations/destinations/iceberg.md index 6b48df61743d..512e7964b649 100644 --- a/docs/integrations/destinations/iceberg.md +++ b/docs/integrations/destinations/iceberg.md @@ -6,10 +6,10 @@ This page guides you through the process of setting up the Iceberg destination c ### Output schema -The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated across different nodes -in the cluster. This connector maps an incoming `stream` to an Iceberg `table` and a `namespace` to an -Iceberg `database`. Fields in the airbyte message become different columns in the Iceberg tables. Each table will -contain the following columns. +The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated +across different nodes in the cluster. This connector maps an incoming `stream` to an Iceberg +`table` and a `namespace` to an Iceberg `database`. Fields in the airbyte message become different +columns in the Iceberg tables. Each table will contain the following columns. - `_airbyte_ab_id`: A random generated uuid. - `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source. @@ -28,14 +28,13 @@ This section should contain a table with the following format: ### Performance considerations -Every ten thousand pieces of incoming airbyte data in a stream ————we call it a batch, would produce one data file( -Parquet/Avro) in an Iceberg table. This batch size can be configurabled by `Data file flushing batch size` -property. -As the quantity of Iceberg data files grows, it causes an unnecessary amount of metadata and less efficient queries from -file open costs. -Iceberg provides data file compaction action to improve this case, you can read more about -compaction [HERE](https://iceberg.apache.org/docs/latest/maintenance/#compact-data-files). -This connector also provides auto compact action when stream closes, by `Auto compact data files` property. Any you can +Every ten thousand pieces of incoming airbyte data in a stream ————we call it a batch, would produce +one data file( Parquet/Avro) in an Iceberg table. This batch size can be configurabled by +`Data file flushing batch size` property. As the quantity of Iceberg data files grows, it causes an +unnecessary amount of metadata and less efficient queries from file open costs. Iceberg provides +data file compaction action to improve this case, you can read more about compaction +[HERE](https://iceberg.apache.org/docs/latest/maintenance/#compact-data-files). This connector also +provides auto compact action when stream closes, by `Auto compact data files` property. Any you can specify the target size of compacted Iceberg data file. ## Getting started @@ -43,24 +42,28 @@ specify the target size of compacted Iceberg data file. ### Requirements - **Iceberg catalog** : Iceberg uses `catalog` to manage tables. this connector already supports: - - [HiveCatalog](https://iceberg.apache.org/docs/latest/hive/#global-hive-catalog) connects to a **Hive metastore** - to keep track of Iceberg tables. - - [HadoopCatalog](https://iceberg.apache.org/docs/latest/java-api-quickstart/#using-a-hadoop-catalog) doesn’t need - to connect to a Hive MetaStore, but can only be used with **HDFS or similar file systems** that support atomic - rename. For `HadoopCatalog`, this connector use **Storage Config** (S3 or HDFS) to manage Iceberg tables. - - [JdbcCatalog](https://iceberg.apache.org/docs/latest/jdbc/) uses a table in a relational database to manage - Iceberg tables through JDBC. So far, this connector supports **PostgreSQL** only. - - [RESTCatalog](https://iceberg.apache.org/docs/latest/spark-configuration/#catalog-configuration) connects to a REST - server, which manages Iceberg tables. -- **Storage medium** means where Iceberg data files storages in. So far, this connector supports **S3/S3N/S3N** - object-storage. When using the RESTCatalog, it is possible to have storage be managed by the server. + - [HiveCatalog](https://iceberg.apache.org/docs/latest/hive/#global-hive-catalog) connects to a + **Hive metastore** to keep track of Iceberg tables. + - [HadoopCatalog](https://iceberg.apache.org/docs/latest/java-api-quickstart/#using-a-hadoop-catalog) + doesn’t need to connect to a Hive MetaStore, but can only be used with **HDFS or similar file + systems** that support atomic rename. For `HadoopCatalog`, this connector use **Storage Config** + (S3 or HDFS) to manage Iceberg tables. + - [JdbcCatalog](https://iceberg.apache.org/docs/latest/jdbc/) uses a table in a relational + database to manage Iceberg tables through JDBC. So far, this connector supports **PostgreSQL** + only. + - [RESTCatalog](https://iceberg.apache.org/docs/latest/spark-configuration/#catalog-configuration) + connects to a REST server, which manages Iceberg tables. +- **Storage medium** means where Iceberg data files storages in. So far, this connector supports + **S3/S3N/S3N** object-storage. When using the RESTCatalog, it is possible to have storage be + managed by the server. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- | :------------- | -| 0.1.4 | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config | -| 0.1.3 | 2023-07-12 | [28158](https://github.com/airbytehq/airbyte/pull/28158) | Bump Iceberg library to 1.3.0 and add REST catalog support | -| 0.1.2 | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Trigger rebuild of image | -| 0.1.1 | 2023-02-27 | [23201](https://github.com/airbytehq/airbyte/pull/23301) | Bump Iceberg library to 1.1.0 | -| 0.1.0 | 2022-11-01 | [18836](https://github.com/airbytehq/airbyte/pull/18836) | Initial Commit | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.5 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.4 | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config | +| 0.1.3 | 2023-07-12 | [28158](https://github.com/airbytehq/airbyte/pull/28158) | Bump Iceberg library to 1.3.0 and add REST catalog support | +| 0.1.2 | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Trigger rebuild of image | +| 0.1.1 | 2023-02-27 | [23201](https://github.com/airbytehq/airbyte/pull/23301) | Bump Iceberg library to 1.1.0 | +| 0.1.0 | 2022-11-01 | [18836](https://github.com/airbytehq/airbyte/pull/18836) | Initial Commit | diff --git a/docs/integrations/destinations/kvdb.md b/docs/integrations/destinations/kvdb.md index 1d9a4341e8fc..cb8ba2c53cb1 100644 --- a/docs/integrations/destinations/kvdb.md +++ b/docs/integrations/destinations/kvdb.md @@ -2,8 +2,27 @@ The KVDB destination for Airbyte +## Prerequisites + +None. + +## Setup guide + +TODO + +## Supported sync modes + +TODO + +## Supported streams + +TODO + ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | -| 0.1.0 | 2021-07-19 | [4786](https://github.com/airbytehq/airbyte/pull/4786) | Python Demo Destination: KVDB | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------- | +| 0.1.3 | 2024-02-19 | [xxx](https://github.com/airbytehq/airbyte/pull/xxx) | bump connector version to publish, convert to base docker image and poetry | +| 0.1.2 | 2024-02-19 | [35422](https://github.com/airbytehq/airbyte/pull/35422) | bump connector version to publish | +| 0.1.1 | 2024-02-16 | [35370](https://github.com/airbytehq/airbyte/pull/35370) | bump connector version to publish | +| 0.1.0 | 2021-07-19 | [4786](https://github.com/airbytehq/airbyte/pull/4786) | Python Demo Destination: KVDB | diff --git a/docs/integrations/destinations/meilisearch.md b/docs/integrations/destinations/meilisearch.md index d7f40201b775..f788f9613057 100644 --- a/docs/integrations/destinations/meilisearch.md +++ b/docs/integrations/destinations/meilisearch.md @@ -33,7 +33,9 @@ The setup only requires two fields. First is the `host` which is the address at | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- | +| 1.0.1 | 2023-12-19 | [27692](https://github.com/airbytehq/airbyte/pull/27692) | Fix incomplete data indexing | | 1.0.0 | 2022-10-26 | [18036](https://github.com/airbytehq/airbyte/pull/18036) | Migrate MeiliSearch to Python CDK | | 0.2.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.2.11 | 2021-12-28 | [9156](https://github.com/airbytehq/airbyte/pull/9156) | Update connector fields title/description | + diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md index 28d619c15683..d2cff5caf09b 100644 --- a/docs/integrations/destinations/milvus.md +++ b/docs/integrations/destinations/milvus.md @@ -109,6 +109,7 @@ vector_store.similarity_search("test") | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.12 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.0.11 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | | 0.0.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | | 0.0.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index 469d24d4fa59..2a19352e8ad7 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -116,6 +116,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +| 0.3.0 | 2023-12-18 | [33468](https://github.com/airbytehq/airbyte/pull/33468) | Upgrade to latest Java CDK | | 0.2.0 | 2023-06-27 | [27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | | 0.1.21 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | diff --git a/docs/integrations/destinations/pinecone.md b/docs/integrations/destinations/pinecone.md index 93f7220cceb0..51dae798ab8e 100644 --- a/docs/integrations/destinations/pinecone.md +++ b/docs/integrations/destinations/pinecone.md @@ -74,6 +74,7 @@ OpenAI and Fake embeddings produce vectors with 1536 dimensions, and the Cohere | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.22 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.0.21 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | | 0.0.20 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.19 | 2023-10-20 | [#31329](https://github.com/airbytehq/airbyte/pull/31373) | Improve error messages | diff --git a/docs/integrations/destinations/postgres-migrations.md b/docs/integrations/destinations/postgres-migrations.md new file mode 100644 index 000000000000..7e9d1a5ba3dd --- /dev/null +++ b/docs/integrations/destinations/postgres-migrations.md @@ -0,0 +1,14 @@ +# Postgres Migration Guide + +## Upgrading to 2.0.0 + +This version introduces [Destinations V2](/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + +Worthy of specific mention, this version includes: + +- Per-record error handling +- Clearer table structure +- Removal of sub-tables for nested properties +- Removal of SCD tables + +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index a05718c145e2..50bd15cc864a 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -82,9 +82,13 @@ From [Postgres SQL Identifiers syntax](https://www.postgresql.org/docs/9.0/sql-s lower case. - In order to make your applications portable and less error-prone, use consistent quoting with each name (either always quote it or never quote it). -Note, that Airbyte Postgres destination will create tables and schemas using the Unquoted -identifiers when possible or fallback to Quoted Identifiers if the names are containing special -characters. +:::info + +Airbyte Postgres destination will create raw tables and schemas using the Unquoted +identifiers by replacing any special characters with an underscore. All final tables and their corresponding +columns are created using Quoted identifiers preserving the case sensitivity. + +::: **For Airbyte Cloud:** @@ -148,17 +152,36 @@ following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s ## Schema map -#### Output Schema +### Output Schema (Raw Tables) -Each stream will be mapped to a separate table in Postgres. Each table will contain 3 columns: +Each stream will be mapped to a separate raw table in Postgres. The default schema in which the raw tables are +created is `airbyte_internal`. This can be overridden in the configuration. +Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in +- `_airbyte_raw_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Postgres is `VARCHAR`. -- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. +- `_airbyte_extracted_at`: a timestamp representing when the event was pulled from the data source. The column type in Postgres is `TIMESTAMP WITH TIME ZONE`. +- `_airbyte_loaded_at`: a timestamp representing when the row was processed into final table. + The column type in Postgres is `TIMESTAMP WITH TIME ZONE`. - `_airbyte_data`: a json blob representing with the event data. The column type in Postgres is `JSONB`. +### Final Tables Data type mapping +| Airbyte Type | Postgres Type | +|:---------------------------|:-------------------------| +| string | VARCHAR | +| number | DECIMAL | +| integer | BIGINT | +| boolean | BOOLEAN | +| object | JSONB | +| array | JSONB | +| timestamp_with_timezone | TIMESTAMP WITH TIME ZONE | +| timestamp_without_timezone | TIMESTAMP | +| time_with_timezone | TIME WITH TIME ZONE | +| time_without_timezone | TIME | +| date | DATE | + ## Tutorials Now that you have set up the Postgres destination connector, check out the following tutorials: @@ -169,7 +192,19 @@ Now that you have set up the Postgres destination connector, check out the follo ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 2.0.1 | 2024-02-22 | [35385](https://github.com/airbytehq/airbyte/pull/35385) | Upgrade CDK to 0.23.0; Gathering required initial state upfront | +| 2.0.0 | 2024-02-09 | [35042](https://github.com/airbytehq/airbyte/pull/35042) | GA release V2 destinations format. | +| 0.6.3 | 2024-02-06 | [34891](https://github.com/airbytehq/airbyte/pull/34891) | Remove varchar limit, use system defaults | +| 0.6.2 | 2024-01-30 | [34683](https://github.com/airbytehq/airbyte/pull/34683) | CDK Upgrade 0.16.3; Fix dependency mismatches in slf4j lib | +| 0.6.1 | 2024-01-29 | [34630](https://github.com/airbytehq/airbyte/pull/34630) | CDK Upgrade; Use lowercase raw table in T+D queries. | +| 0.6.0 | 2024-01-19 | [34372](https://github.com/airbytehq/airbyte/pull/34372) | Add dv2 flag in spec | +| 0.5.5 | 2024-01-18 | [34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.1; Add indexes in raw table for query optimization | +| 0.5.4 | 2024-01-11 | [34177](https://github.com/airbytehq/airbyte/pull/34177) | Add code for DV2 beta (no user-visible changes) | +| 0.5.3 | 2024-01-10 | [34135](https://github.com/airbytehq/airbyte/pull/34135) | Use published CDK missed in previous release | +| 0.5.2 | 2024-01-08 | [33875](https://github.com/airbytehq/airbyte/pull/33875) | Update CDK to get Tunnel heartbeats feature | +| 0.5.1 | 2024-01-04 | [33873](https://github.com/airbytehq/airbyte/pull/33873) | Install normalization to enable DV2 beta | +| 0.5.0 | 2023-12-18 | [33507](https://github.com/airbytehq/airbyte/pull/33507) | Upgrade to latest CDK; Fix DATs and tests | | 0.4.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | | 0.3.27 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | | 0.3.26 | 2022-09-27 | [\#17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres destination | diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md index f7b47a096431..537df671fbd5 100644 --- a/docs/integrations/destinations/qdrant.md +++ b/docs/integrations/destinations/qdrant.md @@ -71,6 +71,7 @@ You should now have all the requirements needed to configure Qdrant as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.10 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.0.9 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | | 0.0.8 | 2023-11-29 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources and fix spec schema | | 0.0.7 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | diff --git a/docs/integrations/destinations/redshift-migrations.md b/docs/integrations/destinations/redshift-migrations.md new file mode 100644 index 000000000000..59d91b557f86 --- /dev/null +++ b/docs/integrations/destinations/redshift-migrations.md @@ -0,0 +1,14 @@ +# Redshift Migration Guide + +## Upgrading to 2.0.0 + +This version introduces [Destinations V2](/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + +Worthy of specific mention, this version includes: + +- Per-record error handling +- Clearer table structure +- Removal of sub-tables for nested properties +- Removal of SCD tables + +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index dc9524b864f4..c37e01dcbb67 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -8,7 +8,12 @@ The Airbyte Redshift destination allows you to sync data to Redshift. This Redshift destination connector has two replication strategies: -1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Mulesoft [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in Redshift documentation [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended for production workloads as this does not scale well**. +1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code + base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Maven Central + [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in + Redshift documentation + [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended + for production workloads as this does not scale well**. For INSERT strategy: @@ -21,77 +26,130 @@ For INSERT strategy: - This database needs to exist within the cluster provided. - **JDBC URL Params** (optional) -2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is the recommended loading approach described by Redshift [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_loading-data-best-practices.html). Requires an S3 bucket and credentials. +2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is + the recommended loading approach described by Redshift + [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_best-practices-single-copy-command.html). + Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file. -Airbyte automatically picks an approach depending on the given configuration - if S3 configuration is present, Airbyte will use the COPY strategy and vice versa. +Airbyte automatically picks an approach depending on the given configuration - if S3 configuration +is present, Airbyte will use the COPY strategy and vice versa. For COPY strategy: - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to + create an S3 bucket. - **S3 Bucket Region** - Place the S3 bucket and the Redshift cluster in the same region to save on networking costs. - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. - **Secret Access Key** - Corresponding key to the above key id. - **Part Size** - - Affects the size limit of an individual Redshift table. Optional. Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care. + - Affects the size limit of an individual Redshift table. Optional. Increase this if syncing + tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each + part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This + is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will + result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get + the memory requirement. Modify this with care. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders + combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. + Please, don't use empty space and not supportable placeholders, as they won't recognized. Optional parameters: - **Bucket Path** - - The directory within the S3 bucket to place the staging data. For example, if you set this to `yourFavoriteSubdirectory`, we will place the staging data inside `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory. + - The directory within the S3 bucket to place the staging data. For example, if you set this to + `yourFavoriteSubdirectory`, we will place the staging data inside + `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory. - **Purge Staging Data** - - Whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. + - Whether to delete the staging files from S3 after completing the sync. Specifically, the + connector will create CSV files named + `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns + (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command + completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. -NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel supports the SQL connection only. S3 is secured through public HTTPS access only. +NOTE: S3 staging does not use the SSH Tunnel option for copying data, if configured. SSH Tunnel supports the SQL +connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table +are executed over using provided SSH Tunnel configuration. ## Step 1: Set up Redshift -1. [Log in](https://aws.amazon.com/console/) to AWS Management console. - If you don't have a AWS account already, you’ll need to [create](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) one in order to use the API. -2. Go to the AWS Redshift service -3. [Create](https://docs.aws.amazon.com/ses/latest/dg/event-publishing-redshift-cluster.html) and activate AWS Redshift cluster if you don't have one ready -4. (Optional) [Allow](https://aws.amazon.com/premiumsupport/knowledge-center/cannot-connect-redshift-cluster/) connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\) -5. (Optional) [Create](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) a staging S3 bucket \(for the COPY strategy\). -6. Create a user with at least create table permissions for the schema. If the schema does not exist you need to add permissions for that, too. Something like this: - -``` -GRANT CREATE ON DATABASE database_name TO airflow_user; -- add create schema permission -GRANT usage, create on schema my_schema TO airflow_user; -- add create table permission +1. [Log in](https://aws.amazon.com/console/) to AWS Management console. If you don't have a AWS + account already, you’ll need to + [create](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) + one in order to use the API. +2. Go to the AWS Redshift service. +3. [Create](https://docs.aws.amazon.com/ses/latest/dg/event-publishing-redshift-cluster.html) and + activate AWS Redshift cluster if you don't have one ready. +4. (Optional) + [Allow](https://aws.amazon.com/premiumsupport/knowledge-center/cannot-connect-redshift-cluster/) + connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\). +5. (Optional) + [Create](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) a + staging S3 bucket \(for the COPY strategy\). + +### Permissions in Redshift +Airbyte writes data into two schemas, whichever schema you want your data to land in, e.g. `my_schema` +and a "Raw Data" schema that Airbyte uses to improve ELT reliability. By default, this raw data schema +is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. +Airbyte also needs to query Redshift's +[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for +metadata about the tables airbyte manages. + +To ensure the `airbyte_user` has the correction permissions to: +- create schemas in your database +- grant usage to any existing schemas you want Airbyte to use +- grant select to the `svv_table_info` table + +You can execute the following SQL statements + +```sql +GRANT CREATE ON DATABASE database_name TO airbyte_user; -- add create schema permission +GRANT usage, create on schema my_schema TO airbyte_user; -- add create table permission +GRANT SELECT ON TABLE SVV_TABLE_INFO TO airbyte_user; -- add select permission for svv_table_info ``` ### Optional Use of SSH Bastion Host -This connector supports the use of a Bastion host as a gateway to a private Redshift cluster via SSH Tunneling. -Setup of the host is beyond the scope of this document but several tutorials are available online to fascilitate this task. -Enter the bastion host, port and credentials in the destination configuration. +This connector supports the use of a Bastion host as a gateway to a private Redshift cluster via SSH +Tunneling. Setup of the host is beyond the scope of this document but several tutorials are +available online to fascilitate this task. Enter the bastion host, port and credentials in the +destination configuration. ## Step 2: Set up the destination connector in Airbyte **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a name for this connector. -4. Fill in all the required fields to use the INSERT or COPY strategy +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a + name for this connector. +4. Fill in all the required fields to use the INSERT or COPY strategy. 5. Click `Set up destination`. **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a name for this connector. -4. Fill in all the required fields to use the INSERT or COPY strategy +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a + name for this connector. +4. Fill in all the required fields to use the INSERT or COPY strategy. 5. Click `Set up destination`. ## Supported sync modes -The Redshift destination connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): +The Redshift destination connector supports the following +[sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): - Full Refresh - Incremental - Append Sync @@ -99,8 +157,8 @@ The Redshift destination connector supports the following [sync modes](https://d ## Performance considerations -Synchronization performance depends on the amount of data to be transferred. -Cluster scaling issues can be resolved directly using the cluster settings in the AWS Redshift console +Synchronization performance depends on the amount of data to be transferred. Cluster scaling issues +can be resolved directly using the cluster settings in the AWS Redshift console. ## Connector-specific features & highlights @@ -110,58 +168,104 @@ From [Redshift Names & Identifiers](https://docs.aws.amazon.com/redshift/latest/ #### Standard Identifiers -- Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte character two to four bytes long. -- Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar signs, or UTF-8 multibyte characters two to four bytes long. +- Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte + character two to four bytes long. +- Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar + signs, or UTF-8 multibyte characters two to four bytes long. - Be between 1 and 127 bytes in length, not including quotation marks for delimited identifiers. - Contain no quotation marks and no spaces. #### Delimited Identifiers -Delimited identifiers \(also known as quoted identifiers\) begin and end with double quotation marks \("\). If you use a delimited identifier, you must use the double quotation marks for every reference to that object. The identifier can contain any standard UTF-8 printable characters other than the double quotation mark itself. Therefore, you can create column or table names that include otherwise illegal characters, such as spaces or the percent symbol. ASCII letters in delimited identifiers are case-insensitive and are folded to lowercase. To use a double quotation mark in a string, you must precede it with another double quotation mark character. +Delimited identifiers \(also known as quoted identifiers\) begin and end with double quotation marks +\("\). If you use a delimited identifier, you must use the double quotation marks for every +reference to that object. The identifier can contain any standard UTF-8 printable characters other +than the double quotation mark itself. Therefore, you can create column or table names that include +otherwise illegal characters, such as spaces or the percent symbol. ASCII letters in delimited +identifiers are case-insensitive and are folded to lowercase. To use a double quotation mark in a +string, you must precede it with another double quotation mark character. -Therefore, Airbyte Redshift destination will create tables and schemas using the Unquoted identifiers when possible or fallback to Quoted Identifiers if the names are containing special characters. +Therefore, Airbyte Redshift destination will create tables and schemas using the Unquoted +identifiers when possible or fallback to Quoted Identifiers if the names are containing special +characters. ### Data Size Limitations -Redshift specifies a maximum limit of 1MB (and 65535 bytes for any VARCHAR fields within the JSON record) to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record. -See docs for [SUPER](https://docs.aws.amazon.com/redshift/latest/dg/r_SUPER_type.html) and [SUPER limitations](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html) +Redshift specifies a maximum limit of 16MB (and 65535 bytes for any VARCHAR fields within the JSON +record) to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift +destination fails to load such data and currently ignores that record. See docs for +[SUPER](https://docs.aws.amazon.com/redshift/latest/dg/r_SUPER_type.html) and +[SUPER limitations](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html). ### Encryption -All Redshift connections are encrypted using SSL +All Redshift connections are encrypted using SSL. ### Output schema Each stream will be output into its own raw table in Redshift. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Redshift is `VARCHAR`. -- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. -- `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is `SUPER`. - -## Data type mapping - -| Redshift Type | Airbyte Type | Notes | -|:----------------------|:--------------------------|:------| -| `boolean` | `boolean` | | -| `int` | `integer` | | -| `float` | `number` | | -| `varchar` | `string` | | -| `date/varchar` | `date` | | -| `time/varchar` | `time` | | -| `timestamptz/varchar` | `timestamp_with_timezone` | | -| `varchar` | `array` | | -| `varchar` | `object` | | +- `_airbyte_raw_id`: a uuid assigned by Airbyte to each event that is processed. The column type in + Redshift is `VARCHAR`. +- `_airbyte_extracted_at`: a timestamp representing when the event was pulled from the data source. + The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. +- `_airbyte_loaded_at`: a timestamp representing when the row was processed into final table. + The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. +- `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is + `SUPER`. + +## Data type map + +| Airbyte type | Redshift type | +|:------------------------------------|:---------------------------------------| +| STRING | VARCHAR | +| STRING (BASE64) | VARCHAR | +| STRING (BIG_NUMBER) | VARCHAR | +| STRING (BIG_INTEGER) | VARCHAR | +| NUMBER | DECIMAL / NUMERIC | +| INTEGER | BIGINT / INT8 | +| BOOLEAN | BOOLEAN / BOOL | +| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMPTZ / TIMESTAMP WITH TIME ZONE | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP | +| STRING (TIME_WITH_TIMEZONE) | TIMETZ / TIME WITH TIME ZONE | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | SUPER | +| ARRAY | SUPER | ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.7.3 | 2023-12-12 | [33367](https://github.com/airbytehq/airbyte/pull/33367) | DV2: fix migration logic | -| 0.7.2 | 2023-12-11 | [33335](https://github.com/airbytehq/airbyte/pull/33335) | DV2: improve data type mapping | -| 0.7.1 | 2023-12-11 | [33307](https://github.com/airbytehq/airbyte/pull/33307) | ~DV2: improve data type mapping~ No changes | -| 0.7.0 | 2023-12-05 | [32326](https://github.com/airbytehq/airbyte/pull/32326) | Opt in beta for v2 destination | -| 0.6.11 | 2023-11-29 | [#32888](https://github.com/airbytehq/airbyte/pull/32888) | Use the new async framework. | -| 0.6.10 | 2023-11-06 | [#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 2.1.8 | 2024-02-09 | [\#35354](https://github.com/airbytehq/airbyte/pull/35354) | Update to CDK 0.23.0; Gather required initial state upfront, remove dependency on svv_table_info for table empty check | +| 2.1.7 | 2024-02-09 | [\#34562](https://github.com/airbytehq/airbyte/pull/34562) | Switch back to jooq-based sql execution for standard insert | +| 2.1.6 | 2024-02-08 | [\#34502](https://github.com/airbytehq/airbyte/pull/34502) | Update to CDK version 0.17.0 | +| 2.1.5 | 2024-01-30 | [\#34680](https://github.com/airbytehq/airbyte/pull/34680) | Update to CDK version 0.16.3 | +| 2.1.4 | 2024-01-29 | [\#34634](https://github.com/airbytehq/airbyte/pull/34634) | Use lowercase raw schema and table in T+D [CDK changes](https://github.com/airbytehq/airbyte/pull/34533) | +| 2.1.3 | 2024-01-26 | [\#34544](https://github.com/airbytehq/airbyte/pull/34544) | Proper string-escaping in raw tables | +| 2.1.2 | 2024-01-24 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 2.1.1 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 2.1.0 | 2024-01-24 | [\#34467](https://github.com/airbytehq/airbyte/pull/34467) | Upgrade CDK to 0.14.0 | +| 2.0.0 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2 | +| 0.8.0 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | +| 0.7.15 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Update check method with svv_table_info permission check, fix bug where s3 staging files were not being deleted. | +| 0.7.14 | 2024-01-08 | [\#34014](https://github.com/airbytehq/airbyte/pull/34014) | Update order of options in spec | +| 0.7.13 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Fix NPE when prepare tables fail; Add case sensitive session for super; Bastion heartbeats added | +| 0.7.12 | 2024-01-03 | [\#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.7.11 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 0.7.10 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Allow users to disable final table creation | +| 0.7.9 | 2024-01-03 | [\#33877](https://github.com/airbytehq/airbyte/pull/33877) | Fix Jooq StackOverflowError | +| 0.7.8 | 2023-12-28 | [\#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names (s3 staging files) | +| 0.7.7 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | +| 0.7.6 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Only run T+D on a stream if it had any records during the sync | +| 0.7.5 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | +| 0.7.4 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Use jdbc common sql implementation | +| 0.7.3 | 2023-12-12 | [\#33367](https://github.com/airbytehq/airbyte/pull/33367) | DV2: fix migration logic | +| 0.7.2 | 2023-12-11 | [\#33335](https://github.com/airbytehq/airbyte/pull/33335) | DV2: improve data type mapping | +| 0.7.1 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | ~DV2: improve data type mapping~ No changes | +| 0.7.0 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Opt in beta for v2 destination | +| 0.6.11 | 2023-11-29 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | Use the new async framework. | +| 0.6.10 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | | 0.6.9 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | | 0.6.8 | 2023-10-10 | [\#31218](https://github.com/airbytehq/airbyte/pull/31218) | Clarify configuration groups | | 0.6.7 | 2023-10-06 | [\#31153](https://github.com/airbytehq/airbyte/pull/31153) | Increase jvm GC retries | diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md index 56dbf42c2bbd..ec0a2ac1bd6c 100644 --- a/docs/integrations/destinations/s3-glue.md +++ b/docs/integrations/destinations/s3-glue.md @@ -14,52 +14,74 @@ List of required fields: - **Glue database** - **Glue serialization library** -1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or an instance profile with read/write permissions configured for the host (ec2, eks). +1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in + separate VPCs\). +2. An S3 bucket with credentials or an instance profile with read/write permissions configured for + the host (ec2, eks). 3. [Enforce encryption of data in transit](https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-best-practices.html#transit) 4. Allow permissions to access the AWS Glue service from the Airbyte connector ## Step 1: Set up S3 -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare S3 bucket that will be used as destination, see [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. +Prepare S3 bucket that will be used as destination, see +[this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create +an S3 bucket. -NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as support for AWS's [shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) +NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts +to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as +support for AWS's +[shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) ## Step 2: Set up Glue -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare the Glue database that will be used as destination, see [this](https://docs.aws.amazon.com/glue/latest/dg/console-databases.html) to create a Glue database +Prepare the Glue database that will be used as destination, see +[this](https://docs.aws.amazon.com/glue/latest/dg/console-databases.html) to create a Glue database ## Step 3: Set up the S3-Glue destination connector in Airbyte **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. - **Secret Access Key** - Corresponding key to the above key id. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - **S3 Bucket Region** - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't recognized. - **Glue database** - The Glue database name that was previously created through the management console or the cli. - **Glue serialization library** @@ -69,34 +91,55 @@ Prepare the Glue database that will be used as destination, see [this](https://d **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. - - If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - See + [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) + on how to create a instanceprofile. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. + - If the Access Key and Secret Access Key are not provided, the authentication will rely on the + instanceprofile. - **Secret Access Key** - Corresponding key to the above key id. - Make sure your S3 bucket is accessible from the machine running Airbyte. - This depends on your networking setup. - - You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). - - If you use instance profile authentication, make sure the role has permission to read/write on the bucket. - - The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. + - You can check AWS S3 documentation with a tutorial on how to properly configure your S3's + access + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). + - If you use instance profile authentication, make sure the role has permission to read/write + on the bucket. + - The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check + connection tool in the UI. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - **S3 Bucket Region** - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't recognized. - **Glue database** - The Glue database name that was previously created through the management console or the cli. - **Glue serialization library** @@ -104,7 +147,8 @@ Prepare the Glue database that will be used as destination, see [this](https://d 5. Click `Set up destination`. -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Policies to use: ```json { @@ -113,18 +157,18 @@ In order for everything to work correctly, it is also necessary that the user wh { "Effect": "Allow", "Action": "s3:*", - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } ``` -For setting up the necessary Glue policies see [this](https://docs.aws.amazon.com/glue/latest/dg/glue-resource-policies.html) and [this](https://docs.aws.amazon.com/glue/latest/dg/create-service-policy.html) +For setting up the necessary Glue policies see +[this](https://docs.aws.amazon.com/glue/latest/dg/glue-resource-policies.html) and +[this](https://docs.aws.amazon.com/glue/latest/dg/create-service-policy.html) -The full path of the output data with the default S3 Path Format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: +The full path of the output data with the default S3 Path Format +`${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: ```text ///__. @@ -153,7 +197,8 @@ The rationales behind this naming pattern are: But it is possible to further customize by using the available variables to format the bucket path: -- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields. +- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace + fields. - `${STREAM_NAME}`: Name of the stream - `${YEAR}`: Year in which the sync was writing the output data in. - `${MONTH}`: Month in which the sync was writing the output data in. @@ -168,33 +213,41 @@ But it is possible to further customize by using the available variables to form Note: - Multiple `/` characters in the S3 path are collapsed into a single `/` character. -- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise. +- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It + uses sequential ID otherwise. -Please note that the stream name may contain a prefix, if it is configured on the connection. -A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . +Please note that the stream name may contain a prefix, if it is configured on the connection. A data +sync may create multiple files as the output files can be partitioned by size (targeting a size of +200MB compressed or lower) . ## Supported sync modes -| Feature | Support | Notes | -| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | +| Feature | Support | Notes | +| :----------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | | Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/using-airbyte/core-concepts/sync-modes/incremental-append#inclusive-cursors) | -| Incremental - Append + Deduped | ❌ | | -| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | +| Incremental - Append + Deduped | ❌ | | +| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | -The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to +its own directory under the bucket. ⚠️ Please note that under "Full Refresh Sync" mode, data in the +configured bucket and path will be wiped out before each sync. We recommend you to provision a +dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ ## Supported Output schema -Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world. +Each stream will be outputted to its dedicated directory according to the configuration. The +complete datastore of each stream includes all the output files under that directory. You can think +of the directory as equivalent of a Table in the database world. - Under Full Refresh Sync mode, old output files will be purged before new files are created. -- Under Incremental - Append Sync mode, new output files will be added that only contain the new data. +- Under Incremental - Append Sync mode, new output files will be added that only contain the new + data. ### JSON Lines \(JSONL\) -[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: +[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a +structure as follows: ```json { @@ -225,7 +278,8 @@ For example, given the following two json objects from a source: ] ``` -depending on whether you want to flatten your data or not (**_available as a configuration option_**) +depending on whether you want to flatten your data or not (**_available as a configuration +option_**) The json objects can have the following formats: @@ -239,17 +293,19 @@ The json objects can have the following formats: { "_airbyte_ab_id": "0a61de1b-9cdd-4455-a739-93572c9a5f20", "_airbyte_emitted_at": "1631948170000", "user_id": 456, "name": { "first": "Jane", "last": "Roe" } } ``` -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.jsonl.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.jsonl.gz`). ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------- | -| 0.1.7 | 2023-05-01 | [25724](https://github.com/airbytehq/airbyte/pull/25724) | Fix decimal type creation syntax to avoid overflow | -| 0.1.6 | 2023-04-13 | [25178](https://github.com/airbytehq/airbyte/pull/25178) | Fix decimal precision and scale to allow for a wider range of numeric values | -| 0.1.5 | 2023-04-11 | [25048](https://github.com/airbytehq/airbyte/pull/25048) | Fix config schema to support new JSONL flattening configuration interface | -| 0.1.4 | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields | -| 0.1.3 | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for \_ab_emitted_at column in table definition | -| 0.1.2 | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition | -| 0.1.1 | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema | -| 0.1.0 | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------------------------------------------------------------- | +| 0.1.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.7 | 2023-05-01 | [25724](https://github.com/airbytehq/airbyte/pull/25724) | Fix decimal type creation syntax to avoid overflow | +| 0.1.6 | 2023-04-13 | [25178](https://github.com/airbytehq/airbyte/pull/25178) | Fix decimal precision and scale to allow for a wider range of numeric values | +| 0.1.5 | 2023-04-11 | [25048](https://github.com/airbytehq/airbyte/pull/25048) | Fix config schema to support new JSONL flattening configuration interface | +| 0.1.4 | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields | +| 0.1.3 | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for \_ab_emitted_at column in table definition | +| 0.1.2 | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition | +| 0.1.1 | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema | +| 0.1.0 | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index 2b0e902b7eaa..d29d32464c4f 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -12,20 +12,27 @@ List of required fields: - **S3 Bucket Path** - **S3 Bucket Region** -1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or an instance profile with read/write permissions configured for the host (ec2, eks). +1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in + separate VPCs\). +2. An S3 bucket with credentials or an instance profile with read/write permissions configured for + the host (ec2, eks). 3. [Enforce encryption of data in transit](https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-best-practices.html#transit) ## Setup guide ### Step 1: Set up S3 -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare S3 bucket that will be used as destination, see [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. +Prepare S3 bucket that will be used as destination, see +[this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create +an S3 bucket. -NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as support for AWS's [shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) +NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts +to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as +support for AWS's +[shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) ### Step 2: Set up the S3 destination connector in Airbyte @@ -34,26 +41,40 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. - **Secret Access Key** - Corresponding key to the above key id. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - **S3 Bucket Region**: - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't be recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't be recognized. 5. Click `Set up destination`. @@ -62,38 +83,46 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. -4. Configure fields: - _ **Access Key Id** - _ See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - _ See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. - _ We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. - _ If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. - _ **Secret Access Key** - _ Corresponding key to the above key id. - _ Make sure your S3 bucket is accessible from the machine running Airbyte. - _ This depends on your networking setup. - _ You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). - _ If you use instance profile authentication, make sure the role has permission to read/write on the bucket. - _ The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. - _ **S3 Bucket Name** - _ See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. - _ **S3 Bucket Path** - _ Subdirectory under the above bucket to sync the data into. - _ **S3 Bucket Region** - _ See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. - _ **S3 Path Format** - _ Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - _ **S3 Endpoint** - _ Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - - - **S3 Filename pattern** \* The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. +4. Configure fields: _ **Access Key Id** _ See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. _ See + [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) + on how to create a instanceprofile. _ We recommend creating an Airbyte-specific user. This user + will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. _ If the Access Key and Secret Access Key are not provided, the + authentication will rely on the instanceprofile. _ **Secret Access Key** _ Corresponding key to + the above key id. _ Make sure your S3 bucket is accessible from the machine running Airbyte. _ + This depends on your networking setup. _ You can check AWS S3 documentation with a tutorial on + how to properly configure your S3's access + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). _ If + you use instance profile authentication, make sure the role has permission to read/write on the + bucket. _ The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the + check connection tool in the UI. _ **S3 Bucket Name** _ See + [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to + create an S3 bucket. _ **S3 Bucket Path** _ Subdirectory under the above bucket to sync the data + into. _ **S3 Bucket Region** _ See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. _ **S3 Path Format** _ Additional string format on how to store data under + S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + _ **S3 Endpoint** _ Leave empty if using AWS S3, fill in S3 URL if using Minio S3. + + - **S3 Filename pattern** \* The pattern allows you to set the file-name format for the S3 + staging file(s), next placeholders combinations are currently supported: `{date}`, + `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, + `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable + placeholders, as they won't recognized. 5. Click `Set up destination`. -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies +to use: ```json { @@ -111,16 +140,14 @@ In order for everything to work correctly, it is also necessary that the user wh "s3:AbortMultipartUpload", "s3:GetBucketLocation" ], - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } ``` -The full path of the output data with the default S3 Path Format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: +The full path of the output data with the default S3 Path Format +`${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: ```text ///__. @@ -149,7 +176,8 @@ The rationales behind this naming pattern are: But it is possible to further customize by using the available variables to format the bucket path: -- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields. +- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace + fields. - `${STREAM_NAME}`: Name of the stream - `${YEAR}`: Year in which the sync was writing the output data in. - `${MONTH}`: Month in which the sync was writing the output data in. @@ -164,10 +192,12 @@ But it is possible to further customize by using the available variables to form Note: - Multiple `/` characters in the S3 path are collapsed into a single `/` character. -- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise. +- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It + uses sequential ID otherwise. -Please note that the stream name may contain a prefix, if it is configured on the connection. -A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . +Please note that the stream name may contain a prefix, if it is configured on the connection. A data +sync may create multiple files as the output files can be partitioned by size (targeting a size of +200MB compressed or lower) . ## Supported sync modes @@ -178,20 +208,29 @@ A data sync may create multiple files as the output files can be partitioned by | Incremental - Append + Deduped | ❌ | | | Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | -The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. +The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to +its own directory under the bucket. -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be +wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to +prevent unexpected data deletion from misconfiguration. ⚠️ ## Supported Output schema -Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world. +Each stream will be outputted to its dedicated directory according to the configuration. The +complete datastore of each stream includes all the output files under that directory. You can think +of the directory as equivalent of a Table in the database world. - Under Full Refresh Sync mode, old output files will be purged before new files are created. -- Under Incremental - Append Sync mode, new output files will be added that only contain the new data. +- Under Incremental - Append Sync mode, new output files will be added that only contain the new + data. ### Avro -[Apache Avro](https://avro.apache.org/) serializes data in a compact binary format. Currently, the Airbyte S3 Avro connector always uses the [binary encoding](http://avro.apache.org/docs/current/spec.html#binary_encoding), and assumes that all data records follow the same schema. +[Apache Avro](https://avro.apache.org/) serializes data in a compact binary format. Currently, the +Airbyte S3 Avro connector always uses the +[binary encoding](http://avro.apache.org/docs/current/spec.html#binary_encoding), and assumes that +all data records follow the same schema. #### Configuration @@ -209,7 +248,9 @@ Here is the available compression codecs: - Range `[0, 9]`. Default to 6. - Level 0-3 are fast with medium compression. - Level 4-6 are fairly slow with high compression. - - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively. + - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. + Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of + memory to use the presets 7, 8, or 9, respectively. - `zstandard` - Compression level - Range `[-5, 22]`. Default to 3. @@ -222,11 +263,17 @@ Here is the available compression codecs: #### Data schema -Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the +JSON object is converted to an Avro record. Because the data stream can come from any data source, +the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source +data is converted to Avro and the current limitations +[here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). ### CSV -Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns. +Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, +an emission timestamp, and the data blob. With the CSV output, it is possible to normalize +\(flatten\) the data blob to multiple columns. | Column | Condition | Description | | :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- | @@ -259,11 +306,13 @@ With root level normalization, the output CSV is: | :------------------------------------- | :-------------------- | :-------- | :----------------------------------- | | `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` | -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.csv.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.csv.gz`). ### JSON Lines \(JSONL\) -[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: +[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a +structure as follows: ```json { @@ -301,7 +350,8 @@ They will be like this in the output file: { "_airbyte_ab_id": "0a61de1b-9cdd-4455-a739-93572c9a5f20", "_airbyte_emitted_at": "1631948170000", "_airbyte_data": { "user_id": 456, "name": { "first": "Jane", "last": "Roe" } } } ``` -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.jsonl.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.jsonl.gz`). ### Parquet @@ -318,13 +368,22 @@ The following configuration is available to configure the Parquet output: | `dictionary_page_size_kb` | integer | 1024 \(KB\) | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. | | `dictionary_encoding` | boolean | `true` | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on. | -These parameters are related to the `ParquetOutputFormat`. See the [Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) for more details. Also see [Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their recommended configurations \(512 - 1024 MB block size, 8 KB page size\). +These parameters are related to the `ParquetOutputFormat`. See the +[Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) +for more details. Also see +[Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their +recommended configurations \(512 - 1024 MB block size, 8 KB page size\). #### Data schema -Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the +JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet +format. Because the data stream can come from any data source, the JSON to Avro conversion process +has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the +current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Policies to use: ```json { @@ -333,10 +392,7 @@ In order for everything to work correctly, it is also necessary that the user wh { "Effect": "Allow", "Action": "s3:*", - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } @@ -346,6 +402,9 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.5.9 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 0.5.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.5.7 | 2023-12-28 | [#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names | | 0.5.6 | 2023-12-08 | [#33263](https://github.com/airbytehq/airbyte/pull/33263) | (incorrect filename format, do not use) Adopt java CDK version 0.7.0. | | 0.5.5 | 2023-12-08 | [#33264](https://github.com/airbytehq/airbyte/pull/33264) | Update UI options with common defaults. | | 0.5.4 | 2023-11-06 | [#32193](https://github.com/airbytehq/airbyte/pull/32193) | (incorrect filename format, do not use) Adopt java CDK version 0.4.1. | diff --git a/docs/integrations/destinations/snowflake-migrations.md b/docs/integrations/destinations/snowflake-migrations.md index 2da5fe727be8..adb75e5126e9 100644 --- a/docs/integrations/destinations/snowflake-migrations.md +++ b/docs/integrations/destinations/snowflake-migrations.md @@ -11,7 +11,7 @@ Worthy of specific mention, this version includes: - Removal of sub-tables for nested properties - Removal of SCD tables -Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). +Learn more about what's new in Destinations V2 [here](/using-airbyte/core-concepts/typing-deduping). ## Upgrading to 2.0.0 diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index da563ad4105d..39be90148e99 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -128,7 +128,7 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent ### Login and Password | Field | Description | -| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|-------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | @@ -142,7 +142,7 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent ### OAuth 2.0 | Field | Description | -| :---------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | @@ -183,7 +183,7 @@ Airbyte outputs each stream into its own raw table in `airbyte_internal` schema ### Raw Table schema | Airbyte field | Description | Column type | -| ---------------------- | ------------------------------------------------------------------ | ------------------------ | +|------------------------|--------------------------------------------------------------------|--------------------------| | \_airbyte_raw_id | A UUID assigned to each processed event | VARCHAR | | \_airbyte_extracted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE | | \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP WITH TIME ZONE | @@ -193,6 +193,25 @@ Airbyte outputs each stream into its own raw table in `airbyte_internal` schema **Note:** By default, Airbyte creates permanent tables. If you prefer transient tables, create a dedicated transient database for Airbyte. For more information, refer to[ Working with Temporary and Transient Tables](https://docs.snowflake.com/en/user-guide/tables-temp-transient.html) +## Data type map + +| Airbyte type | Snowflake type | +|:------------------------------------|:---------------| +| STRING | TEXT | +| STRING (BASE64) | TEXT | +| STRING (BIG_NUMBER) | TEXT | +| STRING (BIG_INTEGER) | TEXT | +| NUMBER | FLOAT | +| INTEGER | NUMBER | +| BOOLEAN | BOOLEAN | +| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMP_TZ | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP_NTZ | +| STRING (TIME_WITH_TIMEZONE) | TEXT | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | OBJECT | +| ARRAY | ARRAY | + ## Supported sync modes The Snowflake destination supports the following sync modes: @@ -227,9 +246,32 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.4.14 | 2023-12-08 | [33263](https://github.com/airbytehq/airbyte/pull/33263) | Adopt java CDK version 0.7.0 | -| 3.4.13 | 2023-12-05 | [32326](https://github.com/airbytehq/airbyte/pull/32326) | Use jdbc metadata for table existence check | -| 3.4.12 | 2023-12-04 | [33084](https://github.com/airbytehq/airbyte/pull/33084) | T&D SQL statements moved to debug log level | +| 3.5.14 | 2024-02-22 | [35456](https://github.com/airbytehq/airbyte/pull/35456) | Adopt CDK 0.23.0; Gather initial state upfront, reduce information_schema calls | +| 3.5.13 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 3.5.12 | 2024-02-15 | [35240](https://github.com/airbytehq/airbyte/pull/35240) | Adopt CDK 0.20.9 | +| 3.5.11 | 2024-02-12 | [35194](https://github.com/airbytehq/airbyte/pull/35194) | Reorder auth options | +| 3.5.10 | 2024-02-12 | [35144](https://github.com/airbytehq/airbyte/pull/35144) | Adopt CDK 0.20.2 | +| 3.5.9 | 2024-02-12 | [35111](https://github.com/airbytehq/airbyte/pull/35111) | Adopt CDK 0.20.1 | +| 3.5.8 | 2024-02-09 | [34574](https://github.com/airbytehq/airbyte/pull/34574) | Adopt CDK 0.20.0 | +| 3.5.7 | 2024-02-08 | [34747](https://github.com/airbytehq/airbyte/pull/34747) | Adopt CDK 0.19.0 | +| 3.5.6 | 2024-02-08 | [\#35027](https://github.com/airbytehq/airbyte/pull/35027) | Upgrade CDK to version 0.17.1 | +| 3.5.5 | 2024-02-08 | [\#34502](https://github.com/airbytehq/airbyte/pull/34502) | Reduce COPY frequency | +| 3.5.4 | 2024-01-24 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 3.5.3 | 2024-01-25 | [\#34528](https://github.com/airbytehq/airbyte/pull/34528) | Fix spurious `check` failure (`UnsupportedOperationException: Snowflake does not use the native JDBC DV2 interface`) | +| 3.5.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 3.5.1 | 2024-01-24 | [\#34501](https://github.com/airbytehq/airbyte/pull/34501) | Internal code changes for Destinations V2 | +| 3.5.0 | 2024-01-24 | [\#34462](https://github.com/airbytehq/airbyte/pull/34462) | Upgrade CDK to 0.14.0 | +| 3.4.22 | 2024-01-12 | [\#34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies | +| 3.4.21 | 2024-01-10 | [\#34083](https://github.com/airbytehq/airbyte/pull/34083) | Emit destination stats as part of the state message | +| 3.4.20 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | +| 3.4.19 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 3.4.18 | 2024-01-02 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | +| 3.4.17 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Update to java CDK 0.10.0 (no changes) | +| 3.4.16 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Seperate from Table Creation | +| 3.4.15 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Only run typing+deduping for a stream if the stream had any records | +| 3.4.14 | 2023-12-08 | [\#33263](https://github.com/airbytehq/airbyte/pull/33263) | Adopt java CDK version 0.7.0 | +| 3.4.13 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Use jdbc metadata for table existence check | +| 3.4.12 | 2023-12-04 | [\#33084](https://github.com/airbytehq/airbyte/pull/33084) | T&D SQL statements moved to debug log level | | 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 3.4.10 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings. | | 3.4.9 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Add separate TRY_CAST transaction to reduce compute usage | diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 74f2b89fd598..8f6bfd22c0f2 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -26,7 +26,7 @@ You'll need the following information to configure the Teradata destination: Each stream will be output into its own table in Teradata. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. +- `_airbyte_ab_id`: a unique uuid assigned by Airbyte to each event that is processed. This is the primary index column. The column type in Teradata is `VARCHAR(256)`. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. - `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. @@ -84,9 +84,11 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------- | :------------------------------- | -| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | -| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :---------------------------------------------- |:--------------------------------------------------------| +| 0.1.5 | 2024-01-12 | https://github.com/airbytehq/airbyte/pull/33872 | Added Primary Index on _airbyte_ab_id to fix NoPI issue | +| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | +| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | +| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file diff --git a/docs/integrations/destinations/typesense.md b/docs/integrations/destinations/typesense.md index f9dfff351c60..6279d47626da 100644 --- a/docs/integrations/destinations/typesense.md +++ b/docs/integrations/destinations/typesense.md @@ -37,5 +37,7 @@ The setup only requires two fields. First is the `host` which is the address at | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.3 | 2024-01-17 | [34336](https://github.com/airbytehq/airbyte/pull/34336) | Fix check() arguments error | +| 0.1.2 | 2023-08-25 | [29817](https://github.com/airbytehq/airbyte/pull/29817) | Fix writing multiple streams | +| 0.1.1 | 2023-08-24 | [29555](https://github.com/airbytehq/airbyte/pull/29555) | Increasing connection timeout | | 0.1.0 | 2022-10-28 | [18349](https://github.com/airbytehq/airbyte/pull/18349) | New Typesense destination | -| 0.1.1 | 2023-22-17 | [29555](https://github.com/airbytehq/airbyte/pull/29555) | Increasing connection timeout | diff --git a/docs/integrations/destinations/vectara.md b/docs/integrations/destinations/vectara.md new file mode 100644 index 000000000000..1834004b8f24 --- /dev/null +++ b/docs/integrations/destinations/vectara.md @@ -0,0 +1,67 @@ +# Vectara + +This page contains the setup guide and reference information for the Vectara destination connector. + +[Vectara](https://vectara.com/) is the trusted GenAI platform that provides Retrieval Augmented Generation or [RAG](https://vectara.com/grounded-generation/) as a service. + +The Vectara destination connector allows you to connect any Airbyte source to Vectara and ingest data into Vectara for your RAG pipeline. + +:::info +In case of issues, the following public channels are available for support: + +* For Airbyte related issues such as data source or processing: [Open a Github issue](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fbug%2Carea%2Fconnectors%2Cneeds-triage&projects=&template=1-issue-connector.yaml) +* For Vectara related issues such as data indexing or RAG: Create a post in the [Vectara forum](https://discuss.vectara.com/) or reach out on [Vectara's Discord server](https://discord.gg/GFb8gMz6UH) + +::: + +## Overview + +The Vectara destination connector supports Full Refresh Overwrite, Full Refresh Append, and Incremental Append. + +### Output schema + +All streams will be output into a corpus in Vectara whose name must be specified in the config. + +Note that there are no restrictions in naming the Vectara corpus and if a corpus with the specified name is not found, a new corpus with that name will be created. Also, if multiple corpora exists with the same name, an error will be returned as Airbyte will be unable to determine the prefered corpus. + +### Features + +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Incremental - Dedupe Sync | Yes | + +## Getting started + +You will need a Vectara account to use Vectara with Airbyte. To get started, use the following steps: +1. [Sign up](https://vectara.com/integrations/airbyte) for a Vectara account if you don't already have one. Once you have completed your sign up you will have a Vectara customer ID. You can find your customer ID by clicking on your name, on the top-right of the Vectara console window. +2. Within your account you can create your corpus, which represents an area that stores text data you want to ingest into Vectara. + * To create a corpus, use the **"Create Corpus"** button in the console. You then provide a name to your corpus as well as a description. If you click on your created corpus, you can see its name and corpus ID right on the top. You can see more details in this [guide](https://docs.vectara.com/docs/console-ui/creating-a-corpus). + * Optionally you can define filtering attributes and apply some advanced options. + * For the Vectara connector to work properly you **must** define a special meta-data field called `_ab_stream` (string typed) which the connector uses to identify source streams. +3. The Vectara destination connector uses [OAuth2.0 Credentials](https://docs.vectara.com/docs/learn/authentication/oauth-2). You will need your `Client ID` and `Client Secret` handy for your connector setup. + +### Setup the Vectara Destination in Airbyte + +You should now have all the requirements needed to configure Vectara as a destination in the UI. + +You'll need the following information to configure the Vectara destination: + +- (Required) OAuth2.0 Credentials + - (Required) **Client ID** + - (Required) **Client Secret** +- (Required) **Customer ID** +- (Required) **Corpus Name**. You can specify a corpus name you've setup manually given the instructions above, or if you specify a corpus name that does not exist, the connector will generate a new corpus in this name and setup the required meta-data filtering fields within that corpus. + +In addition, in the connector UI you define two set of fields for this connector: +* `text_fields` define the source fields which are turned into text in the Vectara side and are used for query or summarization. +* `title_field` define the source field which will be used as a title of the document on the Vectara side +* `metadata_fields` define the source fields which will be added to each document as meta-data. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------- | +| 0.2.0 | 2024-01-29 | [34579](https://github.com/airbytehq/airbyte/pull/34579) | Add document title file configuration | +| 0.1.0 | 2023-11-10 | [31958](https://github.com/airbytehq/airbyte/pull/31958) | 🎉 New Destination: Vectara (Vector Database) | diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index dc055db64708..583247263ac0 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -79,10 +79,15 @@ You can also create the class in Weaviate in advance if you need more control ov As properties have to start will a lowercase letter in Weaviate and can't contain spaces or special characters. Field names might be updated during the loading process. The field names `id`, `_id` and `_additional` are reserved keywords in Weaviate, so they will be renamed to `raw_id`, `raw__id` and `raw_additional` respectively. +When using [multi-tenancy](https://weaviate.io/developers/weaviate/manage-data/multi-tenancy), the tenant id can be configured in the connector configuration. If not specified, multi-tenancy will be disabled. In case you want to index into an already created class, you need to make sure the class is created with multi-tenancy enabled. In case the class doesn't exist, it will be created with multi-tenancy properly configured. If the class already exists but the tenant id is not associated with the class, the connector will automatically add the tenant id to the class. This allows you to configure multiple connections for different tenants on the same schema. + ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.15 | 2023-01-25 | [34529](https://github.com/airbytehq/airbyte/pull/34529) | Fix tests | +| 0.2.14 | 2023-01-15 | [34229](https://github.com/airbytehq/airbyte/pull/34229) | Allow configuring tenant id | +| 0.2.13 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | | 0.2.12 | 2023-12-07 | [33218](https://github.com/airbytehq/airbyte/pull/33218) | Normalize metadata field names | | 0.2.11 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | | 0.2.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | diff --git a/docs/integrations/sources/README.md b/docs/integrations/sources/README.md new file mode 100644 index 000000000000..a50852557d0c --- /dev/null +++ b/docs/integrations/sources/README.md @@ -0,0 +1,13 @@ +import ConnectorRegistry from '@site/src/components/ConnectorRegistry'; + +# Sources + +A source is an API, file, database, or data warehouse that you want to ingest data from. + +Read more about our [Connector Support Levels](/integrations/connector-support-levels) to understand what to expect from a connector. + +## Sources + + + +_[View the connector registry in full](/integrations)_ diff --git a/docs/integrations/sources/airtable.md b/docs/integrations/sources/airtable.md index 5e314da9f1f7..c465f9a960e3 100644 --- a/docs/integrations/sources/airtable.md +++ b/docs/integrations/sources/airtable.md @@ -120,6 +120,7 @@ See information about rate limits [here](https://airtable.com/developers/web/api | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------| +| 4.1.6 | 2024-02-12 | [35149](https://github.com/airbytehq/airbyte/pull/35149) | Manage dependencies with Poetry. | | 4.1.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 4.1.4 | 2023-10-19 | [31360](https://github.com/airbytehq/airbyte/pull/31360) | Update docstings | | 4.1.3 | 2023-10-13 | [31360](https://github.com/airbytehq/airbyte/pull/31360) | Update error message for invalid permissions | @@ -139,4 +140,4 @@ See information about rate limits [here](https://airtable.com/developers/web/api | 1.0.0 | 2022-12-22 | [20846](https://github.com/airbytehq/airbyte/pull/20846) | Migrated to Metadata API for dynamic schema generation | | 0.1.3 | 2022-10-26 | [18491](https://github.com/airbytehq/airbyte/pull/18491) | Improve schema discovery logic | | 0.1.2 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.1 | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | \ No newline at end of file +| 0.1.1 | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | diff --git a/docs/integrations/sources/amazon-ads-migrations.md b/docs/integrations/sources/amazon-ads-migrations.md index 6b95c39af7b9..11f3e15cb5ef 100644 --- a/docs/integrations/sources/amazon-ads-migrations.md +++ b/docs/integrations/sources/amazon-ads-migrations.md @@ -1,5 +1,31 @@ # Amazon Ads Migration Guide +## Upgrading to 4.0.0 + +Streams `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` now have updated schemas. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + ## Upgrading to 3.0.0 A major update of attribution report stream schemas. diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index bac3870c5d5a..532712344b5e 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -13,11 +13,11 @@ This page contains the setup guide and reference information for the Amazon Ads ## Setup guide ### Step 1: Set up Amazon Ads -Create an [Amazon user](https://www.amazon.com) with access to [Amazon Ads account](https://advertising.amazon.com). +Create an [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com). **For Airbyte Open Source:** -To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you must first complete the [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview). The onboarding process has several steps and may take several days to complete. After completing all steps you will have to get Amazon client application `Client ID`, `Client Secret` and `Refresh Token`. +To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you must first complete the [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview). The onboarding process has several steps and may take several days to complete. After completing all steps you will have to get the Amazon client application's `Client ID`, `Client Secret` and `Refresh Token`. ### Step 2: Set up the Amazon Ads connector in Airbyte @@ -28,13 +28,13 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 3. On the source setup page, select **Amazon Ads** from the Source type dropdown and enter a name for this connector. -4. Click `Authenticate your Amazon Ads account`. +4. Click **Authenticate your Amazon Ads account**. 5. Log in and Authorize to the Amazon account. 6. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -7. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile. +7. **Start Date (Optional)** is used for generating reports starting from the specified start date. This should be in YYYY-MM-DD format and not more than 60 days in the past. If a date is not specified, today's date is used. The date is treated in the timezone of the processed profile. 8. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. 9. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._ -10. Click `Set up source`. +10. Click **Set up source**. @@ -44,7 +44,7 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you 2. **Client Secret** of your Amazon Ads developer application. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details. 3. **Refresh Token**. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details. 4. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -5. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile. +5. **Start Date (Optional)** is used for generating reports starting from the specified start date. This should be in YYYY-MM-DD format and not more than 60 days in the past. If a date is not specified, today's date is used. The date is treated in the timezone of the processed profile. 6. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. 7. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._ @@ -85,15 +85,15 @@ This source is capable of syncing the following streams: ## Connector-specific features and highlights -All the reports are generated relative to the target profile' timezone. +All the reports are generated relative to the target profile's timezone. -Campaign reports may sometimes have no data or not presenting in records. This can occur when there are no clicks or views associated with the campaigns on the requested day - [details](https://advertising.amazon.com/API/docs/en-us/guides/reporting/v2/faq#why-is-my-report-empty). +Campaign reports may sometimes have no data or may not be presenting in records. This can occur when there are no clicks or views associated with the campaigns on the requested day - [details](https://advertising.amazon.com/API/docs/en-us/guides/reporting/v2/faq#why-is-my-report-empty). -Report data synchronization only cover the last 60 days - [details](https://advertising.amazon.com/API/docs/en-us/reference/1/reports#parameters). +Report data synchronization only covers the last 60 days - [details](https://advertising.amazon.com/API/docs/en-us/reference/1/reports#parameters). ## Performance considerations -Information about expected report generation waiting time you may find [here](https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes). +Information about expected report generation waiting time can be found [here](https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes). ### Data type mapping @@ -110,6 +110,10 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 4.0.3 | 2024-02-12 | [35180](https://github.com/airbytehq/airbyte/pull/35180) | Manage dependencies with Poetry. | +| 4.0.2 | 2024-02-08 | [35013](https://github.com/airbytehq/airbyte/pull/35013) | Add missing field to `sponsored_display_budget_rules` stream | +| 4.0.1 | 2023-12-28 | [33833](https://github.com/airbytehq/airbyte/pull/33833) | Updated oauth spec to put region, so we can choose oauth consent url based on it | +| 4.0.0 | 2023-12-28 | [33817](https://github.com/airbytehq/airbyte/pull/33817) | Fix schema for streams: `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` | | 3.4.2 | 2023-12-12 | [33361](https://github.com/airbytehq/airbyte/pull/33361) | Fix unexpected crash when handling error messages which don't have `requestId` field | | 3.4.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 3.4.0 | 2023-06-09 | [25913](https://github.com/airbytehq/airbyte/pull/26203) | Add Stream `DisplayCreatives` | @@ -160,4 +164,4 @@ Information about expected report generation waiting time you may find [here](ht | 0.1.3 | 2021-12-28 | [8388](https://github.com/airbytehq/airbyte/pull/8388) | Add retry if recoverable error occured for reporting stream processing | | 0.1.2 | 2021-10-01 | [6367](https://github.com/airbytehq/airbyte/pull/6461) | Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback | | 0.1.1 | 2021-09-22 | [6367](https://github.com/airbytehq/airbyte/pull/6367) | Add seller and vendor filters to profiles stream | -| 0.1.0 | 2021-08-13 | [5023](https://github.com/airbytehq/airbyte/pull/5023) | Initial version | \ No newline at end of file +| 0.1.0 | 2021-08-13 | [5023](https://github.com/airbytehq/airbyte/pull/5023) | Initial version | diff --git a/docs/integrations/sources/amazon-seller-partner-migrations.md b/docs/integrations/sources/amazon-seller-partner-migrations.md index 4f51ba68b60c..5dc3da1be61a 100644 --- a/docs/integrations/sources/amazon-seller-partner-migrations.md +++ b/docs/integrations/sources/amazon-seller-partner-migrations.md @@ -1,21 +1,65 @@ # Amazon Seller Partner Migration Guide +## Upgrading to 3.0.0 + +Streams `GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL` and `GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL` now have updated schemas. + +The following streams now have date-time formatted fields: + +| Stream | Affected fields | Format change | +|-----------------------------------------------|-------------------------------------------------------------------------------|----------------------------------------------------------------------| +| `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` | `estimated-arrival-date` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_LEDGER_DETAIL_VIEW_DATA` | `Date and Time` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANTS_LISTINGS_FYP_REPORT` | `Status Change Date` | `string MMM D[,] YYYY` -> `date-time YYYY-MM-DD` | +| `GET_STRANDED_INVENTORY_UI_DATA` | `Date-to-take-auto-removal` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` | `settlement-start-date`, `settlement-end-date`, `deposit-date`, `posted-date` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_ALL_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_INACTIVE_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | + + +Users will need to refresh the source schemas and reset these streams after upgrading. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + ## Upgrading to 2.0.0 This change removes Brand Analytics and permanently removes deprecated FBA reports (from Airbyte Cloud). Customers who have those streams must refresh their schema OR disable the following streams: -* GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT -* GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT -* GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT -* GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT -* GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT -* GET_SALES_AND_TRAFFIC_REPORT -* GET_VENDOR_SALES_REPORT -* GET_VENDOR_INVENTORY_REPORT +* `GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT` +* `GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT` +* `GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT` +* `GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT` +* `GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT` +* `GET_SALES_AND_TRAFFIC_REPORT` +* `GET_VENDOR_SALES_REPORT` +* `GET_VENDOR_INVENTORY_REPORT` Customers, who have the following streams, will have to disable them: -* GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA -* GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA -* GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA -* GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA -* GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA +* `GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA` +* `GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA` +* `GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA` +* `GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA` +* `GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA` diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 59b05314a171..ed6daa83ca62 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -12,7 +12,9 @@ This page contains the setup guide and reference information for the Amazon Sell - AWS Environment - AWS Region +- AWS Seller Partner Account Type - Granted OAuth access + @@ -21,15 +23,19 @@ This page contains the setup guide and reference information for the Amazon Sell - AWS Environment - AWS Region +- AWS Seller Partner Account Type - LWA Client Id - LWA Client Secret - Refresh Token + ## Setup Guide ## Step 1: Set up Amazon Seller Partner +[Register](https://sellercentral.amazon.com/) your Amazon Seller Partner account. + **Airbyte Open Source setup steps** @@ -40,6 +46,10 @@ This page contains the setup guide and reference information for the Amazon Sell ## Step 2: Set up the source connector in Airbyte +To pass the check for Seller and Vendor accounts, you must have access to the [Orders endpoint](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) and the [Vendor Orders endpoint](https://developer-docs.amazon.com/sp-api/docs/vendor-orders-api-v1-reference#get-vendorordersv1purchaseorders), respectively. + + + **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. @@ -48,11 +58,15 @@ This page contains the setup guide and reference information for the Amazon Sell 4. Enter a name for the Amazon Seller Partner connector. 5. Click `Authenticate your account`. 6. Log in and Authorize to your Amazon Seller Partner account. -7. For Start Date, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. -8. For End Date, enter the date in YYYY-MM-DD format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. +7. For `Start Date`, enter the date in `YYYY-MM-DD` format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. +8. For `End Date`, enter the date in `YYYY-MM-DD` format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. 9. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). 10. Click `Set up source`. + + + + **For Airbyte Open Source:** 1. Using developer application from Step 1, [generate](https://developer-docs.amazon.com/sp-api/docs/self-authorization) refresh token. @@ -64,6 +78,8 @@ This page contains the setup guide and reference information for the Amazon Sell 7. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). 8. Click `Set up source`. + + ## Supported sync modes The Amazon Seller Partner source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): @@ -72,64 +88,65 @@ The Amazon Seller Partner source connector supports the following [sync modes](h ## Supported streams -- [Active Listings Report (GET_MERCHANT_LISTINGS_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [All Listings Report (GET_MERCHANT_LISTINGS_ALL_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [Amazon Search Terms Report (GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS\) -- [Brand Analytics Alternate Purchase Report (GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS\) -- [Brand Analytics Item Comparison Report (GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS\) -- [Repeat Purchase (GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS\) -- [Browse Tree Report (GET_XML_BROWSE_TREE_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-browse-tree) -- [Canceled Listings Report (GET_MERCHANT_CANCELLED_LISTINGS_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [FBA Amazon Fulfilled Inventory Report (GET_AFN_INVENTORY_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) -- [FBA Amazon Fulfilled Shipments Report (GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) -- [FBA Fee Preview Report (GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) -- [FBA Manage Inventory (GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [FBA Manage Inventory Health Report (GET_FBA_INVENTORY_PLANNING_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [FBA Multi-Country Inventory Report (GET_AFN_INVENTORY_DATA_BY_COUNTRY)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) -- [FBA Promotions Report (GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) -- [FBA Reimbursements Report (GET_FBA_REIMBURSEMENTS_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) -- [FBA Removal Order Detail Report (GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) -- [FBA Removal Shipment Detail Report (GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) -- [FBA Replacements Report (GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) -- [FBA Returns Report (GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) -- [FBA Storage Fees Report (GET_FBA_STORAGE_FEE_CHARGES_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [FBA Stranded Inventory Report (GET_STRANDED_INVENTORY_UI_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [Financial Events](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents) -- [Financial Event Groups](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups) -- [Flat File Archived Orders Report (GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) -- [Flat File Feedback Report (GET_SELLER_FEEDBACK_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-performance) -- [Flat File Orders By Last Update Report (GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) -- [Flat File Orders By Order Date Report (GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) -- [Flat File Returns Report by Return Date (GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-returns) -- [Flat File Settlement Report (GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement) -- [Inactive Listings Report (GET_MERCHANT_LISTINGS_INACTIVE_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [Inventory Ledger Report - Detailed View (GET_LEDGER_DETAIL_VIEW_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [Inventory Ledger Report - Summary View (GET_LEDGER_SUMMARY_VIEW_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [Inventory Report (GET_FLAT_FILE_OPEN_LISTINGS_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [Market Basket Analysis Report (GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS\) -- [Net Pure Product Margin Report (GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) -- [Open Listings Report (GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) +- [Active Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [All Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Amazon Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Browse Tree Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-browse-tree) \(incremental\) +- [Canceled Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [FBA Amazon Fulfilled Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Amazon Fulfilled Shipments Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) \(incremental\) +- [FBA Fee Preview Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) \(incremental\) +- [FBA Manage Inventory](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Manage Inventory Health Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Multi-Country Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Promotions Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) \(incremental\) +- [FBA Reimbursements Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) \(incremental\) +- [FBA Removal Order Detail Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) \(incremental\) +- [FBA Removal Shipment Detail Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) \(incremental\) +- [FBA Replacements Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) \(incremental\) +- [FBA Returns Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) \(incremental\) +- [FBA Storage Fees Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Stranded Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Financial Events](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents) \(incremental\) +- [Financial Event Groups](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups) \(incremental\) +- [Flat File Archived Orders Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Feedback Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-performance) \(incremental\) +- [Flat File Orders By Last Update Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Orders By Order Date Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Returns Report by Return Date](https://developer-docs.amazon.com/sp-api/docs/report-type-values-returns) \(incremental\) +- [Flat File Settlement Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement) \(incremental\) +- [Inactive Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Inventory Ledger Report - Detailed View](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Inventory Ledger Report - Summary View](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Market Basket Analysis Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Net Pure Product Margin Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Open Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) - [Orders](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) \(incremental\) - [Order Items](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems) \(incremental\) -- [Rapid Retail Analytics Inventory Report (GET_VENDOR_REAL_TIME_INVENTORY_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) -- [Restock Inventory Report (GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) -- [Sales and Traffic Business Report (GET_SALES_AND_TRAFFIC_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#seller-retail-analytics-reports) -- [Scheduled XML Order Report (Shipping) (GET_ORDER_REPORT_DATA_SHIPPING)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) -- [Subscribe and Save Forecast Report (GET_FBA_SNS_FORECAST_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) -- [Subscribe and Save Performance Report (GET_FBA_SNS_PERFORMANCE_DATA)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) -- [Suppressed Listings Report (GET_MERCHANTS_LISTINGS_FYP_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) -- [Unshipped Orders Report (GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) -- [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference) -- [Vendor Inventory Report (GET_VENDOR_INVENTORY_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) -- [Vendor Sales Report (GET_VENDOR_SALES_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) -- [Vendor Traffic Report (GET_VENDOR_TRAFFIC_REPORT)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) -- [XML Orders By Order Date Report (GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) +- [Rapid Retail Analytics Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Repeat Purchase](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Restock Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Sales and Traffic Business Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#seller-retail-analytics-reports) \(incremental\) +- [Scheduled XML Order Report (Shipping)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) \(incremental\) +- [Subscribe and Save Forecast Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) \(incremental\) +- [Subscribe and Save Performance Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) \(incremental\) +- [Suppressed Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Unshipped Orders Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) \(incremental\) +- [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference) \(incremental\) +- [Vendor Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Vendor Sales Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Vendor Traffic Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [XML Orders By Order Date Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Vendor Orders](https://developer-docs.amazon.com/sp-api/docs/vendor-orders-api-v1-reference#get-vendorordersv1purchaseorders) \(incremental\) ## Report options -Make sure to configure the [required parameters](https://developer-docs.amazon.com/sp-api/docs/report-type-values) in the report options setting for the reports configured. +Report options can be assigned on a per-stream basis that alter the behavior when generating a report. +For the full list, refer to Amazon’s report type values [documentation](https://developer-docs.amazon.com/sp-api/docs/report-type-values). -For `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams maximum value for `period_in_days` 30 days and 60 days. +Certain report types have required parameters that must be defined. +For `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams maximum value for `period_in_days` 30 days and 60 days. So, for any value that exceeds the limit, the `period_in_days` will be automatically reduced to the limit for the stream. ## Performance considerations @@ -151,6 +168,17 @@ Information about rate limits you may find [here](https://developer-docs.amazon. | Version | Date | Pull Request | Subject | |:---------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `3.5.0` | 2024-02-09 | [\#35331](https://github.com/airbytehq/airbyte/pull/35331) | Fix check for Vendor accounts. Add failed report result message | +| `3.4.0` | 2024-02-15 | [\#35273](https://github.com/airbytehq/airbyte/pull/35273) | Add `VendorOrders` stream | +| `3.3.2` | 2024-02-13 | [\#33996](https://github.com/airbytehq/airbyte/pull/33996) | Add integration tests | +| `3.3.1` | 2024-02-09 | [\#35106](https://github.com/airbytehq/airbyte/pull/35106) | Add logs for the failed check command | +| `3.3.0` | 2024-02-09 | [\#35062](https://github.com/airbytehq/airbyte/pull/35062) | Fix the check command for the `Vendor` account type | +| `3.2.2` | 2024-02-07 | [\#34914](https://github.com/airbytehq/airbyte/pull/34914) | Fix date formatting for ledger reports with aggregation by month | +| `3.2.1` | 2024-01-30 | [\#34654](https://github.com/airbytehq/airbyte/pull/34654) | Fix date format in state message for streams with custom dates formatting | +| `3.2.0` | 2024-01-26 | [\#34549](https://github.com/airbytehq/airbyte/pull/34549) | Update schemas for vendor analytics streams | +| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | +| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | +| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | | `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | | `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | | `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | diff --git a/docs/integrations/sources/amazon-sqs.md b/docs/integrations/sources/amazon-sqs.md index a71586ffced3..78fa87bc627f 100644 --- a/docs/integrations/sources/amazon-sqs.md +++ b/docs/integrations/sources/amazon-sqs.md @@ -6,19 +6,20 @@ This source will sync messages from an [SQS Queue](https://docs.aws.amazon.com/s ### Output schema -This source will output one stream for the configured SQS Queue. -The stream record data will have three fields: -* id (a UUIDv4 as a STRING) -* body (message body as a STRING) -* attributes (attributes of the messages as an OBJECT or NULL) +This source will output one stream for the configured SQS Queue. The stream record data will have +three fields: + +- id (a UUIDv4 as a STRING) +- body (message body as a STRING) +- attributes (attributes of the messages as an OBJECT or NULL) ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | yes | | -| Incremental Sync | no | | -| Namespaces | no | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | yes | | +| Incremental Sync | no | | +| Namespaces | no | | ### Performance considerations @@ -26,63 +27,77 @@ The stream record data will have three fields: ### Requirements -* AWS IAM Access Key -* AWS IAM Secret Key -* AWS SQS Queue +- AWS IAM Access Key +- AWS IAM Secret Key +- AWS SQS Queue ### Properties -Required properties are 'Queue URL', 'AWS Region' and 'Delete Messages After Read' as noted in **bold** below. - -* **Queue URL** (STRING) - * The full AWS endpoint URL of the queue e.g. `https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` -* **AWS Region** (STRING) - * The region code for the SQS Queue e.g. eu-west-1 -* **Delete Messages After Read** (BOOLEAN) - * **WARNING:** Setting this option to TRUE can result in data loss, do not enable this option unless you understand the risk. See the **Data loss warning** section below. - * Should the message be deleted from the SQS Queue after being read? This prevents the message being read more than once - * By default messages are NOT deleted, thus can be re-read after the `Message Visibility Timeout` - * Default: False -* Max Batch Size (INTEGER) - * The max amount of messages to consume in a single poll e.g. 5 - * Minimum of 1, maximum of 10 - * Default: 10 -* Max Wait Time (INTEGER) - * The max amount of time (in seconds) to poll for messages before commiting a batch (or timing out) unless we fill a batch (as per `Max Batch Size`) - * Minimum of 1, maximum of 20 - * Default: 20 -* Message Attributes To Return (STRING) - * A comma separated list of Attributes to return for each message - * Default: All -* Message Visibility Timeout (INTEGER) - * After a message is read, how much time (in seconds) should the message be hidden from other consumers - * After this timeout, the message is not deleted and can be re-read - * Default: 30 -* AWS IAM Access Key ID (STRING) - * The Access Key for the IAM User with permissions on this Queue - * If `Delete Messages After Read` is `false` then only `sqs:ReceiveMessage` - * If `Delete Messages After Read` is `true` then `sqs:DeleteMessage` is also needed -* AWS IAM Secret Key (STRING) - * The Secret Key for the IAM User with permissions on this Queue +Required properties are 'Queue URL', 'AWS Region' and 'Delete Messages After Read' as noted in +**bold** below. + +- **Queue URL** (STRING) + - The full AWS endpoint URL of the queue e.g. + `https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` +- **AWS Region** (STRING) + - The region code for the SQS Queue e.g. eu-west-1 +- **Delete Messages After Read** (BOOLEAN) + - **WARNING:** Setting this option to TRUE can result in data loss, do not enable this option + unless you understand the risk. See the **Data loss warning** section below. + - Should the message be deleted from the SQS Queue after being read? This prevents the message + being read more than once + - By default messages are NOT deleted, thus can be re-read after the `Message Visibility Timeout` + - Default: False +- Max Batch Size (INTEGER) + - The max amount of messages to consume in a single poll e.g. 5 + - Minimum of 1, maximum of 10 + - Default: 10 +- Max Wait Time (INTEGER) + - The max amount of time (in seconds) to poll for messages before commiting a batch (or timing + out) unless we fill a batch (as per `Max Batch Size`) + - Minimum of 1, maximum of 20 + - Default: 20 +- Message Attributes To Return (STRING) + - A comma separated list of Attributes to return for each message + - Default: All +- Message Visibility Timeout (INTEGER) + - After a message is read, how much time (in seconds) should the message be hidden from other + consumers + - After this timeout, the message is not deleted and can be re-read + - Default: 30 +- AWS IAM Access Key ID (STRING) + - The Access Key for the IAM User with permissions on this Queue + - If `Delete Messages After Read` is `false` then only `sqs:ReceiveMessage` + - If `Delete Messages After Read` is `true` then `sqs:DeleteMessage` is also needed +- AWS IAM Secret Key (STRING) + - The Secret Key for the IAM User with permissions on this Queue ### Data loss warning -When enabling **Delete Messages After Read**, the Source will delete messages from the SQS Queue after reading them. The message is deleted *after* the configured Destination takes the message from this Source, but makes no guarentee that the downstream destination has commited/persisted the message. This means that it is possible for the Airbyte Destination to read the message from the Source, the Source deletes the message, then the downstream application fails - resulting in the message being lost permanently. +When enabling **Delete Messages After Read**, the Source will delete messages from the SQS Queue +after reading them. The message is deleted _after_ the configured Destination takes the message from +this Source, but makes no guarentee that the downstream destination has commited/persisted the +message. This means that it is possible for the Airbyte Destination to read the message from the +Source, the Source deletes the message, then the downstream application fails - resulting in the +message being lost permanently. Extra care should be taken to understand this risk before enabling this option. ### Setup guide -* [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) -* [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) +- [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) +- [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) > **NOTE**: -> * If `Delete Messages After Read` is `false` then the IAM User needs only `sqs:ReceiveMessage` in the AWS IAM Policy -> * If `Delete Messages After Read` is `true` then both `sqs:ReceiveMessage` and `sqs:DeleteMessage` are needed in the AWS IAM Policy +> +> - If `Delete Messages After Read` is `false` then the IAM User needs only `sqs:ReceiveMessage` in +> the AWS IAM Policy +> - If `Delete Messages After Read` is `true` then both `sqs:ReceiveMessage` and `sqs:DeleteMessage` +> are needed in the AWS IAM Policy ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.0` | 2021-10-10 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | `Initial version` | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------- | +| 0.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.0 | 2021-10-10 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | Initial version | diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md index 2c789ed146cd..0972f2dcd549 100644 --- a/docs/integrations/sources/amplitude.md +++ b/docs/integrations/sources/amplitude.md @@ -52,6 +52,7 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.3.7 | 2024-02-12 | [35162](https://github.com/airbytehq/airbyte/pull/35162) | Manage dependencies with Poetry. | | 0.3.6 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.5 | 2023-09-28 | [30846](https://github.com/airbytehq/airbyte/pull/30846) | Add support of multiple cursor date formats | | 0.3.4 | 2023-09-28 | [30831](https://github.com/airbytehq/airbyte/pull/30831) | Add user friendly error description on 403 error | @@ -89,4 +90,4 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio | 0.1.2 | 2021-09-21 | [6353](https://github.com/airbytehq/airbyte/pull/6353) | Correct output schemas on cohorts, events, active\_users, and average\_session\_lengths streams | | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for kubernetes support | | 0.1.0 | 2021-06-08 | [3664](https://github.com/airbytehq/airbyte/pull/3664) | New Source: Amplitude | - \ No newline at end of file + diff --git a/docs/integrations/sources/apify-dataset.md b/docs/integrations/sources/apify-dataset.md index 7370483c115f..a6546160709d 100644 --- a/docs/integrations/sources/apify-dataset.md +++ b/docs/integrations/sources/apify-dataset.md @@ -69,6 +69,7 @@ The Apify dataset connector uses [Apify Python Client](https://docs.apify.com/ap | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------------- | :-------------------------------------------------------------------------- | +| 2.1.1 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 2.1.0 | 2023-10-13 | [31333](https://github.com/airbytehq/airbyte/pull/31333) | Add stream for arbitrary datasets | | 2.0.0 | 2023-09-18 | [30428](https://github.com/airbytehq/airbyte/pull/30428) | Fix broken stream, manifest refactor | | 1.0.0 | 2023-08-25 | [29859](https://github.com/airbytehq/airbyte/pull/29859) | Migrate to lowcode | diff --git a/docs/integrations/sources/appstore.md b/docs/integrations/sources/appstore.md index b8acae44e28c..a99893048417 100644 --- a/docs/integrations/sources/appstore.md +++ b/docs/integrations/sources/appstore.md @@ -1,5 +1,20 @@ # Appstore +:::warning + +## Deprecation Notice + +The Appstore source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + + ## Sync overview This source can sync data for the [Appstore API](https://developer.apple.com/documentation/appstoreconnectapi). It supports only Incremental syncs. The Appstore API is available for [many types of services](https://developer.apple.com/documentation/appstoreconnectapi). Currently, this API supports syncing Sales and Trends reports. If you'd like to sync data from other endpoints, please create an issue on Github. diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 0bb5243fc5a0..a6e291398e52 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -182,7 +182,7 @@ There are currently no options for JSONL parsing. The Document File Type Format is currently an experimental feature and not subject to SLAs. Use at your own risk. ::: -The Document File Type Format is a special format that allows you to extract text from Markdown, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. +The Document File Type Format is a special format that allows you to extract text from Markdown, TXT, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. @@ -191,12 +191,17 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------| -| 0.2.5 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | -| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | -| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | -| 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | -| 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | -| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.3.4 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | +| 0.3.3 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.3.2 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.3.1 | 2024-01-10 | [34084](https://github.com/airbytehq/airbyte/pull/34084) | Fix bug for running check with document file format | +| 0.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | +| 0.2.5 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | +| 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | +| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | \ No newline at end of file diff --git a/docs/integrations/sources/azure-table.md b/docs/integrations/sources/azure-table.md index 5ad07bb670af..f617018961a6 100644 --- a/docs/integrations/sources/azure-table.md +++ b/docs/integrations/sources/azure-table.md @@ -9,7 +9,7 @@ The Azure table storage supports Full Refresh and Incremental syncs. You can cho This Source have generic schema for all streams. Azure Table storage is a service that stores non-relational structured data (also known as structured NoSQL data). There is no efficient way to read schema for the given table. We use `data` property to have all the properties for any given row. -- data - This property contain all values +- data - This property contains all values - additionalProperties - This property denotes that all the values are in `data` property. ``` @@ -58,16 +58,15 @@ The Azure table storage connector should not run into API limitations under norm Visit the [Azure Portal](https://portal.azure.com). Go to your storage account, you can find : - Azure Storage Account - under the overview tab - Azure Storage Account Key - under the Access keys tab - - Azure Storage Endpoint Suffix - under the Enpoint tab + - Azure Storage Endpoint Suffix - under the Endpoint tab We recommend creating a restricted key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. However, shared access key authentication is not supported by this connector yet. - ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.4 | 2024-01-26 | [34576](https://github.com/airbytehq/airbyte/pull/34576) | Migrate to per-stream/global state | | 0.1.3 | 2022-08-12 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream | | 0.1.2 | 2021-12-23 | [14212](https://github.com/airbytehq/airbyte/pull/14212) | Adding incremental load capability | | 0.1.1 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | - diff --git a/docs/integrations/sources/bigquery.md b/docs/integrations/sources/bigquery.md index c4b3d042b519..b0d73b124294 100644 --- a/docs/integrations/sources/bigquery.md +++ b/docs/integrations/sources/bigquery.md @@ -87,7 +87,10 @@ Once you've configured BigQuery as a source, delete the Service Account Key from ### source-bigquery | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.2 | 2024-02-22 | [35503](https://github.com/airbytehq/airbyte/pull/35503) | Source BigQuery: replicating RECORD REPEATED fields | +| 0.4.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.4.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | | 0.3.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.2.3 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.2.2 | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header | diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 969314f0c29c..3cedce121288 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -55,16 +55,17 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Add the developer token from [Step 1](#step-1-set-up-bing-ads). -7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. -9. For *Custom Reports* - see [custom reports](#custom-reports) section, list of custom reports object: +7. For **Account Names Predicates** - see [predicates](https://learn.microsoft.com/en-us/advertising/customer-management-service/predicate?view=bingads-13) in bing ads docs. Will be used to filter your accounts by specified operator and account name. You can use multiple predicates pairs. The **Operator** is a one of Contains or Equals. The **Account Name** is a value to compare Accounts Name field in rows by specified operator. For example, for operator=Contains and name=Dev, all accounts where name contains dev will be replicated. And for operator=Equals and name=Airbyte, all accounts where name is equal to Airbyte will be replicated. Account Name value is not case-sensitive. +8. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +9. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. +10. For *Custom Reports* - see [custom reports](#custom-reports) section, list of custom reports object: 1. For *Report Name* enter the name that you want for your custom report. 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. 3. For *Columns* add list columns of Reporting Data Object that you want to see in the custom report. 4. For *Aggregation* add time aggregation. See [report aggregation](#report-aggregation) section. -10. Click **Authenticate your Bing Ads account**. -11. Log in and authorize the Bing Ads account. -12. Click **Set up source**. +11. Click **Authenticate your Bing Ads account**. +12. Log in and authorize the Bing Ads account. +13. Click **Set up source**. @@ -77,15 +78,16 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Enter the **Client ID**, **Client Secret**, **Refresh Token**, and **Developer Token** from [Step 1](#step-1-set-up-bing-ads). -7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. -9. For *Custom Reports* - see [custom reports](#custom-reports) section: +7. For **Account Names Predicates** - see [predicates](https://learn.microsoft.com/en-us/advertising/customer-management-service/predicate?view=bingads-13) in bing ads docs. Will be used to filter your accounts by specified operator and account name. You can use multiple predicates pairs. The **Operator** is a one of Contains or Equals. The **Account Name** is a value to compare Accounts Name field in rows by specified operator. For example, for operator=Contains and name=Dev, all accounts where name contains dev will be replicated. And for operator=Equals and name=Airbyte, all accounts where name is equal to Airbyte will be replicated. Account Name value is not case-sensitive. +8. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +9. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. +10. For *Custom Reports* - see [custom reports](#custom-reports) section: 1. For *Report Name* enter the name that you want for your custom report. 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. 3. For *Columns* add columns of Reporting Data Object that you want to see in the custom report. 4. For *Aggregation* select time aggregation. See [report aggregation](#report-aggregation) section. -10. Click **Set up source**. +11. Click **Set up source**. @@ -224,6 +226,11 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.4 | 2024-02-12 | [35179](https://github.com/airbytehq/airbyte/pull/35179) | Manage dependencies with Poetry. | +| 2.1.3 | 2024-01-31 | [34712](https://github.com/airbytehq/airbyte/pull/34712) | Fix duplicated records for report-based streams | +| 2.1.2 | 2024-01-09 | [34045](https://github.com/airbytehq/airbyte/pull/34045) | Speed up record transformation | +| 2.1.1 | 2023-12-15 | [33500](https://github.com/airbytehq/airbyte/pull/33500) | Fix state setter when state was provided | +| 2.1.0 | 2023-12-05 | [33095](https://github.com/airbytehq/airbyte/pull/33095) | Add account filtering | | 2.0.1 | 2023-11-16 | [32597](https://github.com/airbytehq/airbyte/pull/32597) | Fix start date parsing from stream state | | 2.0.0 | 2023-11-07 | [31995](https://github.com/airbytehq/airbyte/pull/31995) | Schema update for Accounts, Campaigns and Search Query Performance Report streams. Convert `date` and `date-time` fields to standard `RFC3339` | | 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | @@ -274,4 +281,4 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | | 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | - \ No newline at end of file + diff --git a/docs/integrations/sources/chargebee.md b/docs/integrations/sources/chargebee.md index 50cf1afec6ac..27b6568ef850 100644 --- a/docs/integrations/sources/chargebee.md +++ b/docs/integrations/sources/chargebee.md @@ -1,10 +1,16 @@ # Chargebee + + This page contains the setup guide and reference information for the Chargebee source connector. + + ## Prerequisites -To set up the Chargebee source connector, you will need a valid [Chargebee API key](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_authentication) and the [Product Catalog version](https://www.chargebee.com/docs/1.0/upgrade-product-catalog.html) of the Chargebee site you are syncing data from. +To set up the Chargebee source connector, you will need: + - [Chargebee API key](https://apidocs.chargebee.com/docs/api/auth) + - [Product Catalog version](https://www.chargebee.com/docs/1.0/upgrade-product-catalog.html) of the Chargebee site you are syncing. :::info All Chargebee sites created from May 5, 2021 onward will have [Product Catalog 2.0](https://www.chargebee.com/docs/2.0/product-catalog.html) enabled by default. Sites created prior to this date will use [Product Catalog 1.0](https://www.chargebee.com/docs/1.0/product-catalog.html). @@ -19,9 +25,11 @@ All Chargebee sites created from May 5, 2021 onward will have [Product Catalog 2 5. For **Site**, enter the site prefix for your Chargebee instance. 6. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 7. For **API Key**, enter the [Chargebee API key](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_authentication). -8. For **Product Catalog**, enter the Chargebee [Product Catalog version](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2). +8. For **Product Catalog**, enter the Chargebee [Product Catalog version](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2). Connector defaults to Product Catalog 2.0 unless otherwise specified. 9. Click **Set up source**. + + ## Supported sync modes The Chargebee source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -29,50 +37,74 @@ The Chargebee source connector supports the following [sync modes](https://docs. * [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) * [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) * [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +* [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) ## Supported streams Most streams are supported regardless of your Chargebee site's [Product Catalog version](https://www.chargebee.com/docs/1.0/upgrade-product-catalog.html), with a few version-specific exceptions. -| Stream | Product Catalog 1.0 | Product Catalog 2.0 | -|------------------------|---------------------|---------------------| -| [Addons](https://apidocs.chargebee.com/docs/api/addons?prod_cat_ver=1) | ✔ | | -| [Attached Items](https://apidocs.chargebee.com/docs/api/attached_items?prod_cat_ver=2) | | ✔ | -| [Contacts](https://apidocs.chargebee.com/docs/api/customers?lang=curl#list_of_contacts_for_a_customer) | ✔ | ✔ | -| [Coupons](https://apidocs.chargebee.com/docs/api/coupons) | ✔ | ✔ | -| [Credit Notes](https://apidocs.chargebee.com/docs/api/credit_notes) | ✔ | ✔ | -| [Customers](https://apidocs.chargebee.com/docs/api/customers) | ✔ | ✔ | -| [Events](https://apidocs.chargebee.com/docs/api/events) | ✔ | ✔ | -| [Gifts](https://apidocs.chargebee.com/docs/api/gifts) | ✔ | ✔ | -| [Hosted Pages](https://apidocs.chargebee.com/docs/api/hosted_pages) | ✔ | ✔ | -| [Invoices](https://apidocs.chargebee.com/docs/api/invoices) | ✔ | ✔ | -| [Items](https://apidocs.chargebee.com/docs/api/items?prod_cat_ver=2) | | ✔ | -| [Item Prices](https://apidocs.chargebee.com/docs/api/item_prices?prod_cat_ver=2) | | ✔ | -| [Orders](https://apidocs.chargebee.com/docs/api/orders) | ✔ | ✔ | -| [Payment Sources](https://apidocs.chargebee.com/docs/api/payment_sources) | ✔ | ✔ | -| [Plans](https://apidocs.chargebee.com/docs/api/plans?prod_cat_ver=1) | ✔ | | -| [Promotional Credits](https://apidocs.chargebee.com/docs/api/promotional_credits) | ✔ | ✔ | -| [Quotes](https://apidocs.chargebee.com/docs/api/quotes) | ✔ | ✔ | -| [Quote Line Groups](https://apidocs.chargebee.com/docs/api/quote_line_groups) | ✔ | ✔ | -| [Subscriptions](https://apidocs.chargebee.com/docs/api/subscriptions) | ✔ | ✔ | -| [Transactions](https://apidocs.chargebee.com/docs/api/transactions) | ✔ | ✔ | -| [Unbilled Charges](https://apidocs.chargebee.com/docs/api/unbilled_charges) | ✔ | ✔ | -| [Virtual Bank Accounts](https://apidocs.chargebee.com/docs/api/virtual_bank_accounts) | ✔ | ✔ | +| Stream | Product Catalog 1.0 | Product Catalog 2.0 | +| ------------------------------------------------------------------------------------------------------ | ------------------- | ------------------- | +| [Addons](https://apidocs.chargebee.com/docs/api/addons?prod_cat_ver=1) | ✔ | | +| [Attached Items](https://apidocs.chargebee.com/docs/api/attached_items?prod_cat_ver=2) | | ✔ | +| [Comments](https://apidocs.chargebee.com/docs/api/comments?prod_cat_ver=2) | ✔ | ✔ | +| [Contacts](https://apidocs.chargebee.com/docs/api/customers?lang=curl#list_of_contacts_for_a_customer) | ✔ | ✔ | +| [Coupons](https://apidocs.chargebee.com/docs/api/coupons) | ✔ | ✔ | +| [Credit Notes](https://apidocs.chargebee.com/docs/api/credit_notes) | ✔ | ✔ | +| [Customers](https://apidocs.chargebee.com/docs/api/customers) | ✔ | ✔ | +| [Differential Prices](https://apidocs.chargebee.com/docs/api/differential_prices) | ✔ | ✔ | +| [Events](https://apidocs.chargebee.com/docs/api/events) | ✔ | ✔ | +| [Gifts](https://apidocs.chargebee.com/docs/api/gifts) | ✔ | ✔ | +| [Hosted Pages](https://apidocs.chargebee.com/docs/api/hosted_pages) | ✔ | ✔ | +| [Invoices](https://apidocs.chargebee.com/docs/api/invoices) | ✔ | ✔ | +| [Items](https://apidocs.chargebee.com/docs/api/items?prod_cat_ver=2) | | ✔ | +| [Item Prices](https://apidocs.chargebee.com/docs/api/item_prices?prod_cat_ver=2) | | ✔ | +| [Item Families](https://apidocs.chargebee.com/docs/api/item_families?prod_cat_ver=2) | | ✔ | +| [Orders](https://apidocs.chargebee.com/docs/api/orders) | ✔ | ✔ | +| [Payment Sources](https://apidocs.chargebee.com/docs/api/payment_sources) | ✔ | ✔ | +| [Plans](https://apidocs.chargebee.com/docs/api/plans?prod_cat_ver=1) | ✔ | | +| [Promotional Credits](https://apidocs.chargebee.com/docs/api/promotional_credits) | ✔ | ✔ | +| [Quotes](https://apidocs.chargebee.com/docs/api/quotes) | ✔ | ✔ | +| [Quote Line Groups](https://apidocs.chargebee.com/docs/api/quote_line_groups) | ✔ | ✔ | +| [Site Migration Details](https://apidocs.chargebee.com/docs/api/site_migration_details) | ✔ | ✔ | +| [Subscriptions](https://apidocs.chargebee.com/docs/api/subscriptions) | ✔ | ✔ | +| [Transactions](https://apidocs.chargebee.com/docs/api/transactions) | ✔ | ✔ | +| [Unbilled Charges](https://apidocs.chargebee.com/docs/api/unbilled_charges) | ✔ | ✔ | +| [Virtual Bank Accounts](https://apidocs.chargebee.com/docs/api/virtual_bank_accounts) | ✔ | ✔ | :::note When using incremental sync mode, the `Attached Items` stream behaves differently than the other streams. Whereas other incremental streams read and output _only new_ records, the `Attached Items` stream reads _all_ records but only outputs _new_ records, making it more demanding on your Chargebee API quota. Each sync incurs API calls equal to the total number of attached items in your Chargebee instance divided by 100, regardless of the actual number of `Attached Items` changed or synced. ::: -## Performance considerations +## Limitations & Troubleshooting + +
      + +Expand to see details about the Chargebee connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The Chargebee connector should not run into [Chargebee API](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_rate_limits) limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you encounter any rate limit issues that are not automatically retried successfully. +### Troubleshooting + +* Check out common troubleshooting issues for the Instagram source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.2.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.2.4 | 2023-08-01 | [28905](https://github.com/airbytehq/airbyte/pull/28905) | Updated the connector to use latest CDK version | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +| 0.4.0 | 2024-02-12 | [34053](https://github.com/airbytehq/airbyte/pull/34053) | Add missing fields to and cleans up schemas, adds incremental support for `gift`, `site_migration_detail`, and `unbilled_charge` streams.` | +| 0.3.1 | 2024-02-12 | [35169](https://github.com/airbytehq/airbyte/pull/35169) | Manage dependencies with Poetry. | +| 0.3.0 | 2023-12-26 | [33696](https://github.com/airbytehq/airbyte/pull/33696) | Add new stream, add fields to existing streams | +| 0.2.6 | 2023-12-19 | [32100](https://github.com/airbytehq/airbyte/pull/32100) | Add new fields in streams | +| 0.2.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.4 | 2023-08-01 | [28905](https://github.com/airbytehq/airbyte/pull/28905) | Updated the connector to use latest CDK version | | 0.2.3 | 2023-03-22 | [24370](https://github.com/airbytehq/airbyte/pull/24370) | Ignore 404 errors for `Contact` stream | | 0.2.2 | 2023-02-17 | [21688](https://github.com/airbytehq/airbyte/pull/21688) | Migrate to CDK beta 0.29; fix schemas | | 0.2.1 | 2023-02-17 | [23207](https://github.com/airbytehq/airbyte/pull/23207) | Edited stream schemas to get rid of unnecessary `enum` | @@ -93,4 +125,6 @@ The Chargebee connector should not run into [Chargebee API](https://apidocs.char | 0.1.3 | 2021-08-17 | [5421](https://github.com/airbytehq/airbyte/pull/5421) | Add support for "Product Catalog 2.0" specific streams: `Items`, `Item prices` and `Attached Items` | | 0.1.2 | 2021-07-30 | [5067](https://github.com/airbytehq/airbyte/pull/5067) | Prepare connector for publishing | | 0.1.1 | 2021-07-07 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add entrypoint and bump version for connector | -| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee | \ No newline at end of file +| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee | + +       diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 912065139d93..2fa69ac5a2a6 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -76,39 +76,42 @@ Using this feature requires additional configuration, when creating the source. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| -| 0.1.17 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | -| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | -| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.2.2 | 2024-02-13 | [35235](https://github.com/airbytehq/airbyte/pull/35235) | Adopt CDK 0.20.4 | +| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.1.17 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | +| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.12 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.1.9 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | -| 0.1.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.7 | 2021-12-24 | [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | -| 0.1.6 | 2021-12-15 | [\#8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.5 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.1.4 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | -| 0.1.3 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added SSL connections support. | -| 0.1.2 | 13.08.2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator. | +| 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.1.9 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | +| 0.1.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.7 | 2021-12-24 | [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | +| 0.1.6 | 2021-12-15 | [\#8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.5 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.1.4 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | +| 0.1.3 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added SSL connections support. | +| 0.1.2 | 13.08.2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator. | ## CHANGELOG source-clickhouse-strict-encrypt -| Version | Date | Pull Request | Subject | -|:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| -| 0.1.17 | 2022-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to| -| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.1.17 | 2022-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.16 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | -| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | -| 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | -| 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.1.1 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | -| 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | +| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | +| 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | +| 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.1.1 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | +| 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | diff --git a/docs/integrations/sources/close-com.md b/docs/integrations/sources/close-com.md index bedfdc116949..d152f845fe9c 100644 --- a/docs/integrations/sources/close-com.md +++ b/docs/integrations/sources/close-com.md @@ -105,8 +105,9 @@ The Close.com connector is subject to rate limits. For more information on this | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------| -| 0.4.3 | 2023-10-28 | [31534](https://github.com/airbytehq/airbyte/pull/31534) | Fixed Email Activities Stream Pagination | -| 0.4.2 | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date` | +| 0.5.0 | 2023-11-30 | [32984](https://github.com/airbytehq/airbyte/pull/32984) | Add support for custom fields | +| 0.4.3 | 2023-10-28 | [31534](https://github.com/airbytehq/airbyte/pull/31534) | Fixed Email Activities Stream Pagination | +| 0.4.2 | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date` | | 0.4.1 | 2023-07-04 | [27950](https://github.com/airbytehq/airbyte/pull/27950) | Add human readable titles to API Key and Start Date fields | | 0.4.0 | 2023-06-27 | [27776](https://github.com/airbytehq/airbyte/pull/27776) | Update the `Email Followup Tasks` stream schema | | 0.3.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Update the `Email sequences` stream schema | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index a32fddf184d9..689f2b0ae817 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -93,25 +93,28 @@ Your database user should now be ready for use with Airbyte. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- | :--- | -| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.21 | 2023-03-14 | [24000](https://github.com/airbytehq/airbyte/pull/24000) | Removed check method call on read. | -| 0.1.20 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect | -| 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.2 | 2024-02-13 | [35234](https://github.com/airbytehq/airbyte/pull/35234) | Adopt CDK 0.20.4 | +| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Removed LEGACY state | +| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.21 | 2023-03-14 | [24000](https://github.com/airbytehq/airbyte/pull/24000) | Removed check method call on read. | +| 0.1.20 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect | +| 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | -| 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | -| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | -| 0.1.4 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | -| 0.1.3 | 2021-10-10 | [7819](https://github.com/airbytehq/airbyte/pull/7819) | Fixed Datatype errors during Cockroach DB parsing | -| 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | +| 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | +| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | +| 0.1.4 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | +| 0.1.3 | 2021-10-10 | [7819](https://github.com/airbytehq/airbyte/pull/7819) | Fixed Datatype errors during Cockroach DB parsing | +| 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | diff --git a/docs/integrations/sources/coin-api.md b/docs/integrations/sources/coin-api.md index 5ef3447b5f2b..dba67e5e646f 100644 --- a/docs/integrations/sources/coin-api.md +++ b/docs/integrations/sources/coin-api.md @@ -10,8 +10,8 @@ syncs. This source is capable of syncing the following streams: -* `ohlcv_historical_data` -* `trades_historical_data` +- `ohlcv_historical_data` +- `trades_historical_data` ### Features @@ -50,5 +50,6 @@ The following fields are required fields for the connector to work: | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------| -| 0.1.1 | 2022-12-19 | [20600](https://github.com/airbytehq/airbyte/pull/20600) | Add quotes historical data stream| -| 0.1.0 | 2022-10-21 | [18302](https://github.com/airbytehq/airbyte/pull/18302) | New source | +| 0.2.0 | 2024-02-05 | [#34826](https://github.com/airbytehq/airbyte/pull/34826) | Fix catalog types for fields `bid_price` and `bid_size` in stream `quotes_historical_data`. | +| 0.1.1 | 2022-12-19 | [#20600](https://github.com/airbytehq/airbyte/pull/20600) | Add quotes historical data stream| +| 0.1.0 | 2022-10-21 | [#18302](https://github.com/airbytehq/airbyte/pull/18302) | New source | diff --git a/docs/integrations/sources/convex.md b/docs/integrations/sources/convex.md index c5acc008eec9..d643940939a0 100644 --- a/docs/integrations/sources/convex.md +++ b/docs/integrations/sources/convex.md @@ -70,9 +70,10 @@ In the Data tab, you should see the tables and a sample of the data that will be ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- | -| 0.3.0 | 2023-09-28 | [30853](https://github.com/airbytehq/airbyte/pull/30853) | 🐛 Convex source switch to clean JSON format | -| 0.2.0 | 2023-06-21 | [27226](https://github.com/airbytehq/airbyte/pull/27226) | 🐛 Convex source fix skipped records | -| 0.1.1 | 2023-03-06 | [23797](https://github.com/airbytehq/airbyte/pull/23797) | 🐛 Convex source connector error messages | -| 0.1.0 | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------- | +| 0.4.0 | 2023-12-13 | [33431](https://github.com/airbytehq/airbyte/pull/33431) | 🐛 Convex source fix bug where full_refresh stops after one page | +| 0.3.0 | 2023-09-28 | [30853](https://github.com/airbytehq/airbyte/pull/30853) | 🐛 Convex source switch to clean JSON format | +| 0.2.0 | 2023-06-21 | [27226](https://github.com/airbytehq/airbyte/pull/27226) | 🐛 Convex source fix skipped records | +| 0.1.1 | 2023-03-06 | [23797](https://github.com/airbytehq/airbyte/pull/23797) | 🐛 Convex source connector error messages | +| 0.1.0 | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex | diff --git a/docs/integrations/sources/courier.md b/docs/integrations/sources/courier.md index 265e72520974..8f0b9ed55c3c 100644 --- a/docs/integrations/sources/courier.md +++ b/docs/integrations/sources/courier.md @@ -1,5 +1,19 @@ # Courier +:::warning + +## Deprecation Notice + +The Courier source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + This page contains the setup guide and reference information for the [Courier](https://www.courier.com/) source connector. ## Prerequisites diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index ff3f706265e6..d138124ed68f 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -58,27 +58,30 @@ You can also enter your own password for the keystore, but if you don't, the pas ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.1.20 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | -| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.2 | 2024-02-13 | [35233](https://github.com/airbytehq/airbyte/pull/35233) | Adopt CDK 0.20.4 | +| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.1.20 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | +| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | -| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** -| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables | -| 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. | -| 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation | -| 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support | -| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | -| 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 | +| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | +| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** +| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables | +| 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. | +| 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation | +| 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support | +| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 | diff --git a/docs/integrations/sources/dv-360.md b/docs/integrations/sources/dv-360.md index b3c095f4691c..ebdcad8d0410 100644 --- a/docs/integrations/sources/dv-360.md +++ b/docs/integrations/sources/dv-360.md @@ -1,5 +1,19 @@ # Display & Video 360 +:::warning + +## Deprecation Notice + +The Display & Video 360 source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + Google DoubleClick Bid Manager (DBM) is the API that enables developers to manage Queries and retrieve Reports from Display & Video 360. DoubleClick Bid Manager API `v1.1` is the latest available and recommended version. diff --git a/docs/integrations/sources/dynamodb.md b/docs/integrations/sources/dynamodb.md index 9ffddd23ac7a..9cf2ed7bcebb 100644 --- a/docs/integrations/sources/dynamodb.md +++ b/docs/integrations/sources/dynamodb.md @@ -1,6 +1,7 @@ # Dynamodb -The Dynamodb source allows you to sync data from Dynamodb. The source supports Full Refresh and Incremental sync strategies. +The Dynamodb source allows you to sync data from Dynamodb. The source supports Full Refresh and +Incremental sync strategies. ## Resulting schema @@ -8,27 +9,29 @@ Dynamodb doesn't have table schemas. The discover phase has three steps: ### Step 1. Retrieve items -The connector scans the table with a scan limit of 1k and if the data set size is > 1MB it will initiate another -scan with the same limit until it has >= 1k items. +The connector scans the table with a scan limit of 1k and if the data set size is > 1MB it will +initiate another scan with the same limit until it has >= 1k items. ### Step 2. Combining attributes -After retrieving the items it will combine all the different top level attributes found in the retrieved items. The implementation -assumes that the same attribute present in different items has the same type and possibly nested attributes values. +After retrieving the items it will combine all the different top level attributes found in the +retrieved items. The implementation assumes that the same attribute present in different items has +the same type and possibly nested attributes values. ### Step 3. Determine property types -For each item attribute found the connector determines its type by calling AttributeValue.type(), depending on the received type it will map the -attribute to one of the supported Airbyte types in the schema. +For each item attribute found the connector determines its type by calling AttributeValue.type(), +depending on the received type it will map the attribute to one of the supported Airbyte types in +the schema. ## Features -| Feature | Supported | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Replicate Incremental Deletes | No | -| Namespaces | No | +| Feature | Supported | +| :---------------------------- | :-------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Replicate Incremental Deletes | No | +| Namespaces | No | ### Full Refresh sync @@ -38,12 +41,15 @@ Works as usual full refresh sync. Cursor field can't be nested, and it needs to be top level attribute in the item. -Cursor should **never** be blank. and it needs to be either a string or integer type - the incremental sync results might be unpredictable and will totally rely on Dynamodb comparison algorithm. +Cursor should **never** be blank. and it needs to be either a string or integer type - the +incremental sync results might be unpredictable and will totally rely on Dynamodb comparison +algorithm. -Only `ISO 8601` and `epoch` cursor types are supported. Cursor type is determined based on the property type present in the previously generated schema: +Only `ISO 8601` and `epoch` cursor types are supported. Cursor type is determined based on the +property type present in the previously generated schema: -* `ISO 8601` - if cursor type is string -* `epoch` - if cursor type is integer +- `ISO 8601` - if cursor type is string +- `epoch` - if cursor type is integer ## Getting started @@ -51,18 +57,22 @@ This guide describes in details how you can configure the connector to connect w ### Сonfiguration Parameters -* **_endpoint_**: aws endpoint of the dynamodb instance -* **_region_**: the region code of the dynamodb instance -* **_access_key_id_**: the access key for the IAM user with the required permissions -* **_secret_access_key_**: the secret key for the IAM user with the required permissions -* **_reserved_attribute_names_**: comma separated list of attribute names present in the replication tables which contain reserved words or special characters. https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html +- **_endpoint_**: aws endpoint of the dynamodb instance +- **_region_**: the region code of the dynamodb instance +- **_access_key_id_**: the access key for the IAM user with the required permissions +- **_secret_access_key_**: the secret key for the IAM user with the required permissions +- **_reserved_attribute_names_**: comma separated list of attribute names present in the replication + tables which contain reserved words or special characters. + https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html ## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:------------------------------------------------|:---------------------------------------------------------------------| -| 0.1.2 | 01-19-2023 | https://github.com/airbytehq/airbyte/pull/20172 | Fix reserved words in projection expression & make them configurable | -| 0.1.1 | 02-09-2023 | https://github.com/airbytehq/airbyte/pull/22682 | Fix build | -| 0.1.0 | 11-14-2022 | https://github.com/airbytehq/airbyte/pull/18750 | Initial version | - +| Version | Date | Pull Request | Subject | +|:--------| :--------- | :-------------------------------------------------------- |:---------------------------------------------------------------------| +| 0.2.3 | 2024-02-13 | [35232](https://github.com/airbytehq/airbyte/pull/35232) | Adopt CDK 0.20.4 | +| 0.2.2 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.2.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.2.0 | 18-12-2023 | https://github.com/airbytehq/airbyte/pull/33485 | Remove LEGACY state | +| 0.1.2 | 01-19-2023 | https://github.com/airbytehq/airbyte/pull/20172 | Fix reserved words in projection expression & make them configurable | +| 0.1.1 | 02-09-2023 | https://github.com/airbytehq/airbyte/pull/22682 | Fix build | +| 0.1.0 | 11-14-2022 | https://github.com/airbytehq/airbyte/pull/18750 | Initial version | diff --git a/docs/integrations/sources/e2e-test-cloud.md b/docs/integrations/sources/e2e-test-cloud.md index 633e65c3e548..d93a76732dc6 100644 --- a/docs/integrations/sources/e2e-test-cloud.md +++ b/docs/integrations/sources/e2e-test-cloud.md @@ -30,6 +30,7 @@ The OSS and Cloud variants have the same version number. The Cloud variant was i | Version | Date | Pull request | Notes | |---------|------------|----------------------------------------------------------|-----------------------------------------------------| +| 2.2.1 | 2024-02-13 | [35231](https://github.com/airbytehq/airbyte/pull/35231) | Adopt JDK 0.20.4. | | 2.1.5 | 2023-10-06 | [31092](https://github.com/airbytehq/airbyte/pull/31092) | Bring in changes from oss | | 2.1.4 | 2023-03-01 | [23656](https://github.com/airbytehq/airbyte/pull/23656) | Fix inheritance between e2e-test and e2e-test-cloud | | 0.1.0 | 2021-07-23 | [9720](https://github.com/airbytehq/airbyte/pull/9720) | Initial release. | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index 9988c9f497a7..76608078b542 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -72,6 +72,8 @@ The OSS and Cloud variants have the same version number. The Cloud variant was i | Version | Date | Pull request | Notes | |---------|------------| ------------------------------------------------------------------ |-------------------------------------------------------------------------------------------------------| +| 2.2.1 | 2024-02-13 | [35231](https://github.com/airbytehq/airbyte/pull/35231) | Adopt JDK 0.20.4. | +| 2.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | | 2.1.5 | 2023-10-04 | [31092](https://github.com/airbytehq/airbyte/pull/31092) | Bump jsonschemafriend dependency version to fix bug | | 2.1.4 | 2023-03-01 | [23656](https://github.com/airbytehq/airbyte/pull/23656) | Add speed benchmark mode to e2e test | | 2.1.3 | 2022-08-25 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Declare supported sync modes in catalogs | diff --git a/docs/integrations/sources/elasticsearch.md b/docs/integrations/sources/elasticsearch.md index 8767d5a7118a..2aa1a3fbb617 100644 --- a/docs/integrations/sources/elasticsearch.md +++ b/docs/integrations/sources/elasticsearch.md @@ -83,6 +83,8 @@ all values in the array must be of the same data type. Hence, every field can be ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------- | +|:--------| :--------- | :------------------------------------------------------- | :-------------- | +| 0.1.2 | 2024-02-13 | [35230](https://github.com/airbytehq/airbyte/pull/35230) | Adopt CDK 0.20.4 | +| `0.1.2` | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | | `0.1.1` | 2022-12-02 | [18118](https://github.com/airbytehq/airbyte/pull/18118) | Avoid too_long_frame_exception | | `0.1.0` | 2022-07-12 | [14118](https://github.com/airbytehq/airbyte/pull/14118) | Initial Release | diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 3b5f19785538..df8eafd7c64e 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -70,7 +70,7 @@ You can use the [Access Token Tool](https://developers.facebook.com/tools/access #### Facebook Marketing Source Settings -1. For **Account ID**, enter the [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. +1. For **Account ID(s)**, enter one or multiple comma-separated [Facebook Ad Account ID Numbers](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. 2. (Optional) For **Start Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. If not set then all data will be replicated for usual streams and only last 2 years for insight streams. :::warning @@ -78,14 +78,11 @@ You can use the [Access Token Tool](https://developers.facebook.com/tools/access ::: 3. (Optional) For **End Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. This is the date until which you'd like to replicate data for all Incremental streams. All data generated between the start date and this end date will be replicated. Not setting this option will result in always syncing the latest data. -4. (Optional) Toggle the **Include Deleted Campaigns, Ads, and AdSets** button to include data from deleted Campaigns, Ads, and AdSets. - - :::info - The Facebook Marketing API does not have a concept of deleting records in the same way that a database does. While you can archive or delete an ad campaign, the API maintains a record of the campaign. Toggling the **Include Deleted** button lets you replicate records for campaigns or ads even if they were archived or deleted from the Facebook platform. - ::: - -5. (Optional) Toggle the **Fetch Thumbnail Images** button to fetch the `thumbnail_url` and store the result in `thumbnail_data_url` for each [Ad Creative](https://developers.facebook.com/docs/marketing-api/creative/). -6. (Optional) In the **Custom Insights** section, you may provide a list of ad statistics entries. Each entry should have a unique name and can contain fields, breakdowns or action_breakdowns. Fields refer to the different data points you can collect from an ad, while breakdowns and action_breakdowns let you segment this data for more detailed insights. Click on **Add** to create a new entry in this list. +4. (Optional) Multiselect the **Campaign Statuses** to include data from Campaigns for particular statuses. +5. (Optional) Multiselect the **AdSet Statuses** to include data from AdSets for particular statuses. +6. (Optional) Multiselect the **Ad Statuses** to include data from Ads for particular statuses. +7. (Optional) Toggle the **Fetch Thumbnail Images** button to fetch the `thumbnail_url` and store the result in `thumbnail_data_url` for each [Ad Creative](https://developers.facebook.com/docs/marketing-api/creative/). +8. (Optional) In the **Custom Insights** section, you may provide a list of ad statistics entries. Each entry should have a unique name and can contain fields, breakdowns or action_breakdowns. Fields refer to the different data points you can collect from an ad, while breakdowns and action_breakdowns let you segment this data for more detailed insights. Click on **Add** to create a new entry in this list. :::note To retrieve specific fields from Facebook Ads Insights combined with other breakdowns, you can choose which fields and breakdowns to sync. However, please note that not all fields can be requested, and many are only functional when combined with specific other fields. For example, the breakdown `app_id` is only supported with the `total_postbacks` field. For more information on the breakdown limitations, refer to the [Facebook documentation](https://developers.facebook.com/docs/marketing-api/insights/breakdowns). @@ -113,9 +110,10 @@ You can use the [Access Token Tool](https://developers.facebook.com/tools/access Additional data streams for your Facebook Marketing connector are dynamically generated according to the Custom Insights you specify. If you have an existing Facebook Marketing source and you decide to update or remove some of your Custom Insights, you must also adjust the connections that sync to these streams. Specifically, you should either disable these connections or refresh the source schema associated with them to reflect the changes. ::: -7. (Optional) For **Page Size of Requests**, you can specify the number of records per page for paginated responses. Most users do not need to set this field unless specific issues arise or there are unique use cases that require tuning the connector's settings. The default value is set to retrieve 100 records per page. -8. (Optional) For **Insights Window Lookback**, you may set a window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. Otherwise, you may leave it at the default value of 28. For more information on action attributions, please refer to [the Meta Help Center](https://www.facebook.com/business/help/458681590974355?id=768381033531365). -9. Click **Set up source** and wait for the tests to complete. +9. (Optional) For **Page Size of Requests**, you can specify the number of records per page for paginated responses. Most users do not need to set this field unless specific issues arise or there are unique use cases that require tuning the connector's settings. The default value is set to retrieve 100 records per page. +10. (Optional) For **Insights Window Lookback**, you may set a window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. Otherwise, you may leave it at the default value of 28. For more information on action attributions, please refer to [the Meta Help Center](https://www.facebook.com/business/help/458681590974355?id=768381033531365). +11. (Optional) For **Insights Job Timeout**, you may set a custom value in range from 10 to 60. It establishes the maximum amount of time (in minutes) of waiting for the report job to complete. +12. Click **Set up source** and wait for the tests to complete. @@ -200,131 +198,140 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | -| 1.2.0 | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extend the `AdCreatives` stream schema | -| 1.1.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 1.1.16 | 2023-10-11 | [31284](https://github.com/airbytehq/airbyte/pull/31284) | Fix error occurring when trying to access the `funding_source_details` field of the `AdAccount` stream | -| 1.1.15 | 2023-10-06 | [31132](https://github.com/airbytehq/airbyte/pull/31132) | Fix permission error for `AdAccount` stream | -| 1.1.14 | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | -| 1.1.13 | 2023-09-22 | [30706](https://github.com/airbytehq/airbyte/pull/30706) | Performance testing - include socat binary in docker image | -| 1.1.12 | 2023-09-22 | [30655](https://github.com/airbytehq/airbyte/pull/30655) | Updated doc; improved schema for custom insight streams; updated SAT or custom insight streams; removed obsolete optional max_batch_size option from spec | -| 1.1.11 | 2023-09-21 | [30650](https://github.com/airbytehq/airbyte/pull/30650) | Fix None issue since start_date is optional | -| 1.1.10 | 2023-09-15 | [30485](https://github.com/airbytehq/airbyte/pull/30485) | added 'status' and 'configured_status' fields for campaigns stream schema | -| 1.1.9 | 2023-08-31 | [29994](https://github.com/airbytehq/airbyte/pull/29994) | Removed batch processing, updated description in specs, added user-friendly error message, removed start_date from required attributes | -| 1.1.8 | 2023-09-04 | [29666](https://github.com/airbytehq/airbyte/pull/29666) | Adding custom field `boosted_object_id` to a streams schema in `campaigns` catalog `CustomAudiences` | -| 1.1.7 | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences` | -| 1.1.6 | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch | -| 1.1.5 | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size | -| 1.1.4 | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream | -| 1.1.3 | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check | -| 1.1.2 | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec` | -| 1.1.1 | 2023-07-26 | [27996](https://github.com/airbytehq/airbyte/pull/27996) | Remove reference to authSpecification | -| 1.1.0 | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | Add new `action_report_time` attribute to `AdInsights` class | -| 1.0.1 | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error` (code 99) with the retry strategy | -| 1.0.0 | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17 | -| 0.5.0 | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2 | -| 0.4.3 | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day | -| 0.4.2 | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec | -| 0.4.1 | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead | -| 0.4.0 | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports | -| 0.3.7 | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors | -| 0.3.6 | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification | -| 0.3.5 | 2023-04-26 | [24994](https://github.com/airbytehq/airbyte/pull/24994) | Emit stream status messages | -| 0.3.4 | 2023-04-18 | [22990](https://github.com/airbytehq/airbyte/pull/22990) | Increase pause interval | -| 0.3.3 | 2023-04-14 | [25204](https://github.com/airbytehq/airbyte/pull/25204) | Fix data retention period validation | -| 0.3.2 | 2023-04-08 | [25003](https://github.com/airbytehq/airbyte/pull/25003) | Don't fetch `thumbnail_data_url` if it's None | -| 0.3.1 | 2023-03-27 | [24600](https://github.com/airbytehq/airbyte/pull/24600) | Reduce request record limit when retrying second page or further | -| 0.3.0 | 2023-03-16 | [19141](https://github.com/airbytehq/airbyte/pull/19141) | Added Level parameter to custom Ads Insights | -| 0.2.86 | 2023-03-01 | [23625](https://github.com/airbytehq/airbyte/pull/23625) | Add user friendly fields description in spec and docs. Extend error message for invalid Account ID case. | -| 0.2.85 | 2023-02-14 | [23003](https://github.com/airbytehq/airbyte/pull/23003) | Bump facebook_business to 16.0.0 | -| 0.2.84 | 2023-01-27 | [22003](https://github.com/airbytehq/airbyte/pull/22003) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.83 | 2023-01-13 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Videos stream remove filtering | -| 0.2.82 | 2023-01-09 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Fix AdAccount schema | -| 0.2.81 | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request | -| 0.2.80 | 2022-12-21 | [20736](https://github.com/airbytehq/airbyte/pull/20736) | Fix update next cursor | -| 0.2.79 | 2022-12-07 | [20402](https://github.com/airbytehq/airbyte/pull/20402) | Exclude Not supported fields from request | -| 0.2.78 | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | Fix fields permission error | -| 0.2.77 | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | Update next cursor value at read start | -| 0.2.76 | 2022-12-03 | [20043](https://github.com/airbytehq/airbyte/pull/20043) | Allows `action_breakdowns` to be an empty list - bugfix for #20016 | -| 0.2.75 | 2022-12-03 | [20016](https://github.com/airbytehq/airbyte/pull/20016) | Allows `action_breakdowns` to be an empty list | -| 0.2.74 | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed | -| 0.2.73 | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations | -| 0.2.72 | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | Handle FacebookBadObjectError for empty results on async jobs | -| 0.2.71 | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry | -| 0.2.70 | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 | -| 0.2.69 | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema | -| 0.2.68 | 2022-10-12 | [17869](https://github.com/airbytehq/airbyte/pull/17869) | Remove "format" from optional datetime `end_date` field | -| 0.2.67 | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema | -| 0.2.65 | 2022-09-29 | [17371](https://github.com/airbytehq/airbyte/pull/17371) | Fix stream CustomConversions `enable_deleted=False` | -| 0.2.64 | 2022-09-22 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | -| 0.2.64 | 2022-09-22 | [17027](https://github.com/airbytehq/airbyte/pull/17027) | Limit time range with 37 months when creating an insight job from lower edge object. Retry bulk request when getting error code `960` | -| 0.2.63 | 2022-09-06 | [15724](https://github.com/airbytehq/airbyte/pull/15724) | Add the Custom Conversion stream | -| 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | -| 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | -| 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | -| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | -| 0.2.58 | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue | -| 0.2.57 | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0 | -| 0.2.56 | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation | -| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | -| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | -| 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | -| 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | -| 0.2.51 | 2022-05-30 | [13317](https://github.com/airbytehq/airbyte/pull/13317) | Change tax_id to string (Canadian has letter in tax_id) | -| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insights streams | -| 0.2.49 | 2022-05-20 | [13047](https://github.com/airbytehq/airbyte/pull/13047) | Fix duplicating records during insights lookback period | -| 0.2.48 | 2022-05-19 | [13008](https://github.com/airbytehq/airbyte/pull/13008) | Update CDK to v0.1.58 avoid crashing on incorrect stream schemas | -| 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | -| 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | -| 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | -| 0.2.44 | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID | -| 0.2.43 | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object | -| 0.2.42 | 2022-04-06 | [11761](https://github.com/airbytehq/airbyte/pull/11761) | Upgrade Facebook Python SDK to version 13 | -| 0.2.41 | 2022-03-28 | [11446](https://github.com/airbytehq/airbyte/pull/11446) | Increase number of attempts for individual jobs | -| 0.2.40 | 2022-02-28 | [10698](https://github.com/airbytehq/airbyte/pull/10698) | Improve sleeps time in rate limit handler | -| 0.2.39 | 2022-03-09 | [10917](https://github.com/airbytehq/airbyte/pull/10917) | Retry connections when FB API returns error code 2 (temporary oauth error) | -| 0.2.38 | 2022-03-08 | [10531](https://github.com/airbytehq/airbyte/pull/10531) | Add `time_increment` parameter to custom insights | -| 0.2.37 | 2022-02-28 | [10655](https://github.com/airbytehq/airbyte/pull/10655) | Add Activities stream | -| 0.2.36 | 2022-02-24 | [10588](https://github.com/airbytehq/airbyte/pull/10588) | Fix `execute_in_batch` for large amount of requests | -| 0.2.35 | 2022-02-18 | [10348](https://github.com/airbytehq/airbyte/pull/10348) | Add error code 104 to backoff triggers | -| 0.2.34 | 2022-02-17 | [10180](https://github.com/airbytehq/airbyte/pull/9805) | Performance and reliability fixes | -| 0.2.33 | 2021-12-28 | [10180](https://github.com/airbytehq/airbyte/pull/10180) | Add AdAccount and Images streams | -| 0.2.32 | 2022-01-07 | [10138](https://github.com/airbytehq/airbyte/pull/10138) | Add `primary_key` for all insights streams. | -| 0.2.31 | 2021-12-29 | [9138](https://github.com/airbytehq/airbyte/pull/9138) | Fix videos stream format field incorrect type | -| 0.2.30 | 2021-12-20 | [8962](https://github.com/airbytehq/airbyte/pull/8962) | Add `asset_feed_spec` field to `ad creatives` stream | -| 0.2.29 | 2021-12-17 | [8649](https://github.com/airbytehq/airbyte/pull/8649) | Retrieve ad_creatives image as data encoded | -| 0.2.28 | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields | -| 0.2.27 | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream | -| 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | -| 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs | -| 0.2.24 | 2021-11-09 | [7744](https://github.com/airbytehq/airbyte/pull/7744) | Fix fail when async job takes too long | -| 0.2.23 | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema | -| 0.2.22 | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | -| 0.2.21 | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns | -| 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook_business package to 12.0 | -| 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.2.18 | 2021-09-28 | [6499](https://github.com/airbytehq/airbyte/pull/6499) | Fix field values converting fail | -| 0.2.17 | 2021-09-14 | [4978](https://github.com/airbytehq/airbyte/pull/4978) | Convert values' types according to schema types | -| 0.2.16 | 2021-09-14 | [6060](https://github.com/airbytehq/airbyte/pull/6060) | Fix schema for `ads_insights` stream | -| 0.2.15 | 2021-09-14 | [5958](https://github.com/airbytehq/airbyte/pull/5958) | Fix url parsing and add report that exposes conversions | -| 0.2.14 | 2021-07-19 | [4820](https://github.com/airbytehq/airbyte/pull/4820) | Improve the rate limit management | -| 0.2.12 | 2021-06-20 | [3743](https://github.com/airbytehq/airbyte/pull/3743) | Refactor connector to use CDK: - Improve error handling. - Improve async job performance \(insights\). - Add new configuration parameter `insights_days_per_job`. - Rename stream `adsets` to `ad_sets`. - Refactor schema logic for insights, allowing to configure any possible insight stream. | -| 0.2.10 | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Update version of facebook_business to 11.0 | -| 0.2.9 | 2021-06-10 | [3996](https://github.com/airbytehq/airbyte/pull/3996) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.2.8 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add 80000 as a rate-limiting error code | -| 0.2.7 | 2021-06-03 | [3646](https://github.com/airbytehq/airbyte/pull/3646) | Add missing fields to AdInsights streams | -| 0.2.6 | 2021-05-25 | [3525](https://github.com/airbytehq/airbyte/pull/3525) | Fix handling call rate limit | -| 0.2.5 | 2021-05-20 | [3396](https://github.com/airbytehq/airbyte/pull/3396) | Allow configuring insights lookback window | -| 0.2.4 | 2021-05-13 | [3395](https://github.com/airbytehq/airbyte/pull/3395) | Fix an issue that caused losing Insights data from the past 28 days while incremental sync | -| 0.2.3 | 2021-04-28 | [3116](https://github.com/airbytehq/airbyte/pull/3116) | Wait longer \(5 min\) for async jobs to start | -| 0.2.2 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.2.1 | 2021-03-12 | [2391](https://github.com/airbytehq/airbyte/pull/2391) | Support FB Marketing API v10 | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.4 | 2021-02-24 | [1902](https://github.com/airbytehq/airbyte/pull/1902) | Add `include_deleted` option in params | -| 0.1.3 | 2021-02-15 | [1990](https://github.com/airbytehq/airbyte/pull/1990) | Support Insights stream via async queries | -| 0.1.2 | 2021-01-22 | [1699](https://github.com/airbytehq/airbyte/pull/1699) | Add incremental support | -| 0.1.1 | 2021-01-15 | [1552](https://github.com/airbytehq/airbyte/pull/1552) | Release Native Facebook Marketing Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.4.2 | 2024-02-22 | [35539](https://github.com/airbytehq/airbyte/pull/35539) | Add missing config migration from `include_deleted` field | +| 1.4.1 | 2024-02-21 | [35467](https://github.com/airbytehq/airbyte/pull/35467) | Fix error with incorrect state transforming in the 1.4.0 version | +| 1.4.0 | 2024-02-20 | [32449](https://github.com/airbytehq/airbyte/pull/32449) | Replace "Include Deleted Campaigns, Ads, and AdSets" option in configuration with specific statuses selection per stream | +| 1.3.3 | 2024-02-15 | [35061](https://github.com/airbytehq/airbyte/pull/35061) | Add integration tests | | +| 1.3.2 | 2024-02-12 | [35178](https://github.com/airbytehq/airbyte/pull/35178) | Manage dependencies with Poetry | +| 1.3.1 | 2024-02-05 | [34845](https://github.com/airbytehq/airbyte/pull/34845) | Add missing fields to schemas | +| 1.3.0 | 2024-01-09 | [33538](https://github.com/airbytehq/airbyte/pull/33538) | Updated the `Ad Account ID(s)` property to support multiple IDs | +| 1.2.3 | 2024-01-04 | [33934](https://github.com/airbytehq/airbyte/pull/33828) | Make ready for airbyte-lib | +| 1.2.2 | 2024-01-02 | [33828](https://github.com/airbytehq/airbyte/pull/33828) | Add insights job timeout to be an option, so a user can specify their own value | +| 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | +| 1.2.0 | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extend the `AdCreatives` stream schema | +| 1.1.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.16 | 2023-10-11 | [31284](https://github.com/airbytehq/airbyte/pull/31284) | Fix error occurring when trying to access the `funding_source_details` field of the `AdAccount` stream | +| 1.1.15 | 2023-10-06 | [31132](https://github.com/airbytehq/airbyte/pull/31132) | Fix permission error for `AdAccount` stream | +| 1.1.14 | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | +| 1.1.13 | 2023-09-22 | [30706](https://github.com/airbytehq/airbyte/pull/30706) | Performance testing - include socat binary in docker image | +| 1.1.12 | 2023-09-22 | [30655](https://github.com/airbytehq/airbyte/pull/30655) | Updated doc; improved schema for custom insight streams; updated SAT or custom insight streams; removed obsolete optional max_batch_size option from spec | +| 1.1.11 | 2023-09-21 | [30650](https://github.com/airbytehq/airbyte/pull/30650) | Fix None issue since start_date is optional | +| 1.1.10 | 2023-09-15 | [30485](https://github.com/airbytehq/airbyte/pull/30485) | added 'status' and 'configured_status' fields for campaigns stream schema | +| 1.1.9 | 2023-08-31 | [29994](https://github.com/airbytehq/airbyte/pull/29994) | Removed batch processing, updated description in specs, added user-friendly error message, removed start_date from required attributes | +| 1.1.8 | 2023-09-04 | [29666](https://github.com/airbytehq/airbyte/pull/29666) | Adding custom field `boosted_object_id` to a streams schema in `campaigns` catalog `CustomAudiences` | +| 1.1.7 | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences` | +| 1.1.6 | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch | +| 1.1.5 | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size | +| 1.1.4 | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream | +| 1.1.3 | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check | +| 1.1.2 | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec` | +| 1.1.1 | 2023-07-26 | [27996](https://github.com/airbytehq/airbyte/pull/27996) | Remove reference to authSpecification | +| 1.1.0 | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | Add new `action_report_time` attribute to `AdInsights` class | +| 1.0.1 | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error` (code 99) with the retry strategy | +| 1.0.0 | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17 | +| 0.5.0 | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2 | +| 0.4.3 | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day | +| 0.4.2 | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec | +| 0.4.1 | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead | +| 0.4.0 | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports | +| 0.3.7 | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors | +| 0.3.6 | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification | +| 0.3.5 | 2023-04-26 | [24994](https://github.com/airbytehq/airbyte/pull/24994) | Emit stream status messages | +| 0.3.4 | 2023-04-18 | [22990](https://github.com/airbytehq/airbyte/pull/22990) | Increase pause interval | +| 0.3.3 | 2023-04-14 | [25204](https://github.com/airbytehq/airbyte/pull/25204) | Fix data retention period validation | +| 0.3.2 | 2023-04-08 | [25003](https://github.com/airbytehq/airbyte/pull/25003) | Don't fetch `thumbnail_data_url` if it's None | +| 0.3.1 | 2023-03-27 | [24600](https://github.com/airbytehq/airbyte/pull/24600) | Reduce request record limit when retrying second page or further | +| 0.3.0 | 2023-03-16 | [19141](https://github.com/airbytehq/airbyte/pull/19141) | Added Level parameter to custom Ads Insights | +| 0.2.86 | 2023-03-01 | [23625](https://github.com/airbytehq/airbyte/pull/23625) | Add user friendly fields description in spec and docs. Extend error message for invalid Account ID case. | +| 0.2.85 | 2023-02-14 | [23003](https://github.com/airbytehq/airbyte/pull/23003) | Bump facebook_business to 16.0.0 | +| 0.2.84 | 2023-01-27 | [22003](https://github.com/airbytehq/airbyte/pull/22003) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.83 | 2023-01-13 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Videos stream remove filtering | +| 0.2.82 | 2023-01-09 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Fix AdAccount schema | +| 0.2.81 | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request | +| 0.2.80 | 2022-12-21 | [20736](https://github.com/airbytehq/airbyte/pull/20736) | Fix update next cursor | +| 0.2.79 | 2022-12-07 | [20402](https://github.com/airbytehq/airbyte/pull/20402) | Exclude Not supported fields from request | +| 0.2.78 | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | Fix fields permission error | +| 0.2.77 | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | Update next cursor value at read start | +| 0.2.76 | 2022-12-03 | [20043](https://github.com/airbytehq/airbyte/pull/20043) | Allows `action_breakdowns` to be an empty list - bugfix for #20016 | +| 0.2.75 | 2022-12-03 | [20016](https://github.com/airbytehq/airbyte/pull/20016) | Allows `action_breakdowns` to be an empty list | +| 0.2.74 | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed | +| 0.2.73 | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations | +| 0.2.72 | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | Handle FacebookBadObjectError for empty results on async jobs | +| 0.2.71 | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry | +| 0.2.70 | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 | +| 0.2.69 | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema | +| 0.2.68 | 2022-10-12 | [17869](https://github.com/airbytehq/airbyte/pull/17869) | Remove "format" from optional datetime `end_date` field | +| 0.2.67 | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema | +| 0.2.65 | 2022-09-29 | [17371](https://github.com/airbytehq/airbyte/pull/17371) | Fix stream CustomConversions `enable_deleted=False` | +| 0.2.64 | 2022-09-22 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.2.64 | 2022-09-22 | [17027](https://github.com/airbytehq/airbyte/pull/17027) | Limit time range with 37 months when creating an insight job from lower edge object. Retry bulk request when getting error code `960` | +| 0.2.63 | 2022-09-06 | [15724](https://github.com/airbytehq/airbyte/pull/15724) | Add the Custom Conversion stream | +| 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | +| 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | +| 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | +| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | +| 0.2.58 | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue | +| 0.2.57 | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0 | +| 0.2.56 | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation | +| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | +| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | +| 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | +| 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | +| 0.2.51 | 2022-05-30 | [13317](https://github.com/airbytehq/airbyte/pull/13317) | Change tax_id to string (Canadian has letter in tax_id) | +| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insights streams | +| 0.2.49 | 2022-05-20 | [13047](https://github.com/airbytehq/airbyte/pull/13047) | Fix duplicating records during insights lookback period | +| 0.2.48 | 2022-05-19 | [13008](https://github.com/airbytehq/airbyte/pull/13008) | Update CDK to v0.1.58 avoid crashing on incorrect stream schemas | +| 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | +| 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | +| 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | +| 0.2.44 | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID | +| 0.2.43 | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object | +| 0.2.42 | 2022-04-06 | [11761](https://github.com/airbytehq/airbyte/pull/11761) | Upgrade Facebook Python SDK to version 13 | +| 0.2.41 | 2022-03-28 | [11446](https://github.com/airbytehq/airbyte/pull/11446) | Increase number of attempts for individual jobs | +| 0.2.40 | 2022-02-28 | [10698](https://github.com/airbytehq/airbyte/pull/10698) | Improve sleeps time in rate limit handler | +| 0.2.39 | 2022-03-09 | [10917](https://github.com/airbytehq/airbyte/pull/10917) | Retry connections when FB API returns error code 2 (temporary oauth error) | +| 0.2.38 | 2022-03-08 | [10531](https://github.com/airbytehq/airbyte/pull/10531) | Add `time_increment` parameter to custom insights | +| 0.2.37 | 2022-02-28 | [10655](https://github.com/airbytehq/airbyte/pull/10655) | Add Activities stream | +| 0.2.36 | 2022-02-24 | [10588](https://github.com/airbytehq/airbyte/pull/10588) | Fix `execute_in_batch` for large amount of requests | +| 0.2.35 | 2022-02-18 | [10348](https://github.com/airbytehq/airbyte/pull/10348) | Add error code 104 to backoff triggers | +| 0.2.34 | 2022-02-17 | [10180](https://github.com/airbytehq/airbyte/pull/9805) | Performance and reliability fixes | +| 0.2.33 | 2021-12-28 | [10180](https://github.com/airbytehq/airbyte/pull/10180) | Add AdAccount and Images streams | +| 0.2.32 | 2022-01-07 | [10138](https://github.com/airbytehq/airbyte/pull/10138) | Add `primary_key` for all insights streams. | +| 0.2.31 | 2021-12-29 | [9138](https://github.com/airbytehq/airbyte/pull/9138) | Fix videos stream format field incorrect type | +| 0.2.30 | 2021-12-20 | [8962](https://github.com/airbytehq/airbyte/pull/8962) | Add `asset_feed_spec` field to `ad creatives` stream | +| 0.2.29 | 2021-12-17 | [8649](https://github.com/airbytehq/airbyte/pull/8649) | Retrieve ad_creatives image as data encoded | +| 0.2.28 | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields | +| 0.2.27 | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream | +| 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | +| 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs | +| 0.2.24 | 2021-11-09 | [7744](https://github.com/airbytehq/airbyte/pull/7744) | Fix fail when async job takes too long | +| 0.2.23 | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema | +| 0.2.22 | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | +| 0.2.21 | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns | +| 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook_business package to 12.0 | +| 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.2.18 | 2021-09-28 | [6499](https://github.com/airbytehq/airbyte/pull/6499) | Fix field values converting fail | +| 0.2.17 | 2021-09-14 | [4978](https://github.com/airbytehq/airbyte/pull/4978) | Convert values' types according to schema types | +| 0.2.16 | 2021-09-14 | [6060](https://github.com/airbytehq/airbyte/pull/6060) | Fix schema for `ads_insights` stream | +| 0.2.15 | 2021-09-14 | [5958](https://github.com/airbytehq/airbyte/pull/5958) | Fix url parsing and add report that exposes conversions | +| 0.2.14 | 2021-07-19 | [4820](https://github.com/airbytehq/airbyte/pull/4820) | Improve the rate limit management | +| 0.2.12 | 2021-06-20 | [3743](https://github.com/airbytehq/airbyte/pull/3743) | Refactor connector to use CDK: - Improve error handling. - Improve async job performance \(insights\). - Add new configuration parameter `insights_days_per_job`. - Rename stream `adsets` to `ad_sets`. - Refactor schema logic for insights, allowing to configure any possible insight stream. | +| 0.2.10 | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Update version of facebook_business to 11.0 | +| 0.2.9 | 2021-06-10 | [3996](https://github.com/airbytehq/airbyte/pull/3996) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.2.8 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add 80000 as a rate-limiting error code | +| 0.2.7 | 2021-06-03 | [3646](https://github.com/airbytehq/airbyte/pull/3646) | Add missing fields to AdInsights streams | +| 0.2.6 | 2021-05-25 | [3525](https://github.com/airbytehq/airbyte/pull/3525) | Fix handling call rate limit | +| 0.2.5 | 2021-05-20 | [3396](https://github.com/airbytehq/airbyte/pull/3396) | Allow configuring insights lookback window | +| 0.2.4 | 2021-05-13 | [3395](https://github.com/airbytehq/airbyte/pull/3395) | Fix an issue that caused losing Insights data from the past 28 days while incremental sync | +| 0.2.3 | 2021-04-28 | [3116](https://github.com/airbytehq/airbyte/pull/3116) | Wait longer \(5 min\) for async jobs to start | +| 0.2.2 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | +| 0.2.1 | 2021-03-12 | [2391](https://github.com/airbytehq/airbyte/pull/2391) | Support FB Marketing API v10 | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.4 | 2021-02-24 | [1902](https://github.com/airbytehq/airbyte/pull/1902) | Add `include_deleted` option in params | +| 0.1.3 | 2021-02-15 | [1990](https://github.com/airbytehq/airbyte/pull/1990) | Support Insights stream via async queries | +| 0.1.2 | 2021-01-22 | [1699](https://github.com/airbytehq/airbyte/pull/1699) | Add incremental support | +| 0.1.1 | 2021-01-15 | [1552](https://github.com/airbytehq/airbyte/pull/1552) | Release Native Facebook Marketing Connector | diff --git a/docs/integrations/sources/faker-migrations.md b/docs/integrations/sources/faker-migrations.md index 46dc3247f93f..7eac0a780b1f 100644 --- a/docs/integrations/sources/faker-migrations.md +++ b/docs/integrations/sources/faker-migrations.md @@ -1,6 +1,11 @@ # Sample Data (Faker) Migration Guide +## Upgrading to 6.0.0 + +All streams (`users`, `products`, and `purchases`) now properly declare `id` as their respective primary keys. Existing sync jobs should still work as expected but you may need to reset your sync and/or update write mode after upgrading to the latest connector version. + ## Upgrading to 5.0.0 + Some columns are narrowing from `number` to `integer`. You may need to force normalization to rebuild your destination tables by manually dropping the SCD and final tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. ## Upgrading to 4.0.0 diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 8e2feb4f99b1..311d563a9ec2 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -2,11 +2,13 @@ ## Sync overview -The Sample Data (Faker) source generates sample data using the python [`mimesis`](https://mimesis.name/en/master/) package. +The Sample Data (Faker) source generates sample data using the python +[`mimesis`](https://mimesis.name/en/master/) package. ### Output schema -This source will generate an "e-commerce-like" dataset with users, products, and purchases. Here's what is produced at a Postgres destination connected to this source: +This source will generate an "e-commerce-like" dataset with users, products, and purchases. Here's +what is produced at a Postgres destination connected to this source: ```sql CREATE TABLE "public"."users" ( @@ -84,9 +86,12 @@ CREATE TABLE "public"."purchases" ( | Incremental Sync | Yes | | | Namespaces | No | | -Of note, if you choose `Incremental Sync`, state will be maintained between syncs, and once you hit `count` records, no new records will be added. +Of note, if you choose `Incremental Sync`, state will be maintained between syncs, and once you hit +`count` records, no new records will be added. -You can choose a specific `seed` (integer) as an option for this connector which will guarantee that the same fake records are generated each time. Otherwise, random data will be created on each subsequent sync. +You can choose a specific `seed` (integer) as an option for this connector which will guarantee that +the same fake records are generated each time. Otherwise, random data will be created on each +subsequent sync. ### Requirements @@ -95,7 +100,12 @@ None! ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:----------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 6.0.2 | 2024-02-12 | [35174](https://github.com/airbytehq/airbyte/pull/35174) | Manage dependencies with Poetry. | +| 6.0.1 | 2024-02-12 | [35172](https://github.com/airbytehq/airbyte/pull/35172) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 6.0.0 | 2024-01-30 | [34644](https://github.com/airbytehq/airbyte/pull/34644) | Declare 'id' columns as primary keys. | +| 5.0.2 | 2024-01-17 | [34344](https://github.com/airbytehq/airbyte/pull/34344) | Ensure unique state messages | +| 5.0.1 | 2023-01-08 | [34033](https://github.com/airbytehq/airbyte/pull/34033) | Add standard entrypoints for usage with AirbyteLib | | 5.0.0 | 2023-08-08 | [29213](https://github.com/airbytehq/airbyte/pull/29213) | Change all `*id` fields and `products.year` to be integer | | 4.0.0 | 2023-07-19 | [28485](https://github.com/airbytehq/airbyte/pull/28485) | Bump to test publication | | 3.0.2 | 2023-07-07 | [27807](https://github.com/airbytehq/airbyte/pull/28060) | Bump to test publication | diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 8afdbf2c912f..a2fef7e3bbf4 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -104,6 +104,7 @@ For example, if the format `CSV` is selected, then options from the [read_csv](h - It is therefore possible to customize the `delimiter` (or `sep`) to in case of tab separated files. - Header line can be ignored with `header=0` and customized with `names` +- If a file has no header, it is required to set `header=null`; otherwise, the first record will be missing - Parse dates for in specified columns - etc @@ -126,7 +127,7 @@ This connector does not support syncing unstructured data files such as raw text ## Supported sync modes | Feature | Supported? | -| ---------------------------------------- | ---------- | +|------------------------------------------|------------| | Full Refresh Sync | Yes | | Incremental Sync | No | | Replicate Incremental Deletes | No | @@ -140,9 +141,9 @@ This source produces a single table for the target file as it replicates only on ## File / Stream Compression | Compression | Supported? | -| ----------- | ---------- | +|-------------|------------| | Gzip | Yes | -| Zip | No | +| Zip | Yes | | Bzip2 | No | | Lzma | No | | Xz | No | @@ -151,7 +152,7 @@ This source produces a single table for the target file as it replicates only on ## Storage Providers | Storage Providers | Supported? | -| ---------------------- | ----------------------------------------------- | +|------------------------|-------------------------------------------------| | HTTPS | Yes | | Google Cloud Storage | Yes | | Amazon Web Services S3 | Yes | @@ -162,13 +163,14 @@ This source produces a single table for the target file as it replicates only on ### File Formats | Format | Supported? | -| --------------------- | ---------- | +|-----------------------|------------| | CSV | Yes | | JSON/JSONL | Yes | | HTML | No | | XML | No | | Excel | Yes | | Excel Binary Workbook | Yes | +| Fixed Width File | Yes | | Feather | Yes | | Parquet | Yes | | Pickle | No | @@ -182,24 +184,24 @@ Normally, Airbyte tries to infer the data type from the source, but you can use Here are a list of examples of possible file inputs: -| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | -| ----------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | +| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | +|-------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------|------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | | hr_and_financials | GCS | gs://airbyte-vault/financial.csv | smart_open or gcfs | `{"type": "service_account", "private_key_id": "XXXXXXXX", ...}` | data from a private bucket, a service account is necessary | -| landsat_index | GCS | gcp-public-data-landsat/index.csv.gz | smart_open | | Using smart_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | +| landsat_index | GCS | gcp-public-data-landsat/index.csv.gz | smart_open | | Using smart_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | Examples with reader options: -| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | -| ------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | +|---------------|---------|-------------------------------------------------|-------------|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| | landsat_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | `{"compression": "gzip"}` | Additional reader options to specify a compression option to `read_csv` | | GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | `{"sep": "\t", "header": null}` | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | | server_logs | local | /local/logs.log | | `{"sep": ";"}` | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | Example for SFTP: -| Dataset Name | Storage | User | Password | Host | URL | Reader Options | Description | -| ------------ | ------- | ---- | -------- | --------------- | ----------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Dataset Name | Storage | User | Password | Host | URL | Reader Options | Description | +|--------------|---------|------|----------|-----------------|-------------------------|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------| | Test Rebext | SFTP | demo | password | test.rebext.net | /pub/example/readme.txt | `{"sep": "\r\n", "header": null, "names": \["text"], "engine": "python"}` | We use `python` engine for `read_csv` in order to handle delimiter of more than 1 character while providing our own column names. | Please see (or add) more at `airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py` for further usages examples. @@ -214,63 +216,66 @@ In order to read large files from a remote location, this connector uses the [sm ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------| -| 0.3.15 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Upgrade to airbyte/python-connector-base:1.0.1 | -| 0.3.14 | 2023-10-13 | [30984](https://github.com/airbytehq/airbyte/pull/30984) | Prevent local file usage on cloud | -| 0.3.13 | 2023-10-12 | [31341](https://github.com/airbytehq/airbyte/pull/31341) | Build from airbyte/python-connector-base:1.0.0 | -| 0.3.12 | 2023-09-19 | [30579](https://github.com/airbytehq/airbyte/pull/30579) | Add ParserError handling for `discovery` | -| 0.3.11 | 2023-06-08 | [27157](https://github.com/airbytehq/airbyte/pull/27157) | Force smart open log level to ERROR | -| 0.3.10 | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107) | Make source-file testable in our new airbyte-ci pipelines | -| 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | Add ParserError handling | -| 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | -| 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | -| 0.3.6 | 2023-05-16 | [26115](https://github.com/airbytehq/airbyte/pull/26115) | Add retry on SSHException('Error reading SSH protocol banner') | -| 0.3.5 | 2023-05-16 | [26117](https://github.com/airbytehq/airbyte/pull/26117) | Check if reader options is a valid JSON object | -| 0.3.4 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | fix Pandas date-time parsing to airbyte type | -| 0.3.3 | 2023-05-04 | [25819](https://github.com/airbytehq/airbyte/pull/25819) | GCP service_account_json is a secret | -| 0.3.2 | 2023-05-01 | [25641](https://github.com/airbytehq/airbyte/pull/25641) | Handle network errors | -| 0.3.1 | 2023-04-27 | [25575](https://github.com/airbytehq/airbyte/pull/25575) | Fix OOM; read Excel files in chunks using `openpyxl` | -| 0.3.0 | 2023-04-24 | [25445](https://github.com/airbytehq/airbyte/pull/25445) | Add datatime format parsing support for csv files | -| 0.2.38 | 2023-04-12 | [23759](https://github.com/airbytehq/airbyte/pull/23759) | Fix column data types for numerical values | -| 0.2.37 | 2023-04-06 | [24525](https://github.com/airbytehq/airbyte/pull/24525) | Fix examples in spec | -| 0.2.36 | 2023-03-27 | [24588](https://github.com/airbytehq/airbyte/pull/24588) | Remove traceback from user messages. | -| 0.2.35 | 2023-03-03 | [24278](https://github.com/airbytehq/airbyte/pull/24278) | Read only file header when checking connectivity; read only a single chunk when discovering the schema. | -| 0.2.34 | 2023-03-03 | [23723](https://github.com/airbytehq/airbyte/pull/23723) | Update description in spec, make user-friendly error messages and docs. | -| 0.2.33 | 2023-01-04 | [21012](https://github.com/airbytehq/airbyte/pull/21012) | Fix special characters bug | -| 0.2.32 | 2022-12-21 | [20740](https://github.com/airbytehq/airbyte/pull/20740) | Source File: increase SSH timeout to 60s | -| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | -| 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | -| 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | -| 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Add retry logic for `Connection reset error - 104` | -| 0.2.27 | 2022-10-26 | [18481](https://github.com/airbytehq/airbyte/pull/18481) | Fix check for wrong format | -| 0.2.26 | 2022-10-18 | [18116](https://github.com/airbytehq/airbyte/pull/18116) | Transform Dropbox shared link | -| 0.2.25 | 2022-10-14 | [17994](https://github.com/airbytehq/airbyte/pull/17994) | Handle `UnicodeDecodeError` during discover step. | -| 0.2.24 | 2022-10-03 | [17504](https://github.com/airbytehq/airbyte/pull/17504) | Validate data for `HTTPS` while `check_connection` | -| 0.2.23 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | -| 0.2.22 | 2022-09-15 | [16772](https://github.com/airbytehq/airbyte/pull/16772) | Fix schema generation for JSON files containing arrays | -| 0.2.21 | 2022-08-26 | [15568](https://github.com/airbytehq/airbyte/pull/15568) | Specify `pyxlsb` library for Excel Binary Workbook files | -| 0.2.20 | 2022-08-23 | [15870](https://github.com/airbytehq/airbyte/pull/15870) | Fix CSV schema discovery | -| 0.2.19 | 2022-08-19 | [15768](https://github.com/airbytehq/airbyte/pull/15768) | Convert 'nan' to 'null' | -| 0.2.18 | 2022-08-16 | [15698](https://github.com/airbytehq/airbyte/pull/15698) | Cache binary stream to file for discover | -| 0.2.17 | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501) | Cache binary stream to file | -| 0.2.16 | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293) | Add support for encoding reader option | -| 0.2.15 | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269) | Bump `smart-open` version to 6.0.0 | -| 0.2.12 | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535) | Fix invalid schema generation for JSON files | -| 0.2.11 | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588) | Add support to YAML format | -| 0.2.9 | 2022-02-01 | [9974](https://github.com/airbytehq/airbyte/pull/9974) | Update airbyte-cdk 0.1.47 | -| 0.2.8 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | -| 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | -| 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | -| 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | -| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | -| 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | -| 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | -| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.10 | 2021-02-18 | [2118](https://github.com/airbytehq/airbyte/pull/2118) | Support JSONL format | -| 0.1.9 | 2021-02-02 | [1768](https://github.com/airbytehq/airbyte/pull/1768) | Add test cases for all formats | -| 0.1.8 | 2021-01-27 | [1738](https://github.com/airbytehq/airbyte/pull/1738) | Adopt connector best practices | -| 0.1.7 | 2020-12-16 | [1331](https://github.com/airbytehq/airbyte/pull/1331) | Refactor Python base connector | -| 0.1.6 | 2020-12-08 | [1249](https://github.com/airbytehq/airbyte/pull/1249) | Handle NaN values | -| 0.1.5 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| +| 0.4.0 | 2024-02-15 | [32354](https://github.com/airbytehq/airbyte/pull/32354) | Add Zip File Support | +| 0.3.17 | 2024-02-13 | [34678](https://github.com/airbytehq/airbyte/pull/34678) | Add Fixed-Width File Support | +| 0.3.16 | 2024-02-12 | [35186](https://github.com/airbytehq/airbyte/pull/35186) | Manage dependencies with Poetry | +| 0.3.15 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Upgrade to airbyte/python-connector-base:1.0.1 | +| 0.3.14 | 2023-10-13 | [30984](https://github.com/airbytehq/airbyte/pull/30984) | Prevent local file usage on cloud | +| 0.3.13 | 2023-10-12 | [31341](https://github.com/airbytehq/airbyte/pull/31341) | Build from airbyte/python-connector-base:1.0.0 | +| 0.3.12 | 2023-09-19 | [30579](https://github.com/airbytehq/airbyte/pull/30579) | Add ParserError handling for `discovery` | +| 0.3.11 | 2023-06-08 | [27157](https://github.com/airbytehq/airbyte/pull/27157) | Force smart open log level to ERROR | +| 0.3.10 | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107) | Make source-file testable in our new airbyte-ci pipelines | +| 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | Add ParserError handling | +| 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | +| 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | +| 0.3.6 | 2023-05-16 | [26115](https://github.com/airbytehq/airbyte/pull/26115) | Add retry on SSHException('Error reading SSH protocol banner') | +| 0.3.5 | 2023-05-16 | [26117](https://github.com/airbytehq/airbyte/pull/26117) | Check if reader options is a valid JSON object | +| 0.3.4 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | fix Pandas date-time parsing to airbyte type | +| 0.3.3 | 2023-05-04 | [25819](https://github.com/airbytehq/airbyte/pull/25819) | GCP service_account_json is a secret | +| 0.3.2 | 2023-05-01 | [25641](https://github.com/airbytehq/airbyte/pull/25641) | Handle network errors | +| 0.3.1 | 2023-04-27 | [25575](https://github.com/airbytehq/airbyte/pull/25575) | Fix OOM; read Excel files in chunks using `openpyxl` | +| 0.3.0 | 2023-04-24 | [25445](https://github.com/airbytehq/airbyte/pull/25445) | Add datatime format parsing support for csv files | +| 0.2.38 | 2023-04-12 | [23759](https://github.com/airbytehq/airbyte/pull/23759) | Fix column data types for numerical values | +| 0.2.37 | 2023-04-06 | [24525](https://github.com/airbytehq/airbyte/pull/24525) | Fix examples in spec | +| 0.2.36 | 2023-03-27 | [24588](https://github.com/airbytehq/airbyte/pull/24588) | Remove traceback from user messages. | +| 0.2.35 | 2023-03-03 | [24278](https://github.com/airbytehq/airbyte/pull/24278) | Read only file header when checking connectivity; read only a single chunk when discovering the schema. | +| 0.2.34 | 2023-03-03 | [23723](https://github.com/airbytehq/airbyte/pull/23723) | Update description in spec, make user-friendly error messages and docs. | +| 0.2.33 | 2023-01-04 | [21012](https://github.com/airbytehq/airbyte/pull/21012) | Fix special characters bug | +| 0.2.32 | 2022-12-21 | [20740](https://github.com/airbytehq/airbyte/pull/20740) | Source File: increase SSH timeout to 60s | +| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | +| 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | +| 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | +| 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Add retry logic for `Connection reset error - 104` | +| 0.2.27 | 2022-10-26 | [18481](https://github.com/airbytehq/airbyte/pull/18481) | Fix check for wrong format | +| 0.2.26 | 2022-10-18 | [18116](https://github.com/airbytehq/airbyte/pull/18116) | Transform Dropbox shared link | +| 0.2.25 | 2022-10-14 | [17994](https://github.com/airbytehq/airbyte/pull/17994) | Handle `UnicodeDecodeError` during discover step. | +| 0.2.24 | 2022-10-03 | [17504](https://github.com/airbytehq/airbyte/pull/17504) | Validate data for `HTTPS` while `check_connection` | +| 0.2.23 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.2.22 | 2022-09-15 | [16772](https://github.com/airbytehq/airbyte/pull/16772) | Fix schema generation for JSON files containing arrays | +| 0.2.21 | 2022-08-26 | [15568](https://github.com/airbytehq/airbyte/pull/15568) | Specify `pyxlsb` library for Excel Binary Workbook files | +| 0.2.20 | 2022-08-23 | [15870](https://github.com/airbytehq/airbyte/pull/15870) | Fix CSV schema discovery | +| 0.2.19 | 2022-08-19 | [15768](https://github.com/airbytehq/airbyte/pull/15768) | Convert 'nan' to 'null' | +| 0.2.18 | 2022-08-16 | [15698](https://github.com/airbytehq/airbyte/pull/15698) | Cache binary stream to file for discover | +| 0.2.17 | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501) | Cache binary stream to file | +| 0.2.16 | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293) | Add support for encoding reader option | +| 0.2.15 | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269) | Bump `smart-open` version to 6.0.0 | +| 0.2.12 | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535) | Fix invalid schema generation for JSON files | +| 0.2.11 | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588) | Add support to YAML format | +| 0.2.9 | 2022-02-01 | [9974](https://github.com/airbytehq/airbyte/pull/9974) | Update airbyte-cdk 0.1.47 | +| 0.2.8 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | +| 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | +| 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | +| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | +| 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | +| 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | +| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.10 | 2021-02-18 | [2118](https://github.com/airbytehq/airbyte/pull/2118) | Support JSONL format | +| 0.1.9 | 2021-02-02 | [1768](https://github.com/airbytehq/airbyte/pull/1768) | Add test cases for all formats | +| 0.1.8 | 2021-01-27 | [1738](https://github.com/airbytehq/airbyte/pull/1738) | Adopt connector best practices | +| 0.1.7 | 2020-12-16 | [1331](https://github.com/airbytehq/airbyte/pull/1331) | Refactor Python base connector | +| 0.1.6 | 2020-12-08 | [1249](https://github.com/airbytehq/airbyte/pull/1249) | Handle NaN values | +| 0.1.5 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index b148533bb801..f1f1b144c3de 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -68,6 +68,8 @@ If you don't use the start date Freshdesk will retrieve only the last 30 days. M | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------ | +| 3.0.7 | 2024-02-12 | [35187](https://github.com/airbytehq/airbyte/pull/35187) | Manage dependencies with Poetry. | +| 3.0.6 | 2024-01-10 | [34101](https://github.com/airbytehq/airbyte/pull/34101) | Base image migration: remove Dockerfile and use the python-connector-base image | | 3.0.5 | 2023-11-30 | [33000](https://github.com/airbytehq/airbyte/pull/33000) | Base image migration: remove Dockerfile and use the python-connector-base image | | 3.0.4 | 2023-06-24 | [27680](https://github.com/airbytehq/airbyte/pull/27680) | Fix formatting | | 3.0.3 | 2023-06-02 | [26978](https://github.com/airbytehq/airbyte/pull/26978) | Skip the stream if subscription level had changed during sync | diff --git a/docs/integrations/sources/freshservice.md b/docs/integrations/sources/freshservice.md index 6481eb888c3f..9ea271f15d07 100644 --- a/docs/integrations/sources/freshservice.md +++ b/docs/integrations/sources/freshservice.md @@ -54,6 +54,8 @@ Please read [How to find your API key](https://api.freshservice.com/#authenticat | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 1.3.1 | 2024-01-29 | [34633](https://github.com/airbytehq/airbyte/pull/34633) | Add backoff policy for `Requested Items` stream | +| 1.3.0 | 2024-01-15 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Add `Requested Items` stream | | 1.2.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK | | 1.1.0 | 2023-05-09 | [25929](https://github.com/airbytehq/airbyte/pull/25929) | Add stream for customer satisfaction survey responses endpoint | | 1.0.0 | 2023-05-02 | [25743](https://github.com/airbytehq/airbyte/pull/25743) | Correct data types in tickets, agents and requesters schemas to match Freshservice API | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 1206c289e4f6..c37c8f0268d4 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -29,17 +29,132 @@ Use the service account ID from above, grant read access to your target bucket. ### Set up the source in Airbyte UI -- Paste the service account JSON key to `service_account` -- Enter your GCS bucket name to `gcs_bucket` -- Enter path to your file(s) to `gcs_path` +- Paste the service account JSON key to the `Service Account Information` field +- Enter your GCS bucket name to the `Bucket` field +- Add a stream + 1. Give a **Name** to the stream + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported format is **CSV**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). +- Configure the optional **Start Date** parameter that marks a starting date and time in UTC for data replication. Any files that have _not_ been modified since this specified date/time will _not_ be replicated. Use the provided datepicker (recommended) or enter the desired date programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. Leaving this field blank will replicate data from all files that have not been excluded by the **Path Pattern** and **Path Prefix**. +- Click **Set up source** and wait for the tests to complete. + +## Path Patterns + +\(tl;dr -> path pattern syntax using [wcmatch.glob](https://facelessuser.github.io/wcmatch/glob/). GLOBSTAR and SPLIT flags are enabled.\) + +This connector can sync multiple files by using glob-style patterns, rather than requiring a specific path for every file. This enables: + +- Referencing many files with just one pattern, e.g. `**` would indicate every file in the folder. +- Referencing future files that don't exist yet \(and therefore don't have a specific path\). + +You must provide a path pattern. You can also provide many patterns split with \| for more complex directory layouts. + +Each path pattern is a reference from the _root_ of the folder, so don't include the root folder name itself in the pattern\(s\). + +Some example patterns: + +- `**` : match everything. +- `**/*.csv` : match all files with specific extension. +- `myFolder/**/*.csv` : match all csv files anywhere under myFolder. +- `*/**` : match everything at least one folder deep. +- `*/*/*/**` : match everything at least three folders deep. +- `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\). +- `x/*/y/*` : match all files that sit in sub-folder x -> any folder -> folder y. +- `**/prefix*.csv` : match all csv files with specific prefix. +- `**/prefix*.parquet` : match all parquet files with specific prefix. + +Let's look at a specific example, matching the following folder layout (`MyFolder` is the folder specified in the connector config as the root folder, which the patterns are relative to): + +```text +MyFolder + -> log_files + -> some_table_files + -> part1.csv + -> part2.csv + -> images + -> more_table_files + -> part3.csv + -> extras + -> misc + -> another_part1.csv +``` + +We want to pick up part1.csv, part2.csv and part3.csv \(excluding another_part1.csv for now\). We could do this a few different ways: + +- We could pick up every csv file called "partX" with the single pattern `**/part*.csv`. +- To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders. +- We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created. +- We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another_part1.csv". + +As you can probably tell, there are many ways to achieve the same goal with path patterns. We recommend using a pattern that ensures clarity and is robust against future additions to the directory structure. + +## User Schema + +When using the Avro, Jsonl, CSV or Parquet format, you can provide a schema to use for the output stream. **Note that this doesn't apply to the experimental Document file type format.** + +Providing a schema allows for more control over the output of this stream. Without a provided schema, columns and datatypes will be inferred from the first created file in the bucket matching your path pattern and suffix. This will probably be fine in most cases but there may be situations you want to enforce a schema instead, e.g.: + +- You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map. +- Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future. +- You want to purposely define types for every column. +- You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map. + +Or any other reason! The schema must be provided as valid JSON as a map of `{"column": "datatype"}` where each datatype is one of: + +- string +- number +- integer +- object +- array +- boolean +- null + +For example: + +- `{"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}` +- `{"username": "string", "friends": "array", "information": "object"}` + +## File Format Settings + +### CSV + +Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time. + +- **Header Definition**: How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can set a value for the "Skip rows before header" option to ignore the header row. +- **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`. +- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. +- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. +- **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data: + +``` +Product,Description,Price +Jeans,"Navy Blue, Bootcut, 34\"",49.99 +``` + +The backslash (`\`) is used directly before the second double quote (`"`) to indicate that it is _not_ the closing quote for the field, but rather a literal double quote character that should be included in the value (in this example, denoting the size of the jeans in inches: `34"` ). + +Leaving this field blank (default option) will disallow escaping. + +- **False Values**: A set of case-sensitive strings that should be interpreted as false values. +- **Null Values**: A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field. +- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Skip Rows After Header**: The number of rows to skip after the header row. +- **Skip Rows Before Header**: The number of rows to skip before the header row. +- **Strings Can Be Null**: Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself. +- **True Values**: A set of case-sensitive strings that should be interpreted as true values. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| -| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | -| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | -| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | -| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | -| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | -| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------| +| 0.3.7 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | +| 0.3.6 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | +| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | +| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | +| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 4160ad4722a0..03d20e32b24a 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -171,11 +171,25 @@ Expand to see details about GitHub connector limitations and troubleshooting. ### Connector limitations #### Rate limiting -The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +You can use a personal access token to make API requests. Additionally, you can authorize a GitHub App or OAuth app, which can then make API requests on your behalf. +All of these requests count towards your personal rate limit of 5,000 requests per hour (15,000 requests per hour if the app is owned by a GitHub Enterprise Cloud organization ). + +:::info `REST API` and `GraphQL API` rate limits are counted separately +::: + +:::tip +In the event that limits are reached before all streams have been read, it is recommended to take the following actions: +1. Utilize Incremental sync mode. +2. Set a higher sync interval. +3. Divide the sync into separate connections with a smaller number of streams. +::: + +Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). #### Permissions and scopes -If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **read:project**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: @@ -193,6 +207,13 @@ Your token should have at least the `repo` scope. Depending on which streams you | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.6.3 | 2024-02-15 | [35271](https://github.com/airbytehq/airbyte/pull/35271) | Update branches schema | +| 1.6.2 | 2024-02-12 | [34933](https://github.com/airbytehq/airbyte/pull/34933) | Update Airbyte CDK for integration tests | +| 1.6.1 | 2024-02-09 | [35087](https://github.com/airbytehq/airbyte/pull/35087) | Manage dependencies with Poetry. | +| 1.6.0 | 2024-02-02 | [34700](https://github.com/airbytehq/airbyte/pull/34700) | Continue Sync on Stream failure | +| 1.5.7 | 2024-01-29 | [34598](https://github.com/airbytehq/airbyte/pull/34598) | Fix MultipleToken sleep time | +| 1.5.6 | 2024-01-26 | [34503](https://github.com/airbytehq/airbyte/pull/34503) | Fix MultipleToken rotation logic | +| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | | 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | | 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | | 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response | @@ -301,4 +322,4 @@ Your token should have at least the `repo` scope. Depending on which streams you | 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | | 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | - \ No newline at end of file + diff --git a/docs/integrations/sources/gitlab-migrations.md b/docs/integrations/sources/gitlab-migrations.md index e1a597fb8a77..9e819b30f4d1 100644 --- a/docs/integrations/sources/gitlab-migrations.md +++ b/docs/integrations/sources/gitlab-migrations.md @@ -1,8 +1,63 @@ # Gitlab Migration Guide -## Upgrading to 2.0.0 +## Upgrading to 3.0.0 + +In this release, `merge_request_commits` stream schema has been fixed so that it returns commits for each merge_request. +Users will need to refresh the source schema and reset `merge_request_commits` stream after upgrading. + +## Connector Upgrade Guide + +### For Airbyte Open Source: Update the local connector image + +Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: + +1. Select **Settings** in the main navbar. + 1. Select **Sources**. +2. Find Gitlab in the list of connectors. + +:::note +You will see two versions listed, the current in-use version and the latest version available. +::: + +3. Select **Change** to update your OSS version to the latest available version. + +### Update the connector version +1. Select **Sources** in the main navbar. +2. Select the instance of the connector you wish to upgrade. + +:::note +Each instance of the connector must be updated separately. If you have created multiple instances of a connector, updating one will not affect the others. +::: + +3. Select **Upgrade** + 1. Follow the prompt to confirm you are ready to upgrade to the new version. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main nav bar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +:::note +Depending on destination type you may not be prompted to reset your data. +::: +4. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + +## Upgrading to 2.0.0 In the 2.0.0 config change, several streams were updated to date-time field format, as declared in the Gitlab API. These changes impact `pipeline.created_at` and` pipeline.updated_at` fields for stream Deployments and `expires_at` field for stream Group Members and stream Project Members. -You will need to refresh the source schema and reset affected streams after upgrading. \ No newline at end of file +You will need to refresh the source schema and reset affected streams after upgrading. diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index d4e26685717f..b4a83d6add6e 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -107,43 +107,47 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| -| 2.0.0 | 2023-10-23 | [31700](https://github.com/airbytehq/airbyte/pull/31700) | Add correct date-time format for Deployments, Projects and Groups Members streams | -| 1.8.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 1.8.3 | 2023-10-18 | [31547](https://github.com/airbytehq/airbyte/pull/31547) | Add validation for invalid `groups_list` and/or `projects_list` | -| 1.8.2 | 2023-10-17 | [31492](https://github.com/airbytehq/airbyte/pull/31492) | Expand list of possible error status codes when handling expired `access_token` | -| 1.8.1 | 2023-10-12 | [31375](https://github.com/airbytehq/airbyte/pull/31375) | Mark `start_date` as optional, migrate `groups` and `projects` to array | -| 1.8.0 | 2023-10-12 | [31339](https://github.com/airbytehq/airbyte/pull/31339) | Add undeclared fields to streams schemas, validate date/date-time format in stream schemas | -| 1.7.1 | 2023-10-10 | [31210](https://github.com/airbytehq/airbyte/pull/31210) | Added expired `access_token` handling, while checking the connection | -| 1.7.0 | 2023-08-08 | [27869](https://github.com/airbytehq/airbyte/pull/29203) | Add Deployments stream | -| 1.6.0 | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups | -| 1.5.1 | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting | -| 1.5.0 | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec. | -| 1.4.2 | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud. | -| 1.4.1 | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date. | -| 1.4.0 | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs. | -| 1.3.1 | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups | -| 1.3.0 | 2023-06-08 | [27150](https://github.com/airbytehq/airbyte/pull/27150) | Update stream schemas | -| 1.2.1 | 2023-06-02 | [26947](https://github.com/airbytehq/airbyte/pull/26947) | New field `name` added to `Pipelines` and `PipelinesExtended` stream schema | -| 1.2.0 | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys | -| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled | -| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema | -| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme | -| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification | -| 1.0.2 | 2023-01-27 | [22001](https://github.com/airbytehq/airbyte/pull/22001) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.1 | 2023-01-23 | [21713](https://github.com/airbytehq/airbyte/pull/21713) | Fix missing data issue | -| 1.0.0 | 2022-12-05 | [7506](https://github.com/airbytehq/airbyte/pull/7506) | Add `OAuth2.0` authentication option | -| 0.1.12 | 2022-12-15 | [20542](https://github.com/airbytehq/airbyte/pull/20542) | Revert HttpAvailability changes, run on cdk 0.15.0 | -| 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | -| 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | -| 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | -| 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream | -| 0.1.7 | 2022-12-01 | [19986](https://github.com/airbytehq/airbyte/pull/19986) | Fix `GroupMilestones` stream schema | -| 0.1.6 | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream | -| 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | -| 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | -| 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | -| 0.1.2 | 2021-10-18 | [7108](https://github.com/airbytehq/airbyte/pull/7108) | Allow all domains to be used as `api_url` | -| 0.1.1 | 2021-10-12 | [6932](https://github.com/airbytehq/airbyte/pull/6932) | Fix pattern field in spec file, remove unused fields from config files, use cache from CDK | -| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | Initial Release | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.0.0 | 2024-01-25 | [34548](https://github.com/airbytehq/airbyte/pull/34548) | Fix merge_request_commits stream to return commits for each merge request | +| 2.1.2 | 2024-02-12 | [35167](https://github.com/airbytehq/airbyte/pull/35167) | Manage dependencies with Poetry. | +| 2.1.1 | 2024-01-12 | [34203](https://github.com/airbytehq/airbyte/pull/34203) | prepare for airbyte-lib | +| 2.1.0 | 2023-12-20 | [33676](https://github.com/airbytehq/airbyte/pull/33676) | Add fields to Commits (extended_trailers), Groups (emails_enabled, service_access_tokens_expiration_enforced) and Projects (code_suggestions, model_registry_access_level) streams | +| 2.0.0 | 2023-10-23 | [31700](https://github.com/airbytehq/airbyte/pull/31700) | Add correct date-time format for Deployments, Projects and Groups Members streams | +| 1.8.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.8.3 | 2023-10-18 | [31547](https://github.com/airbytehq/airbyte/pull/31547) | Add validation for invalid `groups_list` and/or `projects_list` | +| 1.8.2 | 2023-10-17 | [31492](https://github.com/airbytehq/airbyte/pull/31492) | Expand list of possible error status codes when handling expired `access_token` | +| 1.8.1 | 2023-10-12 | [31375](https://github.com/airbytehq/airbyte/pull/31375) | Mark `start_date` as optional, migrate `groups` and `projects` to array | +| 1.8.0 | 2023-10-12 | [31339](https://github.com/airbytehq/airbyte/pull/31339) | Add undeclared fields to streams schemas, validate date/date-time format in stream schemas | +| 1.7.1 | 2023-10-10 | [31210](https://github.com/airbytehq/airbyte/pull/31210) | Added expired `access_token` handling, while checking the connection | +| 1.7.0 | 2023-08-08 | [27869](https://github.com/airbytehq/airbyte/pull/29203) | Add Deployments stream | +| 1.6.0 | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups | +| 1.5.1 | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting | +| 1.5.0 | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec. | +| 1.4.2 | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud. | +| 1.4.1 | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date. | +| 1.4.0 | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs. | +| 1.3.1 | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups | +| 1.3.0 | 2023-06-08 | [27150](https://github.com/airbytehq/airbyte/pull/27150) | Update stream schemas | +| 1.2.1 | 2023-06-02 | [26947](https://github.com/airbytehq/airbyte/pull/26947) | New field `name` added to `Pipelines` and `PipelinesExtended` stream schema | +| 1.2.0 | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys | +| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled | +| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema | +| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme | +| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification | +| 1.0.2 | 2023-01-27 | [22001](https://github.com/airbytehq/airbyte/pull/22001) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.1 | 2023-01-23 | [21713](https://github.com/airbytehq/airbyte/pull/21713) | Fix missing data issue | +| 1.0.0 | 2022-12-05 | [7506](https://github.com/airbytehq/airbyte/pull/7506) | Add `OAuth2.0` authentication option | +| 0.1.12 | 2022-12-15 | [20542](https://github.com/airbytehq/airbyte/pull/20542) | Revert HttpAvailability changes, run on cdk 0.15.0 | +| 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | +| 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | +| 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | +| 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream | +| 0.1.7 | 2022-12-01 | [19986](https://github.com/airbytehq/airbyte/pull/19986) | Fix `GroupMilestones` stream schema | +| 0.1.6 | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream | +| 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | +| 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | +| 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | +| 0.1.2 | 2021-10-18 | [7108](https://github.com/airbytehq/airbyte/pull/7108) | Allow all domains to be used as `api_url` | +| 0.1.1 | 2021-10-12 | [6932](https://github.com/airbytehq/airbyte/pull/6932) | Fix pattern field in spec file, remove unused fields from config files, use cache from CDK | +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | Initial Release | diff --git a/docs/integrations/sources/gong.md b/docs/integrations/sources/gong.md index c01ca9482657..04aeb7da4ff2 100644 --- a/docs/integrations/sources/gong.md +++ b/docs/integrations/sources/gong.md @@ -36,4 +36,5 @@ By default Gong limits your company's access to the service to 3 API calls per s | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------ | +| 0.1.1 | 2024-02-05 | [34847](https://github.com/airbytehq/airbyte/pull/34847) | Adjust stream schemas and make ready for airbyte-lib | | 0.1.0 | 2022-10-27 | [18819](https://github.com/airbytehq/airbyte/pull/18819) | Add Gong Source Connector | diff --git a/docs/integrations/sources/google-ads-migrations.md b/docs/integrations/sources/google-ads-migrations.md index 416233e7ba72..22dcc734b26c 100644 --- a/docs/integrations/sources/google-ads-migrations.md +++ b/docs/integrations/sources/google-ads-migrations.md @@ -1,5 +1,47 @@ # Google Ads Migration Guide +## Upgrading to 3.0.0 + +This release upgrades the Google Ads API from Version 13 to Version 15 which causes the following changes in the schemas: + +| Stream | Current field name | New field name | +|----------------------------|----------------------------------------------------------------------------|--------------------------------------------------------------------------| +| ad_listing_group_criterion | ad_group_criterion.listing_group.case_value.product_bidding_category.id | ad_group_criterion.listing_group.case_value.product_category.category_id | +| ad_listing_group_criterion | ad_group_criterion.listing_group.case_value.product_bidding_category.level | ad_group_criterion.listing_group.case_value.product_category.level | +| shopping_performance_view | segments.product_bidding_category_level1 | segments.product_category_level1 | +| shopping_performance_view | segments.product_bidding_category_level2 | segments.product_category_level2 | +| shopping_performance_view | segments.product_bidding_category_level3 | segments.product_category_level3 | +| shopping_performance_view | segments.product_bidding_category_level4 | segments.product_category_level4 | +| shopping_performance_view | segments.product_bidding_category_level5 | segments.product_category_level5 | +| campaign | campaign.shopping_setting.sales_country | This field has been deleted | + +Users should: +- Refresh the source schema +- Reset affected streams after upgrading to ensure uninterrupted syncs. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + ## Upgrading to 2.0.0 This release updates the Source Google Ads connector so that its default streams and stream names match the related resources in [Google Ads API](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad). @@ -10,4 +52,4 @@ Users should: ## Upgrading to 1.0.0 -This release introduced fixes to the creation of custom query schemas. For instance, the field ad_group_ad.ad.final_urls in the custom query has had its type changed from `{"type": "string"}` to `{"type": ["null", "array"], "items": {"type": "string"}}`. Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. \ No newline at end of file +This release introduced fixes to the creation of custom query schemas. For instance, the field ad_group_ad.ad.final_urls in the custom query has had its type changed from `{"type": "string"}` to `{"type": ["null", "array"], "items": {"type": "string"}}`. Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 240f88b90ea2..703474a83533 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -62,13 +62,14 @@ To set up Google Ads as a source in Airbyte Cloud: 3. Find and select **Google Ads** from the list of available sources. 4. Enter a **Source name** of your choosing. 5. Click **Sign in with Google** to authenticate your Google Ads account. In the pop-up, select the appropriate Google account and click **Continue** to proceed. -6. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -7. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) -8. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -9. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -10. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -11. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -12. Click **Set up source** and wait for the tests to complete. +6. (Optional) Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). Leaving this field blank will replicate data from all connected accounts. +7. (Optional) Enter customer statuses to filter customers. Leaving this field blank will replicate data from all accounts. Check [Google Ads documentation](https://developers.google.com/google-ads/api/reference/rpc/v15/CustomerStatusEnum.CustomerStatus) for more info. +8. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) +9. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). +10. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. +11. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. +12. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. +13. Click **Set up source** and wait for the tests to complete. @@ -83,13 +84,14 @@ To set up Google Ads as a source in Airbyte Open Source: 4. Enter a **Source name** of your choosing. 5. Enter the **Developer Token** you obtained from Google. 6. To authenticate your Google account, enter your Google application's **Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**. -7. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -8. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) -9. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -10. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -11. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, see the section on [Conversion Windows](#note-on-conversion-windows) below, or refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -12. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -13. Click **Set up source** and wait for the tests to complete. +7. (Optional) Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). Leaving this field blank will replicate data from all connected accounts. +8. (Optional) Enter customer statuses to filter customers. Leaving this field blank will replicate data from all accounts. Check [Google Ads documentation](https://developers.google.com/google-ads/api/reference/rpc/v15/CustomerStatusEnum.CustomerStatus) for more info. +9. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) +10. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). +11. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. +12. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, see the section on [Conversion Windows](#note-on-conversion-windows) below, or refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. +13. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. +14. Click **Set up source** and wait for the tests to complete. @@ -104,9 +106,9 @@ The Google Ads source connector supports the following [sync modes](https://docs #### Incremental Events Streams List of stream: -- [ad_group_criterions](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion) -- [ad_listing_group_criterions](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion) -- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v14/campaign_criterion) +- [ad_group_criterions](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) +- [ad_listing_group_criterions](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) +- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v15/campaign_criterion) These streams support incremental updates, including deletions, leveraging the Change Status stream. However, they only capture updates from the most recent three months. @@ -121,45 +123,45 @@ The Google Ads source connector can sync the following tables. It can also sync ### Main Tables -- [customer](https://developers.google.com/google-ads/api/fields/v14/customer) +- [customer](https://developers.google.com/google-ads/api/fields/v15/customer) Highlights the setup and configurations of a Google Ads account. It encompasses features like call reporting and conversion tracking, giving a clear picture of the account's operational settings and features. -- [customer_label](https://developers.google.com/google-ads/api/fields/v14/customer_label) -- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v14/campaign_criterion) +- [customer_label](https://developers.google.com/google-ads/api/fields/v15/customer_label) +- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v15/campaign_criterion) Targeting option for a campaign, such as a keyword, placement, or audience. -- [campaign_bidding_strategy](https://developers.google.com/google-ads/api/fields/v14/campaign) +- [campaign_bidding_strategy](https://developers.google.com/google-ads/api/fields/v15/campaign) Represents the bidding strategy at the campaign level. -- [campaign_label](https://developers.google.com/google-ads/api/fields/v14/campaign_label) -- [label](https://developers.google.com/google-ads/api/fields/v14/label) +- [campaign_label](https://developers.google.com/google-ads/api/fields/v15/campaign_label) +- [label](https://developers.google.com/google-ads/api/fields/v15/label) Represents labels that can be attached to different entities such as campaigns or ads. -- [ad_group_ad](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad) +- [ad_group_ad](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad) Different attributes of ads from ad groups segmented by date. -- [ad_group_ad_label](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad_label) -- [ad_group](https://developers.google.com/google-ads/api/fields/v14/ad_group) +- [ad_group_ad_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label) +- [ad_group](https://developers.google.com/google-ads/api/fields/v15/ad_group) Represents an ad group within a campaign. Ad groups contain one or more ads which target a shared set of keywords. -- [ad_group_label](https://developers.google.com/google-ads/api/fields/v14/ad_group_label) -- [ad_group_bidding_strategy](https://developers.google.com/google-ads/api/fields/v14/ad_group) +- [ad_group_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_label) +- [ad_group_bidding_strategy](https://developers.google.com/google-ads/api/fields/v15/ad_group) Represents the bidding strategy at the ad group level. -- [ad_group_criterion](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion) +- [ad_group_criterion](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) Represents criteria in an ad group, such as keywords or placements. -- [ad_listing_group_criterion](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion) +- [ad_listing_group_criterion](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) Represents criteria for listing group ads. -- [ad_group_criterion_label](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label) -- [audience](https://developers.google.com/google-ads/api/fields/v14/audience) +- [ad_group_criterion_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion_label) +- [audience](https://developers.google.com/google-ads/api/fields/v15/audience) Represents user lists that are defined by the advertiser to target specific users. -- [user_interest](https://developers.google.com/google-ads/api/fields/v14/user_interest) +- [user_interest](https://developers.google.com/google-ads/api/fields/v15/user_interest) A particular interest-based vertical to be targeted. -- [click_view](https://developers.google.com/google-ads/api/reference/rpc/v14/ClickView) +- [click_view](https://developers.google.com/google-ads/api/reference/rpc/v15/ClickView) A click view with metrics aggregated at each click level, including both valid and invalid clicks. @@ -170,36 +172,36 @@ Note that `ad_group`, `ad_group_ad`, and `campaign` contain a `labels` field, wh - [account_performance_report](https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance) Provides in-depth metrics related to ads interactions, including viewability, click-through rates, and conversions. Segments data by various factors, offering a granular look into how ads perform across different contexts. -- [campaign](https://developers.google.com/google-ads/api/fields/v14/campaign) +- [campaign](https://developers.google.com/google-ads/api/fields/v15/campaign) Represents a campaign in Google Ads. -- [campaign_budget](https://developers.google.com/google-ads/api/fields/v13/campaign_budget) +- [campaign_budget](https://developers.google.com/google-ads/api/fields/v15/campaign_budget) Represents the budget settings of a campaign. -- [geographic_view](https://developers.google.com/google-ads/api/fields/v14/geographic_view) +- [geographic_view](https://developers.google.com/google-ads/api/fields/v15/geographic_view) Geographic View includes all metrics aggregated at the country level. It reports metrics at either actual physical location of the user or an area of interest. -- [user_location_view](https://developers.google.com/google-ads/api/fields/v14/user_location_view) +- [user_location_view](https://developers.google.com/google-ads/api/fields/v15/user_location_view) User Location View includes all metrics aggregated at the country level. It reports metrics at the actual physical location of the user by targeted or not targeted location. -- [display_keyword_view](https://developers.google.com/google-ads/api/fields/v14/display_keyword_view) +- [display_keyword_view](https://developers.google.com/google-ads/api/fields/v15/display_keyword_view) Metrics for display keywords, which are keywords that are targeted in display campaigns. -- [topic_view](https://developers.google.com/google-ads/api/fields/v14/topic_view) +- [topic_view](https://developers.google.com/google-ads/api/fields/v15/topic_view) Reporting view that shows metrics aggregated by topic, which are broad categories of interests that users have. -- [shopping_performance_view](https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance) +- [shopping_performance_view](https://developers.google.com/google-ads/api/fields/v15/shopping_performance_view) Provides Shopping campaign statistics aggregated at several product dimension levels. Product dimension values from Merchant Center such as brand, category, custom attributes, product condition and product type will reflect the state of each dimension as of the date and time when the corresponding event was recorded. -- [keyword_view](https://developers.google.com/google-ads/api/fields/v14/keyword_view) +- [keyword_view](https://developers.google.com/google-ads/api/fields/v15/keyword_view) Provides metrics related to the performance of keywords in the campaign. -- [ad_group_ad_legacy](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad) +- [ad_group_ad_legacy](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad) Metrics and attributes of legacy ads from ad groups. :::note -Due to Google Ads API constraints, the `click_view` stream retrieves data one day at a time and can only retrieve data newer than 90 days ago. Also, [metrics](https://developers.google.com/google-ads/api/fields/v14/metrics) cannot be requested for a Google Ads Manager account. Report streams are only available when pulling data from a non-manager account. +Due to Google Ads API constraints, the `click_view` stream retrieves data one day at a time and can only retrieve data newer than 90 days ago. Also, [metrics](https://developers.google.com/google-ads/api/fields/v15/metrics) cannot be requested for a Google Ads Manager account. Report streams are only available when pulling data from a non-manager account. ::: :::warning @@ -208,7 +210,7 @@ If you have this type of campaign Google will remove them from the results for t More [info](https://github.com/airbytehq/airbyte/issues/11062) and [Google Discussions](https://groups.google.com/g/adwords-api/c/_mxbgNckaLQ). ::: -For incremental streams, data is synced up to the previous day using your Google Ads account time zone since Google Ads can filter data only by [date](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad#segments.date) without time. Also, some reports cannot load data real-time due to Google Ads [limitations](https://support.google.com/google-ads/answer/2544985?hl=en). +For incremental streams, data is synced up to the previous day using your Google Ads account time zone since Google Ads can filter data only by [date](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad#segments.date) without time. Also, some reports cannot load data real-time due to Google Ads [limitations](https://support.google.com/google-ads/answer/2544985?hl=en). ### Reasoning Behind Primary Key Selection @@ -218,7 +220,7 @@ Primary keys are chosen to uniquely identify records within streams. In this sel Additional streams for Google Ads can be dynamically created using custom queries. -The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v13/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source. +The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v15/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source. Example GAQL Custom Query: @@ -230,7 +232,7 @@ SELECT FROM ad_group ``` -Note the segments.date is automatically added to the output, and does not need to be specified in the custom query. All custom reports will by synced by day. +Note that `segments.date` is automatically added to the `WHERE` clause if it is included in the `SELECT` clause. Custom reports including `segments.date` in the `SELECT` clause will be synced by day. Each custom query in the input configuration must work for all the customer account IDs. Otherwise, the customer ID will be skipped for every query that fails the validation test. For example, if your query contains metrics fields in the select clause, it will not be executed against manager accounts. @@ -255,7 +257,7 @@ You can then monitor ad performance, update campaigns, and manage other account While both types of accounts can access a wide range of resources in the API, the difference lies in their scope and purpose. Manager accounts have a broader oversight, while client accounts delve into the specifics of advertising operations. -For detailed information, refer to the [official documentation.](https://developers.google.com/google-ads/api/fields/v14/overview) +For detailed information, refer to the [official documentation.](https://developers.google.com/google-ads/api/fields/v15/overview) ## Note on Conversion Windows @@ -278,7 +280,18 @@ Due to a limitation in the Google Ads API which does not allow getting performan | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| -| `2.0.4` | 2023-11-10 | [32414](https://github.com/airbytehq/airbyte/pull/32414) | Add backoff strategy for read_records method | +| `3.3.4` | 2024-02-21 | [35493](https://github.com/airbytehq/airbyte/pull/35493) | Rolling back the patch 3.3.3 made for `user_interest` steam | +| `3.3.3` | 2024-02-14 | [35280](https://github.com/airbytehq/airbyte/pull/35280) | Temporary patch that disables some fields to avoid 500 error when syncing `user_interest` steam | +| `3.3.2` | 2024-02-12 | [35158](https://github.com/airbytehq/airbyte/pull/35158) | Manage dependencies with Poetry. | +| `3.3.1` | 2024-01-16 | [34007](https://github.com/airbytehq/airbyte/pull/34007) | prepare for airbyte-lib | +| `3.3.0` | 2024-01-12 | [34212](https://github.com/airbytehq/airbyte/pull/34212) | Remove metric from query in Ad Group stream for non-manager account | +| `3.2.1` | 2024-01-12 | [34200](https://github.com/airbytehq/airbyte/pull/34200) | Disable raising error for not enabled accounts | +| `3.2.0` | 2024-01-09 | [33707](https://github.com/airbytehq/airbyte/pull/33707) | Add possibility to sync all connected accounts | +| `3.1.0` | 2024-01-09 | [33603](https://github.com/airbytehq/airbyte/pull/33603) | Fix two issues in the custom queries: automatic addition of `segments.date` in the query; incorrect field type for `DATE` fields. | +| `3.0.2` | 2024-01-08 | [33494](https://github.com/airbytehq/airbyte/pull/33494) | Add handling for 401 error while parsing response. Add `metrics.cost_micros` field to Ad Group stream. | +| `3.0.1` | 2023-12-26 | [33769](https://github.com/airbytehq/airbyte/pull/33769) | Run a read function in a separate thread to enforce a time limit for its execution | +| `3.0.0` | 2023-12-07 | [33120](https://github.com/airbytehq/airbyte/pull/33120) | Upgrade API version to v15 | +| `2.0.4` | 2023-11-10 | [32414](https://github.com/airbytehq/airbyte/pull/32414) | Add backoff strategy for read_records method | | `2.0.3` | 2023-11-02 | [32102](https://github.com/airbytehq/airbyte/pull/32102) | Fix incremental events streams | | `2.0.2` | 2023-10-31 | [32001](https://github.com/airbytehq/airbyte/pull/32001) | Added handling (retry) for `InternalServerError` while reading the streams | | `2.0.1` | 2023-10-27 | [31908](https://github.com/airbytehq/airbyte/pull/31908) | Base image migration: remove Dockerfile and use the python-connector-base image | diff --git a/docs/integrations/sources/google-analytics-data-api.md b/docs/integrations/sources/google-analytics-data-api.md index 03e569c0997f..c728f8304eb6 100644 --- a/docs/integrations/sources/google-analytics-data-api.md +++ b/docs/integrations/sources/google-analytics-data-api.md @@ -30,7 +30,7 @@ If the Property Settings shows a "Tracking Id" such as "UA-123...-1", this denot ::: 7. (Optional) In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports. -8. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. +8. (Optional) In the **Custom Reports** field, you may optionally describe any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. 9. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. :::caution @@ -91,8 +91,9 @@ If the start date is not provided, the default value will be used, which is two Many analyses and data investigations may require 24-48 hours to process information from your website or app. To ensure the accuracy of the data, we subtract two days from the starting date. For more details, please refer to [Google's documentation](https://support.google.com/analytics/answer/9333790?hl=en). ::: -7. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. -8. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. +7. (Optional) Toggle the switch **Keep Empty Rows** if you want each row with all metrics equal to 0 to be returned. +8. (Optional) In the **Custom Reports** field, you may optionally describe any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. +9. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. :::caution @@ -192,19 +193,6 @@ Custom reports in Google Analytics allow for flexibility in querying specific da A full list of dimensions and metrics supported in the API can be found [here](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema). To ensure your dimensions and metrics are compatible for your GA4 property, you can use the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). -Custom reports should be constructed as an array of JSON objects in the following format: - -```json -[ - { - "name": "", - "dimensions": ["", ...], - "metrics": ["", ...], - "cohortSpec": {/* cohortSpec object */}, - "pivots": [{/* pivot object */}, ...] - } -] -``` The following is an example of a basic User Engagement report to track sessions and bounce rate, segmented by city: @@ -266,7 +254,7 @@ The Google Analytics connector is subject to Google Analytics Data API quotas. P ## Data type map | Integration Type | Airbyte Type | -|:-----------------|:-------------| +| :--------------- | :----------- | | `string` | `string` | | `number` | `number` | | `array` | `array` | @@ -274,36 +262,45 @@ The Google Analytics connector is subject to Google Analytics Data API quotas. P ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------| -| 2.0.3 | 2023-11-03 | [32149](https://github.com/airbytehq/airbyte/pull/32149) | Fixed bug with missing `metadata` when the credentials are not valid | -| 2.0.2 | 2023-11-02 | [32094](https://github.com/airbytehq/airbyte/pull/32094) | Added handling for `JSONDecodeError` while checking for `api qouta` limits | -| 2.0.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 2.0.0 | 2023-09-29 | [30930](https://github.com/airbytehq/airbyte/pull/30930) | Use distinct stream naming in case there are multiple properties in the config. | -| 1.6.0 | 2023-09-19 | [30460](https://github.com/airbytehq/airbyte/pull/30460) | Migrated custom reports from string to array; add `FilterExpressions` support | -| 1.5.1 | 2023-09-20 | [30608](https://github.com/airbytehq/airbyte/pull/30608) | Revert `:` auto replacement name to underscore | -| 1.5.0 | 2023-09-18 | [30421](https://github.com/airbytehq/airbyte/pull/30421) | Add `yearWeek`, `yearMonth`, `year` dimensions cursor | -| 1.4.1 | 2023-09-17 | [30506](https://github.com/airbytehq/airbyte/pull/30506) | Fix None type error when metrics or dimensions response does not have name | -| 1.4.0 | 2023-09-15 | [30417](https://github.com/airbytehq/airbyte/pull/30417) | Change start date to optional; add suggested streams and update errors handling | -| 1.3.1 | 2023-09-14 | [30424](https://github.com/airbytehq/airbyte/pull/30424) | Fixed duplicated stream issue | -| 1.2.0 | 2023-09-11 | [30290](https://github.com/airbytehq/airbyte/pull/30290) | Add new preconfigured reports | -| 1.1.3 | 2023-08-04 | [29103](https://github.com/airbytehq/airbyte/pull/29103) | Update input field descriptions | -| 1.1.2 | 2023-07-03 | [27909](https://github.com/airbytehq/airbyte/pull/27909) | Limit the page size of custom report streams | -| 1.1.1 | 2023-06-26 | [27718](https://github.com/airbytehq/airbyte/pull/27718) | Limit the page size when calling `check()` | -| 1.1.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | -| 1.0.0 | 2023-06-22 | [26283](https://github.com/airbytehq/airbyte/pull/26283) | Added primary_key and lookback window | -| 0.2.7 | 2023-06-21 | [27531](https://github.com/airbytehq/airbyte/pull/27531) | Fix formatting | -| 0.2.6 | 2023-06-09 | [27207](https://github.com/airbytehq/airbyte/pull/27207) | Improve api rate limit messages | -| 0.2.5 | 2023-06-08 | [27175](https://github.com/airbytehq/airbyte/pull/27175) | Improve Error Messages | -| 0.2.4 | 2023-06-01 | [26887](https://github.com/airbytehq/airbyte/pull/26887) | Remove `authSpecification` from connector spec in favour of `advancedAuth` | -| 0.2.3 | 2023-05-16 | [26126](https://github.com/airbytehq/airbyte/pull/26126) | Fix pagination | -| 0.2.2 | 2023-05-12 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately | -| 0.2.1 | 2023-05-11 | [26008](https://github.com/airbytehq/airbyte/pull/26008) | Added handling for `429 - potentiallyThresholdedRequestsPerHour` error | -| 0.2.0 | 2023-04-13 | [25179](https://github.com/airbytehq/airbyte/pull/25179) | Implement support for custom Cohort and Pivot reports | -| 0.1.3 | 2023-03-10 | [23872](https://github.com/airbytehq/airbyte/pull/23872) | Fix parse + cursor for custom reports | -| 0.1.2 | 2023-03-07 | [23822](https://github.com/airbytehq/airbyte/pull/23822) | Improve `rate limits` customer faced error messages and retry logic for `429` | -| 0.1.1 | 2023-01-10 | [21169](https://github.com/airbytehq/airbyte/pull/21169) | Slicer updated, unit tests added | -| 0.1.0 | 2023-01-08 | [20889](https://github.com/airbytehq/airbyte/pull/20889) | Improved config validation, SAT | -| 0.0.3 | 2022-08-15 | [15229](https://github.com/airbytehq/airbyte/pull/15229) | Source Google Analytics Data Api: code refactoring | -| 0.0.2 | 2022-07-27 | [15087](https://github.com/airbytehq/airbyte/pull/15087) | fix documentationUrl | -| 0.0.1 | 2022-05-09 | [12701](https://github.com/airbytehq/airbyte/pull/12701) | Introduce Google Analytics Data API source | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------- | +| 2.4.1 | 2024-02-09 | [35073](https://github.com/airbytehq/airbyte/pull/35073) | Manage dependencies with Poetry. | +| 2.4.0 | 2024-02-07 | [34951](https://github.com/airbytehq/airbyte/pull/34951) | Replace the spec parameter from previous version to convert all `conversions:*` fields | +| 2.3.0 | 2024-02-06 | [34907](https://github.com/airbytehq/airbyte/pull/34907) | Add new parameter to spec to convert `conversions:purchase` field to float | +| 2.2.2 | 2024-02-01 | [34708](https://github.com/airbytehq/airbyte/pull/34708) | Add rounding integer values that may be float | +| 2.2.1 | 2024-01-18 | [34352](https://github.com/airbytehq/airbyte/pull/34352) | Add incorrect custom reports config handling | +| 2.2.0 | 2024-01-10 | [34176](https://github.com/airbytehq/airbyte/pull/34176) | Add a report option keepEmptyRows | +| 2.1.1 | 2024-01-08 | [34018](https://github.com/airbytehq/airbyte/pull/34018) | prepare for airbyte-lib | +| 2.1.0 | 2023-12-28 | [33802](https://github.com/airbytehq/airbyte/pull/33802) | Add `CohortSpec` to custom report in specification | +| 2.0.3 | 2023-11-03 | [32149](https://github.com/airbytehq/airbyte/pull/32149) | Fixed bug with missing `metadata` when the credentials are not valid | +| 2.0.2 | 2023-11-02 | [32094](https://github.com/airbytehq/airbyte/pull/32094) | Added handling for `JSONDecodeError` while checking for `api qouta` limits | +| 2.0.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.0 | 2023-09-29 | [30930](https://github.com/airbytehq/airbyte/pull/30930) | Use distinct stream naming in case there are multiple properties in the config. | +| 1.6.0 | 2023-09-19 | [30460](https://github.com/airbytehq/airbyte/pull/30460) | Migrated custom reports from string to array; add `FilterExpressions` support | +| 1.5.1 | 2023-09-20 | [30608](https://github.com/airbytehq/airbyte/pull/30608) | Revert `:` auto replacement name to underscore | +| 1.5.0 | 2023-09-18 | [30421](https://github.com/airbytehq/airbyte/pull/30421) | Add `yearWeek`, `yearMonth`, `year` dimensions cursor | +| 1.4.1 | 2023-09-17 | [30506](https://github.com/airbytehq/airbyte/pull/30506) | Fix None type error when metrics or dimensions response does not have name | +| 1.4.0 | 2023-09-15 | [30417](https://github.com/airbytehq/airbyte/pull/30417) | Change start date to optional; add suggested streams and update errors handling | +| 1.3.1 | 2023-09-14 | [30424](https://github.com/airbytehq/airbyte/pull/30424) | Fixed duplicated stream issue | +| 1.3.0 | 2023-09-13 | [30152](https://github.com/airbytehq/airbyte/pull/30152) | Ability to add multiple property ids | +| 1.2.0 | 2023-09-11 | [30290](https://github.com/airbytehq/airbyte/pull/30290) | Add new preconfigured reports | +| 1.1.3 | 2023-08-04 | [29103](https://github.com/airbytehq/airbyte/pull/29103) | Update input field descriptions | +| 1.1.2 | 2023-07-03 | [27909](https://github.com/airbytehq/airbyte/pull/27909) | Limit the page size of custom report streams | +| 1.1.1 | 2023-06-26 | [27718](https://github.com/airbytehq/airbyte/pull/27718) | Limit the page size when calling `check()` | +| 1.1.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | +| 1.0.0 | 2023-06-22 | [26283](https://github.com/airbytehq/airbyte/pull/26283) | Added primary_key and lookback window | +| 0.2.7 | 2023-06-21 | [27531](https://github.com/airbytehq/airbyte/pull/27531) | Fix formatting | +| 0.2.6 | 2023-06-09 | [27207](https://github.com/airbytehq/airbyte/pull/27207) | Improve api rate limit messages | +| 0.2.5 | 2023-06-08 | [27175](https://github.com/airbytehq/airbyte/pull/27175) | Improve Error Messages | +| 0.2.4 | 2023-06-01 | [26887](https://github.com/airbytehq/airbyte/pull/26887) | Remove `authSpecification` from connector spec in favour of `advancedAuth` | +| 0.2.3 | 2023-05-16 | [26126](https://github.com/airbytehq/airbyte/pull/26126) | Fix pagination | +| 0.2.2 | 2023-05-12 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately | +| 0.2.1 | 2023-05-11 | [26008](https://github.com/airbytehq/airbyte/pull/26008) | Added handling for `429 - potentiallyThresholdedRequestsPerHour` error | +| 0.2.0 | 2023-04-13 | [25179](https://github.com/airbytehq/airbyte/pull/25179) | Implement support for custom Cohort and Pivot reports | +| 0.1.3 | 2023-03-10 | [23872](https://github.com/airbytehq/airbyte/pull/23872) | Fix parse + cursor for custom reports | +| 0.1.2 | 2023-03-07 | [23822](https://github.com/airbytehq/airbyte/pull/23822) | Improve `rate limits` customer faced error messages and retry logic for `429` | +| 0.1.1 | 2023-01-10 | [21169](https://github.com/airbytehq/airbyte/pull/21169) | Slicer updated, unit tests added | +| 0.1.0 | 2023-01-08 | [20889](https://github.com/airbytehq/airbyte/pull/20889) | Improved config validation, SAT | +| 0.0.3 | 2022-08-15 | [15229](https://github.com/airbytehq/airbyte/pull/15229) | Source Google Analytics Data Api: code refactoring | +| 0.0.2 | 2022-07-27 | [15087](https://github.com/airbytehq/airbyte/pull/15087) | fix documentationUrl | +| 0.0.1 | 2022-05-09 | [12701](https://github.com/airbytehq/airbyte/pull/12701) | Introduce Google Analytics Data API source | diff --git a/docs/integrations/sources/google-analytics-v4-service-account-only.md b/docs/integrations/sources/google-analytics-v4-service-account-only.md new file mode 100644 index 000000000000..cb670a545a2b --- /dev/null +++ b/docs/integrations/sources/google-analytics-v4-service-account-only.md @@ -0,0 +1,286 @@ +# Google Analytics (Universal Analytics) + + + +This page contains the setup guide and reference information for the Google Analytics (Universal Analytics) source connector. + +This connector supports Universal Analytics properties through the [Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4). + + + +:::caution + +**The Google Analytics (Universal Analytics) connector will be deprecated soon.** + +Google is phasing out Universal Analytics in favor of Google Analytics 4 (GA4). In consequence, we are deprecating the Google Analytics (Universal Analytics) connector and recommend that you migrate to the [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) as soon as possible to ensure your syncs are not affected. + +Due to this deprecation, we will not be accepting new contributions for this source. + +For more information, see ["Universal Analytics is going away"](https://support.google.com/analytics/answer/11583528). + +::: + +:::note + +Google Analytics Universal Analytics (UA) connector, uses the older version of Google Analytics, which has been the standard for tracking website and app user behavior since 2012. + +[Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) is the latest version of Google Analytics, which was introduced in 2020. It offers a new data model that emphasizes events and user properties, rather than pageviews and sessions. This new model allows for more flexible and customizable reporting, as well as more accurate measurement of user behavior across devices and platforms. + +::: + +## Prerequisites + +A Google Cloud account with [Viewer permissions](https://support.google.com/analytics/answer/2884495) and [Google Analytics Reporting API](https://console.developers.google.com/apis/api/analyticsreporting.googleapis.com/overview) and [Google Analytics API](https://console.developers.google.com/apis/api/analytics.googleapis.com/overview) enabled. + +## Setup guide + + + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. +4. For Name, enter a name for the Google Analytics connector. +5. Authenticate your Google account via Service Account Key Authentication. + - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Make sure the Service Account has the Project Viewer permission. +6. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from. +8. Leave **Data request time increment in days (Optional)** blank or set to 1. For faster syncs, set this value to more than 1 but that might result in the Google Analytics API returning [sampled data](#sampled-data-in-reports), potentially causing inaccuracies in the returned results. The maximum allowed value is 364. + + + + +**For Airbyte Open Source:** + +1. Navigate to the Airbyte Open Source dashboard. +2. Go to the Airbyte UI and click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. +4. Enter a name for the Google Analytics connector. +5. Authenticate your Google account via Service Account Key Authentication: + - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API and grant [Read and Analyze permissions](https://support.google.com/analytics/answer/2884495). +5. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. + +6. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from. +7. Optionally, enter a JSON object as a string in the **Custom Reports** field. For details, refer to [Requesting custom reports](#requesting-custom-reports) +8. Leave **Data request time increment in days (Optional)** blank or set to 1. For faster syncs, set this value to more than 1 but that might result in the Google Analytics API returning [sampled data](#sampled-data-in-reports), potentially causing inaccuracies in the returned results. The maximum allowed value is 364. + + + +## Supported sync modes + +The Google Analytics source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) + +:::caution + +You need to add the service account email address on the account level, not the property level. Otherwise, an 403 error will be returned. + +::: + +## Supported streams + +The Google Analytics (Universal Analytics) source connector can sync the following tables: + +| Stream name | Schema | +|:-------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| website_overview | `{"ga_date":"2021-02-11","ga_users":1,"ga_newUsers":0,"ga_sessions":9,"ga_sessionsPerUser":9.0,"ga_avgSessionDuration":28.77777777777778,"ga_pageviews":63,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.685185185185185,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | +| traffic_sources | `{"ga_date":"2021-02-11","ga_source":"(direct)","ga_medium":"(none)","ga_socialNetwork":"(not set)","ga_users":1,"ga_newUsers":0,"ga_sessions":9,"ga_sessionsPerUser":9.0,"ga_avgSessionDuration":28.77777777777778,"ga_pageviews":63,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.685185185185185,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | +| pages | `{"ga_date":"2021-02-11","ga_hostname":"mydemo.com","ga_pagePath":"/home5","ga_pageviews":63,"ga_uniquePageviews":9,"ga_avgTimeOnPage":4.685185185185185,"ga_entrances":9,"ga_entranceRate":14.285714285714285,"ga_bounceRate":0.0,"ga_exits":9,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | +| locations | `{"ga_date":"2021-02-11","ga_continent":"Americas","ga_subContinent":"Northern America","ga_country":"United States","ga_region":"Iowa","ga_metro":"Des Moines-Ames IA","ga_city":"Des Moines","ga_users":1,"ga_newUsers":0,"ga_sessions":1,"ga_sessionsPerUser":1.0,"ga_avgSessionDuration":29.0,"ga_pageviews":7,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.666666666666667,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | +| monthly_active_users | `{"ga_date":"2021-02-11","ga_30dayUsers":1,"view_id":"211669975"}` | +| four_weekly_active_users | `{"ga_date":"2021-02-11","ga_28dayUsers":1,"view_id":"211669975"}` | +| two_weekly_active_users | `{"ga_date":"2021-02-11","ga_14dayUsers":1,"view_id":"211669975"}` | +| weekly_active_users | `{"ga_date":"2021-02-11","ga_7dayUsers":1,"view_id":"211669975"}` | +| daily_active_users | `{"ga_date":"2021-02-11","ga_1dayUsers":1,"view_id":"211669975"}` | +| devices | `{"ga_date":"2021-02-11","ga_deviceCategory":"desktop","ga_operatingSystem":"Macintosh","ga_browser":"Chrome","ga_users":1,"ga_newUsers":0,"ga_sessions":9,"ga_sessionsPerUser":9.0,"ga_avgSessionDuration":28.77777777777778,"ga_pageviews":63,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.685185185185185,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | +| Any custom reports | See [below](https://docs.airbyte.com/integrations/sources/google-analytics-v4#reading-custom-reports) for details. | + +Reach out to us on Slack or [create an issue](https://github.com/airbytehq/airbyte/issues) if you need to send custom Google Analytics report data with Airbyte. + +## Rate Limits and Performance Considerations \(Airbyte Open Source\) + +[Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas) + +- Number of requests per day per project: 50,000 +- Number of requests per view (profile) per day: 10,000 (cannot be increased) +- Number of requests per 100 seconds per project: 2,000 +- Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000). + +The Google Analytics connector should not run into the "requests per 100 seconds" limitation under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully and try increasing the `window_in_days` value. + +## Sampled data in reports + +If you are not on the Google Analytics 360 tier, the Google Analytics API may return sampled data if the amount of data in your Google Analytics account exceeds Google's [pre-determined compute thresholds](https://support.google.com/analytics/answer/2637192?hl=en&ref_topic=2601030&visit_id=637868645346124317-2833523666&rd=1#thresholds&zippy=%2Cin-this-article). This means the data returned in the report is an estimate which may have some inaccuracy. This [Google page](https://support.google.com/analytics/answer/2637192) provides a comprehensive overview of how Google applies sampling to your data. + +In order to minimize the chances of sampling being applied to your data, Airbyte makes data requests to Google in one day increments (the smallest allowed date increment). This reduces the amount of data the Google API processes per request, thus minimizing the chances of sampling being applied. The downside of requesting data in one day increments is that it increases the time it takes to export your Google Analytics data. If sampling is not a concern, you can override this behavior by setting the optional `window_in_day` parameter to specify the number of days to look back and avoid sampling. +When sampling occurs, a warning is logged to the sync log. + +## Requesting Custom Reports + +Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). + +A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]` + +Example of a custom report: +```json +[{ + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:sessionDefaultChannelGrouping" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] +}] +``` +Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. +Example of multiple custom reports: +```json +[ + { + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] + }, + { + "name" : "sessions_by_region", + "dimensions" :[ + "ga:date", + "ga:region" + ], + "metrics" :[ + "ga:totalUsers", + "ga:sessions" + ] + } +] +``` + +Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: +```json +[ + { + "name": "", + "dimensions": ["", ...], + "metrics": ["", ...], + "segments": ["", ...], + "filter": "" + } +] +``` + +* When using segments, make sure you also add the `ga:segment` dimension. + +Example of a custom report with segments and/or filters: +```json +[{ "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:segment" + ], + "metrics" :[ + "ga:sessions", + "ga:totalUsers" + ], + "segments" :[ + "ga:sessionSource!=(direct)" + ], + "filter" :[ + "ga:sessionSource!=(direct);ga:sessionSource!=(not set)" + ] +}] +``` + +To create a list of dimensions, you can use default Google Analytics dimensions (listed below) or custom dimensions if you have some defined. Each report can contain no more than 7 dimensions, and they must all be unique. The default Google Analytics dimensions are: + +- `ga:browser` +- `ga:city` +- `ga:continent` +- `ga:country` +- `ga:date` +- `ga:deviceCategory` +- `ga:hostname` +- `ga:medium` +- `ga:metro` +- `ga:operatingSystem` +- `ga:pagePath` +- `ga:region` +- `ga:socialNetwork` +- `ga:source` +- `ga:subContinent` + +To create a list of metrics, use a default Google Analytics metric (values from the list below) or custom metrics if you have defined them. +A custom report can contain no more than 10 unique metrics. The default available Google Analytics metrics are: + +- `ga:14dayUsers` +- `ga:1dayUsers` +- `ga:28dayUsers` +- `ga:30dayUsers` +- `ga:7dayUsers` +- `ga:avgSessionDuration` +- `ga:avgTimeOnPage` +- `ga:bounceRate` +- `ga:entranceRate` +- `ga:entrances` +- `ga:exitRate` +- `ga:exits` +- `ga:newUsers` +- `ga:pageviews` +- `ga:pageviewsPerSession` +- `ga:sessions` +- `ga:sessionsPerUser` +- `ga:uniquePageviews` +- `ga:users` + +Incremental sync is supported only if you add `ga:date` dimension to your custom report. + +## Limitations & Troubleshooting + +
      + +Expand to see details about Google Analytics v4 connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +[Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas) + +- Number of requests per day per project: 50,000 +- Number of requests per view (profile) per day: 10,000 (cannot be increased) +- Number of requests per 100 seconds per project: 2,000 +- Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000). + +The Google Analytics connector should not run into the "requests per 100 seconds" limitation under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully and try increasing the `window_in_days` value. + +### Troubleshooting + + + +* Check out common troubleshooting issues for the Google Analytics v4 source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:----------------------------------------------------------|:----------------| +| 0.0.1 | 2023-01-22 | [34323](https://github.com/airbytehq/airbyte/pull/34323) | Initial Release | + +       \ No newline at end of file diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 85538f77acef..e3b6d1720c9e 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -283,7 +283,10 @@ The Google Analytics connector should not run into the "requests per 100 seconds | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------- | -| 0.2.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.5 | 2024-02-09 | [35101](https://github.com/airbytehq/airbyte/pull/35101) | Manage dependencies with Poetry. | +| 0.2.4 | 2024-01-22 | [34323](https://github.com/airbytehq/airbyte/pull/34323) | Update setup dependencies | +| 0.2.3 | 2024-01-18 | [34353](https://github.com/airbytehq/airbyte/pull/34353) | Add End date option | +| 0.2.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.2.1 | 2023-07-11 | [28149](https://github.com/airbytehq/airbyte/pull/28149) | Specify date format to support datepicker in UI | | 0.2.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | | 0.1.36 | 2023-04-13 | [22223](https://github.com/airbytehq/airbyte/pull/22223) | Fix custom report with Segments dimensions | @@ -322,4 +325,4 @@ The Google Analytics connector should not run into the "requests per 100 seconds | 0.1.1 | 2021-08-25 | [5655](https://github.com/airbytehq/airbyte/pull/5655) | Corrected validation of empty custom report | | 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release | - \ No newline at end of file + diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index 6edf09e745c7..9ea6d11db1f4 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -237,7 +237,7 @@ There are currently no options for JSONL parsing. The Document file type format is currently an experimental feature and not subject to SLAs. Use at your own risk. ::: -The Document file type format is a special format that allows you to extract text from Markdown, PDF, Word, Powerpoint and Google documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`) or Docx (`docx`) file. +The Document file type format is a special format that allows you to extract text from Markdown, TXT, PDF, Word, Powerpoint and Google documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`) or Docx (`docx`) file. One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. @@ -247,6 +247,11 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|-----------------------------------------------------------|--------------------------------------------------------------| +| 0.0.9 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | +| 0.0.8 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.0.7 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 0.0.5 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | | 0.0.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.0.3 | 2023-11-16 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Improve folder id input and update document file type parser | | 0.0.2 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Allow syncs on shared drives | diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md index 12d4ba6128b1..6b2c2549e6c6 100644 --- a/docs/integrations/sources/google-search-console.md +++ b/docs/integrations/sources/google-search-console.md @@ -202,6 +202,7 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------| +| 1.3.7 | 2024-02-12 | [35163](https://github.com/airbytehq/airbyte/pull/35163) | Manage dependencies with Poetry. | | `1.3.6` | 2023-10-26 | [31863](https://github.com/airbytehq/airbyte/pull/31863) | Base image migration: remove Dockerfile and use the python-connector-base image | | `1.3.5` | 2023-09-28 | [30822](https://github.com/airbytehq/airbyte/pull/30822) | Fix primary key for custom reports | | `1.3.4` | 2023-09-27 | [30785](https://github.com/airbytehq/airbyte/pull/30785) | Do not migrate config for the newly created connections | @@ -239,4 +240,4 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | `0.1.1` | 2021-09-22 | [6315](https://github.com/airbytehq/airbyte/pull/6315) | Verify access to all sites when performing connection check | | `0.1.0` | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350) | Initial Release | - \ No newline at end of file + diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index 4fb8abb013c2..1d07a81ce708 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -118,7 +118,7 @@ The Google Sheets source connector supports the following sync modes: ## Data type map | Integration Type | Airbyte Type | Notes | -|:-----------------|:-------------|:------| +| :--------------- | :----------- | :---- | | any type | `string` | | ## Limitations & Troubleshooting @@ -150,8 +150,13 @@ Airbyte batches requests to the API in order to efficiently pull data and respec ## Changelog | Version | Date | Pull Request | Subject | -|---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| -| 0.3.11 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| ------- | ---------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | +| 0.3.16 | 2024-02-12 | [35136](https://github.com/airbytehq/airbyte/pull/35136) | Fix license in `pyproject.toml`. | +| 0.3.15 | 2024-02-07 | [34944](https://github.com/airbytehq/airbyte/pull/34944) | Manage dependencies with Poetry. | +| 0.3.14 | 2024-01-23 | [34437](https://github.com/airbytehq/airbyte/pull/34437) | Fix header cells filtering | +| 0.3.13 | 2024-01-19 | [34376](https://github.com/airbytehq/airbyte/pull/34376) | Fix names conversion | +| 0.3.12 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 0.3.11 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.10 | 2023-09-27 | [30487](https://github.com/airbytehq/airbyte/pull/30487) | Fix bug causing rows to be skipped when batch size increased due to rate limits. | | 0.3.9 | 2023-09-25 | [30749](https://github.com/airbytehq/airbyte/pull/30749) | Performance testing - include socat binary in docker image | | 0.3.8 | 2023-09-25 | [30747](https://github.com/airbytehq/airbyte/pull/30747) | Performance testing - include socat binary in docker image | @@ -198,4 +203,4 @@ Airbyte batches requests to the API in order to efficiently pull data and respec | 0.1.5 | 2020-12-30 | [1438](https://github.com/airbytehq/airbyte/pull/1438) | Implement backoff | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | - \ No newline at end of file + diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md index 02984acb4850..05b270d34d2e 100644 --- a/docs/integrations/sources/greenhouse.md +++ b/docs/integrations/sources/greenhouse.md @@ -62,20 +62,22 @@ The Greenhouse connector should not run into Greenhouse API limitations under no ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------- | -| 0.4.4 | 2023-11-29 | [32397](https://github.com/airbytehq/airbyte/pull/32397) | Increase test coverage and migrate to base image | -| 0.4.3 | 2023-09-20 | [30648](https://github.com/airbytehq/airbyte/pull/30648) | Update candidates.json | -| 0.4.2 | 2023-08-02 | [28969](https://github.com/airbytehq/airbyte/pull/28969) | Update CDK version | -| 0.4.1 | 2023-06-28 | [27773](https://github.com/airbytehq/airbyte/pull/27773) | Update following state breaking changes | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.5.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | +| 0.4.5 | 2024-02-09 | [35077](https://github.com/airbytehq/airbyte/pull/35077) | Manage dependencies with Poetry. | +| 0.4.4 | 2023-11-29 | [32397](https://github.com/airbytehq/airbyte/pull/32397) | Increase test coverage and migrate to base image | +| 0.4.3 | 2023-09-20 | [30648](https://github.com/airbytehq/airbyte/pull/30648) | Update candidates.json | +| 0.4.2 | 2023-08-02 | [28969](https://github.com/airbytehq/airbyte/pull/28969) | Update CDK version | +| 0.4.1 | 2023-06-28 | [27773](https://github.com/airbytehq/airbyte/pull/27773) | Update following state breaking changes | | 0.4.0 | 2023-04-26 | [25332](https://github.com/airbytehq/airbyte/pull/25332) | Add new streams: `ActivityFeed`, `Approvals`, `Disciplines`, `Eeoc`, `EmailTemplates`, `Offices`, `ProspectPools`, `Schools`, `Tags`, `UserPermissions`, `UserRoles` | -| 0.3.1 | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version | -| 0.3.0 | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema | -| 0.2.11 | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK | -| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs | -| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | -| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | -| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | -| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | -| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | -| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions | +| 0.3.1 | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version | +| 0.3.0 | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema | +| 0.2.11 | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK | +| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs | +| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | +| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | +| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | +| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | +| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | +| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions | diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index ac19b7dffdb7..0da8d6445662 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -79,26 +79,28 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------- | -| 0.1.21 | 2023-11-30 | [33003](https://github.com/airbytehq/airbyte/pull/33003) | Update expected records | -| 0.1.20 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.1.19 | 2023-07-26 | [28755](https://github.com/airbytehq/airbyte/pull/28755) | Changed parameters for Time Reports to use 365 days as opposed to 1 year | -| 0.1.18 | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth` | -| 0.1.17 | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification | -| 0.1.16 | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy | -| 0.1.15 | 2023-01-27 | [22008](https://github.com/airbytehq/airbyte/pull/22008) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | -| 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | -| 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | -| 0.1.11 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | -| 0.1.10 | 2022-08-08 | [15221](https://github.com/airbytehq/airbyte/pull/15221) | Added `parent_id` for all streams which have parent stream | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------| :-------------------------------------------------------------------------------- | +| 0.1.23 | 2024-02-19 | [35305](https://github.com/airbytehq/airbyte/pull/35305) | Fix pendulum parsing error | +| 0.1.22 | 2024-02-12 | [35154](https://github.com/airbytehq/airbyte/pull/35154) | Manage dependencies with Poetry. | +| 0.1.21 | 2023-11-30 | [33003](https://github.com/airbytehq/airbyte/pull/33003) | Update expected records | +| 0.1.20 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.1.19 | 2023-07-26 | [28755](https://github.com/airbytehq/airbyte/pull/28755) | Changed parameters for Time Reports to use 365 days as opposed to 1 year | +| 0.1.18 | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth` | +| 0.1.17 | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification | +| 0.1.16 | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy | +| 0.1.15 | 2023-01-27 | [22008](https://github.com/airbytehq/airbyte/pull/22008) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | +| 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | +| 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | +| 0.1.11 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | +| 0.1.10 | 2022-08-08 | [15221](https://github.com/airbytehq/airbyte/pull/15221) | Added `parent_id` for all streams which have parent stream | | 0.1.9 | 2022-08-04 | [15312](https://github.com/airbytehq/airbyte/pull/15312) | Fix `started_time` and `ended_time` format schema error and updated report slicing | -| 0.1.8 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | -| 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | -| 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | -| 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | -| 0.1.2 | 2021-06-07 | [4222](https://github.com/airbytehq/airbyte/pull/4222) | Correct specification parameter name | -| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.0 | 2021-06-07 | [3709](https://github.com/airbytehq/airbyte/pull/3709) | Release Harvest connector! | +| 0.1.8 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | +| 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | +| 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | +| 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | +| 0.1.2 | 2021-06-07 | [4222](https://github.com/airbytehq/airbyte/pull/4222) | Correct specification parameter name | +| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.0 | 2021-06-07 | [3709](https://github.com/airbytehq/airbyte/pull/3709) | Release Harvest connector! | diff --git a/docs/integrations/sources/hubspot-migrations.md b/docs/integrations/sources/hubspot-migrations.md index d76eb507dab4..32c8f6ff9997 100644 --- a/docs/integrations/sources/hubspot-migrations.md +++ b/docs/integrations/sources/hubspot-migrations.md @@ -1,28 +1,75 @@ # HubSpot Migration Guide +## Upgrading to 3.0.0 + +:::note +This change is only breaking if you are syncing the Marketing Emails stream. +::: + +This update brings extended schema with data type changes for the Marketing Emails stream. + +Users should: + - Refresh the source schema for the Marketing Emails stream. + - Reset the stream after upgrading to ensure uninterrupted syncs. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main nav bar. + 1. Select the connection affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. + +:::note +Any detected schema changes will be listed for your review. +::: + +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. + +:::note +Depending on destination type you may not be prompted to reset your data. +::: + +4. Select **Save connection**. + +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset) + + ## Upgrading to 2.0.0 -Note: this change is only breaking if you are using the PropertyHistory stream. +:::note +This change is only breaking if you are syncing the Property History stream. +::: + +With this update, you can now access historical property changes for Deals and Companies, in addition to Contacts. To facilitate this change, the Property History stream has been renamed to Contacts Property History (since it contained historical property changes from Contacts) and two new streams have been added: Deals Property History and Companies Property History. -With this update, you can now access historical property changes for Deals and Companies, in addition to Contacts. Property History stream has been renamed to Contacts Property History (since it historically contained historical property changes from Contacts) and two new streams were added: Deals Property History and Companies Property History. -This is a breaking change because Property History has been replaced with Contacts Property History, so please follow the instructions below to migrate to version 2.0.0: +This constitutes a breaking change as the Property History stream has been deprecated and replaced with the Contacts Property History. Please follow the instructions below to migrate to version 2.0.0: 1. Select **Connections** in the main navbar. -1.1 Select the connection(s) affected by the update. + 1. Select the connection(s) affected by the update. 2. Select the **Replication** tab. -2.1 Select **Refresh source schema**. - ```note - Any detected schema changes will be listed for your review. - ``` -2.2 Select **OK**. + 1. Select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** to proceed. +::: + 3. Select **Save changes** at the bottom of the page. -3.1 Ensure the **Reset affected streams** option is checked. - ```note - Depending on destination type you may not be prompted to reset your data - ``` + 1. Ensure the **Reset affected streams** option is checked. + +:::note +Depending on destination type you may not be prompted to reset your data +::: + 4. Select **Save connection**. - ```note - This will reset the data in your destination and initiate a fresh sync. - ``` + +:::note +This will reset the data in your destination and initiate a fresh sync. +::: For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 7b258c453131..9304a709827f 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -66,6 +66,7 @@ Next, you need to configure the appropriate scopes for the following streams. Pl | `contact_lists` | `crm.objects.lists.read` | | `contacts` | `crm.objects.contacts.read` | | `contacts_list_memberships` | `crm.objects.contacts.read` | +| `contacts_form_submissions` | `crm.objects.contacts.read` | | Custom CRM Objects | `crm.objects.custom.read` | | `deal_pipelines` | `crm.objects.contacts.read` | | `deals` | `crm.objects.deals.read`, `crm.schemas.deals.read` | @@ -166,6 +167,8 @@ The HubSpot source connector supports the following streams: - [Contact Lists](http://developers.hubspot.com/docs/methods/lists/get_lists) \(Incremental\) - [Contacts](https://developers.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\) - [Contacts List Memberships](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) +- [Contacts Form Submissions](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) +- [Contacts Merged Audit](https://legacydocs.hubspot.com/docs/methods/contacts/get_batch_by_vid) - [Deal Pipelines](https://developers.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type) \(Client-Side Incremental\) - [Deals](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\) - Records that have been deleted (archived) and stored in HubSpot's recycle bin will only be kept for 90 days, see [response from HubSpot Team](https://community.hubspot.com/t5/APIs-Integrations/Archived-deals-deleted-or-different/m-p/714157) @@ -184,6 +187,7 @@ The HubSpot source connector supports the following streams: - [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) \(Incremental\) - [Marketing Emails](https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-email-statistics) - [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental\) +- [Owners Archived](https://legacydocs.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental) - [Products](https://developers.hubspot.com/docs/api/crm/products) \(Incremental\) - [Contacts Property History](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) \(Client-Side Incremental\) - [Companies Property History](https://legacydocs.hubspot.com/docs/methods/companies/get-all-companies) \(Client-Side Incremental\) @@ -205,6 +209,11 @@ The HubSpot source connector supports the following streams: - [LineItemsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) - [ProductsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +### Notes on the `property_history` streams + +Even though the stream is Incremental, there are some record types that are not affected by the last sync timestamp pointer. For example records of type `CALCULATED` will allways have most recent timestamp equal to the requset time, so whenever you sync there will be a bunch of records in return. + + ### Notes on the `engagements` stream 1. Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. Depending on the type of engagement, different properties are set for that object in the `engagements_metadata` table in the destination: @@ -224,6 +233,10 @@ The HubSpot source connector supports the following streams: Because of this, the `engagements` stream can be slow to sync if it hasn't synced within the last 30 days and/or is generating large volumes of new data. We therefore recommend scheduling frequent syncs. +### Notes on the `Forms` and `Form Submissions` stream + +This stream sync only marketing forms. If you need other forms types try sync `Contacts Form Submissions`. + ## Limitations & Troubleshooting
      @@ -237,6 +250,13 @@ Expand to see details about Hubspot connector limitations and troubleshooting. The connector is restricted by normal HubSpot [rate limitations](https://legacydocs.hubspot.com/apps/api_guidelines). +| Product tier | Limits | +|:----------------------------|:-----------------------------------------| +| `Free & Starter` | Burst: 100/10 seconds, Daily: 250,000 | +| `Professional & Enterprise` | Burst: 150/10 seconds, Daily: 500,000 | +| `API add-on (any tier)` | Burst: 200/10 seconds, Daily: 1,000,000 | + + ### Troubleshooting * Consider checking out the following Hubspot tutorial: [Build a single customer view with open-source tools](https://airbyte.com/tutorials/single-customer-view). @@ -300,126 +320,133 @@ The connector is restricted by normal HubSpot [rate limitations](https://legacyd ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.0.0 | 2023-12-08 | [33266](https://github.com/airbytehq/airbyte/pull/33266) | Added ContactsPropertyHistory, CompaniesPropertyHistory, DealsPropertyHistory streams | -| 1.9.0 | 2023-12-04 | [33042](https://github.com/airbytehq/airbyte/pull/33042) | Added Web Analytics streams | -| 1.8.0 | 2023-11-23 | [32778](https://github.com/airbytehq/airbyte/pull/32778) | Extend `PropertyHistory` stream to support incremental sync | -| 1.7.0 | 2023-11-01 | [32035](https://github.com/airbytehq/airbyte/pull/32035) | Extend the `Forms` stream schema | -| 1.6.1 | 2023-10-20 | [31644](https://github.com/airbytehq/airbyte/pull/31644) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 1.6.0 | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Added new field `aifeatures` to the `marketing emails` stream schema | -| 1.5.1 | 2023-10-04 | [31050](https://github.com/airbytehq/airbyte/pull/31050) | Add type transformer for `Engagements` stream | -| 1.5.0 | 2023-09-11 | [30322](https://github.com/airbytehq/airbyte/pull/30322) | Unnest stream schemas | -| 1.4.1 | 2023-08-22 | [29715](https://github.com/airbytehq/airbyte/pull/29715) | Fix python package configuration stream | -| 1.4.0 | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream | -| 1.3.3 | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string | -| 1.3.2 | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` | -| 1.3.1 | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync | -| 1.3.0 | 2023-08-01 | [28909](https://github.com/airbytehq/airbyte/pull/28909) | Add handling of source connection errors | -| 1.2.0 | 2023-07-27 | [27091](https://github.com/airbytehq/airbyte/pull/27091) | Add new stream `ContactsMergedAudit` | -| 1.1.2 | 2023-07-27 | [28558](https://github.com/airbytehq/airbyte/pull/28558) | Improve error messages during connector setup | -| 1.1.1 | 2023-07-25 | [28705](https://github.com/airbytehq/airbyte/pull/28705) | Fix retry handler for token expired error | -| 1.1.0 | 2023-07-18 | [28349](https://github.com/airbytehq/airbyte/pull/28349) | Add unexpected fields in schemas of streams `email_events`, `email_subscriptions`, `engagements`, `campaigns` | -| 1.0.1 | 2023-06-23 | [27658](https://github.com/airbytehq/airbyte/pull/27658) | Use fully qualified name to retrieve custom objects | -| 1.0.0 | 2023-06-08 | [27161](https://github.com/airbytehq/airbyte/pull/27161) | Fixed increment sync for engagements stream, 'Recent' API is used for recent syncs of last recent 30 days and less than 10k records, otherwise full sync if performed by 'All' API | -| 0.9.0 | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | -| 0.8.4 | 2023-05-17 | [25667](https://github.com/airbytehq/airbyte/pull/26082) | Fixed bug with wrong parsing of boolean encoded like "false" parsed as True | -| 0.8.3 | 2023-05-31 | [26831](https://github.com/airbytehq/airbyte/pull/26831) | Remove authSpecification from connector specification in favour of advancedAuth | -| 0.8.2 | 2023-05-16 | [26418](https://github.com/airbytehq/airbyte/pull/26418) | Added custom availability strategy which catches permission errors from parent streams | -| 0.8.1 | 2023-05-29 | [26719](https://github.com/airbytehq/airbyte/pull/26719) | Handle issue when `state` value is literally `"" (empty str)` | -| 0.8.0 | 2023-04-10 | [16032](https://github.com/airbytehq/airbyte/pull/16032) | Add new stream `Custom Object` | -| 0.7.0 | 2023-04-10 | [24450](https://github.com/airbytehq/airbyte/pull/24450) | Add new stream `Goals` | -| 0.6.2 | 2023-04-28 | [25667](https://github.com/airbytehq/airbyte/pull/25667) | Fixed bug with `Invalid Date` like `2000-00-00T00:00:00Z` while settip up the connector | -| 0.6.1 | 2023-04-10 | [21423](https://github.com/airbytehq/airbyte/pull/21423) | Update scope for `DealPipelines` stream to only `crm.objects.contacts.read` | -| 0.6.0 | 2023-04-07 | [24980](https://github.com/airbytehq/airbyte/pull/24980) | Add new stream `DealsArchived` | -| 0.5.2 | 2023-04-07 | [24915](https://github.com/airbytehq/airbyte/pull/24915) | Fix field key parsing (replace whitespace with uderscore) | -| 0.5.1 | 2023-04-05 | [22982](https://github.com/airbytehq/airbyte/pull/22982) | Specified date formatting in specification | -| 0.5.0 | 2023-03-30 | [24711](https://github.com/airbytehq/airbyte/pull/24711) | Add incremental sync support for `campaigns`, `deal_pipelines`, `ticket_pipelines`, `forms`, `form_submissions`, `form_submissions`, `workflows`, `owners` | -| 0.4.0 | 2023-03-31 | [22910](https://github.com/airbytehq/airbyte/pull/22910) | Add `email_subscriptions` stream | -| 0.3.4 | 2023-03-28 | [24641](https://github.com/airbytehq/airbyte/pull/24641) | Convert to int only numeric values | -| 0.3.3 | 2023-03-27 | [24591](https://github.com/airbytehq/airbyte/pull/24591) | Fix pagination for `marketing emails` stream | -| 0.3.2 | 2023-02-07 | [22479](https://github.com/airbytehq/airbyte/pull/22479) | Turn on default HttpAvailabilityStrategy | -| 0.3.1 | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.3.0 | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available | -| 0.2.2 | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation | -| 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | -| 0.2.0 | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API. | -| 0.1.83 | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated. | -| 0.1.82 | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync | -| 0.1.81 | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema | -| 0.1.80 | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth | -| 0.1.79 | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues | -| 0.1.78 | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode | -| 0.1.77 | 2022-07-26 | [15035](https://github.com/airbytehq/airbyte/pull/15035) | Make PropertyHistory stream read historic data not limited to 30 days | -| 0.1.76 | 2022-07-25 | [14999](https://github.com/airbytehq/airbyte/pull/14999) | Partially revert changes made in v0.1.75 | -| 0.1.75 | 2022-07-18 | [14744](https://github.com/airbytehq/airbyte/pull/14744) | Remove override of private CDK method | -| 0.1.74 | 2022-07-25 | [14412](https://github.com/airbytehq/airbyte/pull/14412) | Add private app authentication | -| 0.1.73 | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream | -| 0.1.72 | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging | -| 0.1.71 | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code | -| 0.1.70 | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue | -| 0.1.69 | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue | -| 0.1.68 | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records | -| 0.1.67 | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user | -| 0.1.66 | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream | -| 0.1.65 | 2022-06-03 | [13455](https://github.com/airbytehq/airbyte/pull/13455) | Discover only returns streams for which required scopes were granted | -| 0.1.64 | 2022-06-03 | [13218](https://github.com/airbytehq/airbyte/pull/13218) | Transform `contact_lists` data to comply with schema | -| 0.1.63 | 2022-06-02 | [13320](https://github.com/airbytehq/airbyte/pull/13320) | Fix connector incremental state handling | -| 0.1.62 | 2022-06-01 | [13383](https://github.com/airbytehq/airbyte/pull/13383) | Add `line items` to `deals` stream | -| 0.1.61 | 2022-05-25 | [13381](https://github.com/airbytehq/airbyte/pull/13381) | Requests scopes as optional instead of required | -| 0.1.60 | 2022-05-25 | [13159](https://github.com/airbytehq/airbyte/pull/13159) | Use RFC3339 datetime | -| 0.1.59 | 2022-05-10 | [12711](https://github.com/airbytehq/airbyte/pull/12711) | Ensure oauth2.0 token has all needed scopes in "check" command | -| 0.1.58 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | -| 0.1.57 | 2022-05-04 | [12198](https://github.com/airbytehq/airbyte/pull/12198) | Add deals associations for quotes | -| 0.1.56 | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors | -| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | -| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | -| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Change spec json to yaml format | -| 0.1.52 | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams | -| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | -| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | -| 0.1.49 | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration | -| 0.1.48 | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs | -| 0.1.47 | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate | -| 0.1.46 | 2022-03-14 | [10700](https://github.com/airbytehq/airbyte/pull/10700) | Handle 10k+ records reading in Hubspot streams | -| 0.1.45 | 2022-03-04 | [10707](https://github.com/airbytehq/airbyte/pull/10707) | Remove stage history from deals stream to increase efficiency | -| 0.1.44 | 2022-02-24 | [9027](https://github.com/airbytehq/airbyte/pull/9027) | Add associations companies to deals, ticket and contact stream | -| 0.1.43 | 2022-02-24 | [10576](https://github.com/airbytehq/airbyte/pull/10576) | Cast timestamp to date/datetime | -| 0.1.42 | 2022-02-22 | [10492](https://github.com/airbytehq/airbyte/pull/10492) | Add `date-time` format to datetime fields | -| 0.1.41 | 2022-02-21 | [10177](https://github.com/airbytehq/airbyte/pull/10177) | Migrate to CDK | -| 0.1.40 | 2022-02-10 | [10142](https://github.com/airbytehq/airbyte/pull/10142) | Add associations to ticket stream | -| 0.1.39 | 2022-02-10 | [10055](https://github.com/airbytehq/airbyte/pull/10055) | Bug fix: reading not initialized stream | -| 0.1.38 | 2022-02-03 | [9786](https://github.com/airbytehq/airbyte/pull/9786) | Add new streams for engagements(calls, emails, meetings, notes and tasks) | -| 0.1.37 | 2022-01-27 | [9555](https://github.com/airbytehq/airbyte/pull/9555) | Getting form_submission for all forms | -| 0.1.36 | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784) | Add Property History Stream | -| 0.1.35 | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081) | Add Feedback Submissions stream and update Ticket Pipelines stream | -| 0.1.34 | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641) | Add more fields for `email_events` stream | -| 0.1.33 | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887) | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams | -| 0.1.32 | 2022-01-13 | [8011](https://github.com/airbytehq/airbyte/pull/8011) | Add new stream form_submissions | -| 0.1.31 | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream | -| 0.1.30 | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams | -| 0.1.29 | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams | -| 0.1.28 | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update fields and descriptions | -| 0.1.27 | 2021-12-09 | [8658](https://github.com/airbytehq/airbyte/pull/8658) | Fixed config backward compatibility issue by allowing additional properties in the spec | -| 0.1.26 | 2021-11-30 | [8329](https://github.com/airbytehq/airbyte/pull/8329) | Removed 'skip_dynamic_fields' config param | -| 0.1.25 | 2021-11-23 | [8216](https://github.com/airbytehq/airbyte/pull/8216) | Add skip dynamic fields for testing only | -| 0.1.24 | 2021-11-09 | [7683](https://github.com/airbytehq/airbyte/pull/7683) | Fix name issue 'Hubspot' -> 'HubSpot' | -| 0.1.23 | 2021-11-08 | [7730](https://github.com/airbytehq/airbyte/pull/7730) | Fix OAuth flow schema | -| 0.1.22 | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562) | Migrate Hubspot source to CDK structure | -| 0.1.21 | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405) | Change of package `import` from `urllib` to `urllib.parse` | -| 0.1.20 | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393) | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length | -| 0.1.19 | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams | -| 0.1.18 | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840) | Add new marketing emails (with statistics) stream | -| 0.1.17 | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995) | Update `discover` method: disable `quotes` stream when using OAuth config | -| 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator | -| 0.1.15 | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374) | Use correct schema for `owners` stream | -| 0.1.14 | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693) | Include deal_to_contact association when pulling deal stream and include contact ID in contact stream | -| 0.1.13 | 2021-09-08 | [5834](https://github.com/airbytehq/airbyte/pull/5834) | Fixed array fields without items property in schema | -| 0.1.12 | 2021-09-02 | [5798](https://github.com/airbytehq/airbyte/pull/5798) | Treat empty string values as None for field with format to fix normalization errors | -| 0.1.11 | 2021-08-26 | [5685](https://github.com/airbytehq/airbyte/pull/5685) | Remove all date-time format from schemas | -| 0.1.10 | 2021-08-17 | [5463](https://github.com/airbytehq/airbyte/pull/5463) | Fix fail on reading stream using `API Key` without required permissions | -| 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype | -| 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions | -| 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema | - - \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.2.0 | 2024-02-15 | [35328](https://github.com/airbytehq/airbyte/pull/35328) | Add mailingIlsListsIncluded and mailingIlsListsExcluded fields to Marketing emails stream schema | +| 3.1.1 | 2024-02-12 | [35165](https://github.com/airbytehq/airbyte/pull/35165) | Manage dependencies with Poetry. | +| 3.1.0 | 2024-02-05 | [34829](https://github.com/airbytehq/airbyte/pull/34829) | Add `Contacts Form Submissions` stream | +| 3.0.1 | 2024-01-29 | [34635](https://github.com/airbytehq/airbyte/pull/34635) | Fix pagination for `CompaniesPropertyHistory` stream | +| 3.0.0 | 2024-01-25 | [34492](https://github.com/airbytehq/airbyte/pull/34492) | Update `marketing_emails` stream schema | +| 2.0.2 | 2023-12-15 | [33844](https://github.com/airbytehq/airbyte/pull/33844) | Make property_history PK combined to support Incremental/Deduped sync type | +| 2.0.1 | 2023-12-15 | [33527](https://github.com/airbytehq/airbyte/pull/33527) | Make query string calculated correctly for ProertyHistory streams to avoid 414 HTTP Errors | +| 2.0.0 | 2023-12-08 | [33266](https://github.com/airbytehq/airbyte/pull/33266) | Add ContactsPropertyHistory, CompaniesPropertyHistory, DealsPropertyHistory streams | +| 1.9.0 | 2023-12-04 | [33042](https://github.com/airbytehq/airbyte/pull/33042) | Add Web Analytics streams | +| 1.8.0 | 2023-11-23 | [32778](https://github.com/airbytehq/airbyte/pull/32778) | Extend `PropertyHistory` stream to support incremental sync | +| 1.7.0 | 2023-11-01 | [32035](https://github.com/airbytehq/airbyte/pull/32035) | Extend the `Forms` stream schema | +| 1.6.1 | 2023-10-20 | [31644](https://github.com/airbytehq/airbyte/pull/31644) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.6.0 | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Add new field `aifeatures` to the `marketing emails` stream schema | +| 1.5.1 | 2023-10-04 | [31050](https://github.com/airbytehq/airbyte/pull/31050) | Add type transformer for `Engagements` stream | +| 1.5.0 | 2023-09-11 | [30322](https://github.com/airbytehq/airbyte/pull/30322) | Unnest stream schemas | +| 1.4.1 | 2023-08-22 | [29715](https://github.com/airbytehq/airbyte/pull/29715) | Fix python package configuration stream | +| 1.4.0 | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream | +| 1.3.3 | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string | +| 1.3.2 | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` | +| 1.3.1 | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync | +| 1.3.0 | 2023-08-01 | [28909](https://github.com/airbytehq/airbyte/pull/28909) | Add handling of source connection errors | +| 1.2.0 | 2023-07-27 | [27091](https://github.com/airbytehq/airbyte/pull/27091) | Add new stream `ContactsMergedAudit` | +| 1.1.2 | 2023-07-27 | [28558](https://github.com/airbytehq/airbyte/pull/28558) | Improve error messages during connector setup | +| 1.1.1 | 2023-07-25 | [28705](https://github.com/airbytehq/airbyte/pull/28705) | Fix retry handler for token expired error | +| 1.1.0 | 2023-07-18 | [28349](https://github.com/airbytehq/airbyte/pull/28349) | Add unexpected fields in schemas of streams `email_events`, `email_subscriptions`, `engagements`, `campaigns` | +| 1.0.1 | 2023-06-23 | [27658](https://github.com/airbytehq/airbyte/pull/27658) | Use fully qualified name to retrieve custom objects | +| 1.0.0 | 2023-06-08 | [27161](https://github.com/airbytehq/airbyte/pull/27161) | Fix increment sync for engagements stream, 'Recent' API is used for recent syncs of last recent 30 days and less than 10k records, otherwise full sync if performed by 'All' API | +| 0.9.0 | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | +| 0.8.4 | 2023-05-17 | [25667](https://github.com/airbytehq/airbyte/pull/26082) | Fixed bug with wrong parsing of boolean encoded like "false" parsed as True | +| 0.8.3 | 2023-05-31 | [26831](https://github.com/airbytehq/airbyte/pull/26831) | Remove authSpecification from connector specification in favour of advancedAuth | +| 0.8.2 | 2023-05-16 | [26418](https://github.com/airbytehq/airbyte/pull/26418) | Add custom availability strategy which catches permission errors from parent streams | +| 0.8.1 | 2023-05-29 | [26719](https://github.com/airbytehq/airbyte/pull/26719) | Handle issue when `state` value is literally `"" (empty str)` | +| 0.8.0 | 2023-04-10 | [16032](https://github.com/airbytehq/airbyte/pull/16032) | Add new stream `Custom Object` | +| 0.7.0 | 2023-04-10 | [24450](https://github.com/airbytehq/airbyte/pull/24450) | Add new stream `Goals` | +| 0.6.2 | 2023-04-28 | [25667](https://github.com/airbytehq/airbyte/pull/25667) | Fix bug with `Invalid Date` like `2000-00-00T00:00:00Z` while settip up the connector | +| 0.6.1 | 2023-04-10 | [21423](https://github.com/airbytehq/airbyte/pull/21423) | Update scope for `DealPipelines` stream to only `crm.objects.contacts.read` | +| 0.6.0 | 2023-04-07 | [24980](https://github.com/airbytehq/airbyte/pull/24980) | Add new stream `DealsArchived` | +| 0.5.2 | 2023-04-07 | [24915](https://github.com/airbytehq/airbyte/pull/24915) | Fix field key parsing (replace whitespace with uderscore) | +| 0.5.1 | 2023-04-05 | [22982](https://github.com/airbytehq/airbyte/pull/22982) | Specified date formatting in specification | +| 0.5.0 | 2023-03-30 | [24711](https://github.com/airbytehq/airbyte/pull/24711) | Add incremental sync support for `campaigns`, `deal_pipelines`, `ticket_pipelines`, `forms`, `form_submissions`, `form_submissions`, `workflows`, `owners` | +| 0.4.0 | 2023-03-31 | [22910](https://github.com/airbytehq/airbyte/pull/22910) | Add `email_subscriptions` stream | +| 0.3.4 | 2023-03-28 | [24641](https://github.com/airbytehq/airbyte/pull/24641) | Convert to int only numeric values | +| 0.3.3 | 2023-03-27 | [24591](https://github.com/airbytehq/airbyte/pull/24591) | Fix pagination for `marketing emails` stream | +| 0.3.2 | 2023-02-07 | [22479](https://github.com/airbytehq/airbyte/pull/22479) | Turn on default HttpAvailabilityStrategy | +| 0.3.1 | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.3.0 | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available | +| 0.2.2 | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation | +| 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | +| 0.2.0 | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API. | +| 0.1.83 | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated. | +| 0.1.82 | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync | +| 0.1.81 | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema | +| 0.1.80 | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth | +| 0.1.79 | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues | +| 0.1.78 | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode | +| 0.1.77 | 2022-07-26 | [15035](https://github.com/airbytehq/airbyte/pull/15035) | Make PropertyHistory stream read historic data not limited to 30 days | +| 0.1.76 | 2022-07-25 | [14999](https://github.com/airbytehq/airbyte/pull/14999) | Partially revert changes made in v0.1.75 | +| 0.1.75 | 2022-07-18 | [14744](https://github.com/airbytehq/airbyte/pull/14744) | Remove override of private CDK method | +| 0.1.74 | 2022-07-25 | [14412](https://github.com/airbytehq/airbyte/pull/14412) | Add private app authentication | +| 0.1.73 | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream | +| 0.1.72 | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging | +| 0.1.71 | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code | +| 0.1.70 | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue | +| 0.1.69 | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue | +| 0.1.68 | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records | +| 0.1.67 | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user | +| 0.1.66 | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream | +| 0.1.65 | 2022-06-03 | [13455](https://github.com/airbytehq/airbyte/pull/13455) | Discover only returns streams for which required scopes were granted | +| 0.1.64 | 2022-06-03 | [13218](https://github.com/airbytehq/airbyte/pull/13218) | Transform `contact_lists` data to comply with schema | +| 0.1.63 | 2022-06-02 | [13320](https://github.com/airbytehq/airbyte/pull/13320) | Fix connector incremental state handling | +| 0.1.62 | 2022-06-01 | [13383](https://github.com/airbytehq/airbyte/pull/13383) | Add `line items` to `deals` stream | +| 0.1.61 | 2022-05-25 | [13381](https://github.com/airbytehq/airbyte/pull/13381) | Requests scopes as optional instead of required | +| 0.1.60 | 2022-05-25 | [13159](https://github.com/airbytehq/airbyte/pull/13159) | Use RFC3339 datetime | +| 0.1.59 | 2022-05-10 | [12711](https://github.com/airbytehq/airbyte/pull/12711) | Ensure oauth2.0 token has all needed scopes in "check" command | +| 0.1.58 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | +| 0.1.57 | 2022-05-04 | [12198](https://github.com/airbytehq/airbyte/pull/12198) | Add deals associations for quotes | +| 0.1.56 | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors | +| 0.1.55 | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream | +| 0.1.54 | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s | +| 0.1.53 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Change spec json to yaml format | +| 0.1.52 | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams | +| 0.1.51 | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue | +| 0.1.50 | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination | +| 0.1.49 | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration | +| 0.1.48 | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs | +| 0.1.47 | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate | +| 0.1.46 | 2022-03-14 | [10700](https://github.com/airbytehq/airbyte/pull/10700) | Handle 10k+ records reading in Hubspot streams | +| 0.1.45 | 2022-03-04 | [10707](https://github.com/airbytehq/airbyte/pull/10707) | Remove stage history from deals stream to increase efficiency | +| 0.1.44 | 2022-02-24 | [9027](https://github.com/airbytehq/airbyte/pull/9027) | Add associations companies to deals, ticket and contact stream | +| 0.1.43 | 2022-02-24 | [10576](https://github.com/airbytehq/airbyte/pull/10576) | Cast timestamp to date/datetime | +| 0.1.42 | 2022-02-22 | [10492](https://github.com/airbytehq/airbyte/pull/10492) | Add `date-time` format to datetime fields | +| 0.1.41 | 2022-02-21 | [10177](https://github.com/airbytehq/airbyte/pull/10177) | Migrate to CDK | +| 0.1.40 | 2022-02-10 | [10142](https://github.com/airbytehq/airbyte/pull/10142) | Add associations to ticket stream | +| 0.1.39 | 2022-02-10 | [10055](https://github.com/airbytehq/airbyte/pull/10055) | Bug fix: reading not initialized stream | +| 0.1.38 | 2022-02-03 | [9786](https://github.com/airbytehq/airbyte/pull/9786) | Add new streams for engagements(calls, emails, meetings, notes and tasks) | +| 0.1.37 | 2022-01-27 | [9555](https://github.com/airbytehq/airbyte/pull/9555) | Getting form_submission for all forms | +| 0.1.36 | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784) | Add Property History Stream | +| 0.1.35 | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081) | Add Feedback Submissions stream and update Ticket Pipelines stream | +| 0.1.34 | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641) | Add more fields for `email_events` stream | +| 0.1.33 | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887) | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams | +| 0.1.32 | 2022-01-13 | [8011](https://github.com/airbytehq/airbyte/pull/8011) | Add new stream form_submissions | +| 0.1.31 | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream | +| 0.1.30 | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams | +| 0.1.29 | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams | +| 0.1.28 | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update fields and descriptions | +| 0.1.27 | 2021-12-09 | [8658](https://github.com/airbytehq/airbyte/pull/8658) | Fix config backward compatibility issue by allowing additional properties in the spec | +| 0.1.26 | 2021-11-30 | [8329](https://github.com/airbytehq/airbyte/pull/8329) | Remove 'skip_dynamic_fields' config param | +| 0.1.25 | 2021-11-23 | [8216](https://github.com/airbytehq/airbyte/pull/8216) | Add skip dynamic fields for testing only | +| 0.1.24 | 2021-11-09 | [7683](https://github.com/airbytehq/airbyte/pull/7683) | Fix name issue 'Hubspot' -> 'HubSpot' | +| 0.1.23 | 2021-11-08 | [7730](https://github.com/airbytehq/airbyte/pull/7730) | Fix OAuth flow schema | +| 0.1.22 | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562) | Migrate Hubspot source to CDK structure | +| 0.1.21 | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405) | Change of package `import` from `urllib` to `urllib.parse` | +| 0.1.20 | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393) | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length | +| 0.1.19 | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams | +| 0.1.18 | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840) | Add new marketing emails (with statistics) stream | +| 0.1.17 | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995) | Update `discover` method: disable `quotes` stream when using OAuth config | +| 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator | +| 0.1.15 | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374) | Use correct schema for `owners` stream | +| 0.1.14 | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693) | Include deal_to_contact association when pulling deal stream and include contact ID in contact stream | +| 0.1.13 | 2021-09-08 | [5834](https://github.com/airbytehq/airbyte/pull/5834) | Fix array fields without items property in schema | +| 0.1.12 | 2021-09-02 | [5798](https://github.com/airbytehq/airbyte/pull/5798) | Treat empty string values as None for field with format to fix normalization errors | +| 0.1.11 | 2021-08-26 | [5685](https://github.com/airbytehq/airbyte/pull/5685) | Remove all date-time format from schemas | +| 0.1.10 | 2021-08-17 | [5463](https://github.com/airbytehq/airbyte/pull/5463) | Fix fail on reading stream using `API Key` without required permissions | +| 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype | +| 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions | +| 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema | + + diff --git a/docs/integrations/sources/instagram-migrations.md b/docs/integrations/sources/instagram-migrations.md index f9009b09e3b5..49326bc1e4f8 100644 --- a/docs/integrations/sources/instagram-migrations.md +++ b/docs/integrations/sources/instagram-migrations.md @@ -1,5 +1,70 @@ # Instagram Migration Guide +## Upgrading to 3.0.0 + +The Instagram connector has been upgrade to API v18 (following the deprecation of v11). Connector will be upgraded to API v18. Affected Streams and their corresponding changes are listed below: + +- `Media Insights` + + Old metric will be replaced with the new ones, refer to the [IG Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights#metrics) for more info. + + | Old metric | New metric | + |----------------------------|--------------------| + | carousel_album_engagement | total_interactions | + | carousel_album_impressions | impressions | + | carousel_album_reach | reach | + | carousel_album_saved | saved | + | carousel_album_video_views | video_views | + | engagement | total_interactions | + +:::note + +You may see different results: `engagement` count includes likes, comments, and saves while `total_interactions` count includes likes, comments, and saves, as well as shares. + +::: + + New metrics for Reels: `ig_reels_avg_watch_time`, `ig_reels_video_view_total_time` + +- `User Lifetime Insights` + + - Metric `audience_locale` will become unavailable. + - Metrics `audience_city`, `audience_country`, and `audience_gender_age` will be consolidated into a single metric named `follower_demographics`, featuring respective breakdowns for `city`, `country`, and `age,gender`. + - Primary key will be changed to `["business_account_id", "breakdown"]`. + +:::note + +Due to Instagram limitations, the "Metric Type" will be set to `total_value` for `follower_demographics` metric. Refer to the [docs](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights#metric-type) for more info. + +::: + + +- `Story Insights` + + Metrics: `exits`, `taps_back`, `taps_forward` will become unavailable. + + +Please follow the instructions below to migrate to version 3.0.0: + +1. Select **Connections** in the main navbar. +1.1 Select the connection(s) affected by the update. +2. Select the **Replication** tab. +2.1 Select **Refresh source schema**. + ```note + Any detected schema changes will be listed for your review. + ``` +2.2 Select **OK**. +3. Select **Save changes** at the bottom of the page. +3.1 Ensure the **Reset affected streams** option is checked. + ```note + Depending on destination type you may not be prompted to reset your data + ``` +4. Select **Save connection**. + ```note + This will reset the data in your destination and initiate a fresh sync. + ``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + ## Upgrading to 2.0.0 This release adds a default primary key for the streams UserLifetimeInsights and UserInsights, and updates the format of timestamp fields in the UserLifetimeInsights, UserInsights, Media and Stories streams to include timezone information. diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 7b4999945fd4..2ba71bfbbd3c 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -2,7 +2,7 @@ -This page contains the setup guide and reference information for the Instagram source connector. +This page contains the setup guide and reference information for the [Instagram](https://www.instagram.com/) source connector. @@ -10,7 +10,8 @@ This page contains the setup guide and reference information for the Instagram s - [Meta for Developers account](https://developers.facebook.com) - [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page -- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte +- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte + - [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) to your Facebook app - [Facebook Instagram OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference) @@ -42,8 +43,8 @@ This page contains the setup guide and reference information for the Instagram s 3. On the Set up the source page, select **Instagram** from the **Source type** dropdown. 4. Enter a name for your source. 5. Enter **Access Token** generated using [Graph API Explorer](https://developers.facebook.com/tools/explorer/) or [by using an app you can create on Facebook](https://developers.facebook.com/docs/instagram-api/getting-started) with the required permissions: instagram_basic, instagram_manage_insights, pages_show_list, pages_read_engagement. -7. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. -8. Click **Set up source**. +6. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. +7. Click **Set up source**. @@ -74,25 +75,49 @@ The Instagram source connector supports the following streams. For more informat - [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/) - [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) -### Rate Limiting and Performance Considerations - -Instagram limits the number of requests that can be made at a time, but the Instagram connector gracefully handles rate limiting. See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. +:::info +The Instagram connector syncs data related to Users, Media, and Stories and their insights from the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/). For performance data related to Instagram Ads, use the Facebook Marketing source. +::: ## Data type map AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/) system. This means that all sources must produce schemas and records within these types and all destinations must handle records that conform to this type system. | Integration Type | Airbyte Type | -| :--------------- | :----------- | +|:-----------------|:-------------| | `string` | `string` | | `number` | `number` | | `array` | `array` | | `object` | `object` | +## Limitations & Troubleshooting + +
      + +Expand to see details about Instagram connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +Instagram limits the number of requests that can be made at a time. See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. + +### Troubleshooting + +* Check out common troubleshooting issues for the Instagram source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------| +| 3.0.3 | 2024-02-12 | [35177](https://github.com/airbytehq/airbyte/pull/35177) | Manage dependencies with Poetry. | +| 3.0.2 | 2024-01-15 | [34254](https://github.com/airbytehq/airbyte/pull/34254) | prepare for airbyte-lib | +| 3.0.1 | 2024-01-08 | [33989](https://github.com/airbytehq/airbyte/pull/33989) | Remove metrics from video feed | +| 3.0.0 | 2024-01-05 | [33930](https://github.com/airbytehq/airbyte/pull/33930) | Upgrade to API v18.0 | +| 2.0.1 | 2024-01-03 | [33889](https://github.com/airbytehq/airbyte/pull/33889) | Change requested metrics for stream `media_insights` | | 2.0.0 | 2023-11-17 | [32500](https://github.com/airbytehq/airbyte/pull/32500) | Add primary keys for UserLifetimeInsights and UserInsights; add airbyte_type to timestamp fields | | 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | | 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | @@ -113,9 +138,9 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | | 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | | 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | -       \ No newline at end of file + diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 8300aeca49d4..aebeb77e3101 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -74,6 +74,10 @@ The Intercom connector should not run into Intercom API limitations under normal | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| 0.6.0 | 2024-02-12 | [35176](https://github.com/airbytehq/airbyte/pull/35176) | Update the connector to use `2.10` API version | +| 0.5.1 | 2024-02-12 | [35148](https://github.com/airbytehq/airbyte/pull/35148) | Manage dependencies with Poetry. | +| 0.5.0 | 2024-02-09 | [35063](https://github.com/airbytehq/airbyte/pull/35063) | Add missing fields for mutiple streams | +| 0.4.0 | 2024-01-11 | [33882](https://github.com/airbytehq/airbyte/pull/33882) | Add new stream `Activity Logs` | | 0.3.2 | 2023-12-07 | [33223](https://github.com/airbytehq/airbyte/pull/33223) | Ignore 404 error for `Conversation Parts` | | 0.3.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.0 | 2023-05-25 | [29598](https://github.com/airbytehq/airbyte/pull/29598) | Update custom components to make them compatible with latest cdk version, simplify logic, update schemas | @@ -112,4 +116,4 @@ The Intercom connector should not run into Intercom API limitations under normal | 0.1.3 | 2021-09-08 | [5908](https://github.com/airbytehq/airbyte/pull/5908) | Corrected timestamp and arrays in schemas | | 0.1.2 | 2021-08-19 | [5531](https://github.com/airbytehq/airbyte/pull/5531) | Corrected pagination | | 0.1.1 | 2021-07-31 | [5123](https://github.com/airbytehq/airbyte/pull/5123) | Corrected rate limit | -| 0.1.0 | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676) | Release Intercom CDK Connector | \ No newline at end of file +| 0.1.0 | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676) | Release Intercom CDK Connector | diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index ec0bb73ea0fb..ce4b7d200967 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -78,29 +78,34 @@ The Iterable source connector supports the following [sync modes](https://docs.a ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------- | -| 0.1.30 | 2023-07-19 | [28457](https://github.com/airbytehq/airbyte/pull/28457) | Fixed TypeError for StreamSlice in debug mode | -| 0.1.29 | 2023-05-24 | [26459](https://github.com/airbytehq/airbyte/pull/26459) | Added requests reading timeout 300 seconds | -| 0.1.28 | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream | -| 0.1.27 | 2023-04-06 | [24962](https://github.com/airbytehq/airbyte/pull/24962) | `UserList` stream when meet `500 - Generic Error` will skip a broken slice and keep going with the next one | -| 0.1.26 | 2023-03-10 | [23938](https://github.com/airbytehq/airbyte/pull/23938) | Improve retry for `500 - Generic Error` | -| 0.1.25 | 2023-03-07 | [23821](https://github.com/airbytehq/airbyte/pull/23821) | Added retry for `500 - Generic Error`, increased max attempts number to `6` to handle `ChunkedEncodingError` | -| 0.1.24 | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification | -| 0.1.23 | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.22 | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak | -| 0.1.21 | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery | -| 0.1.20 | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors | -| 0.1.19 | 2022-10-05 | [17602](https://github.com/airbytehq/airbyte/pull/17602) | Add check for stream permissions | -| 0.1.18 | 2022-10-04 | [17573](https://github.com/airbytehq/airbyte/pull/17573) | Limit time range for SATs | -| 0.1.17 | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams | -| 0.1.16 | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header | -| 0.1.15 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | -| 0.1.14 | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380) | Update `Events` stream to use `export/userEvents` endpoint | -| 0.1.13 | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091) | Adjust slice ranges for email streams | -| 0.1.12 | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780) | Split EmailSend stream into slices to fix premature connection close error | -| 0.1.11 | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619) | Bugfix type error while incrementally loading the `Templates` stream | -| 0.1.10 | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591) | Optimize export streams memory consumption for large requests | -| 0.1.9 | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Enable campaign_metrics stream | -| 0.1.8 | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Add new streams: campaign_metrics, events | -| 0.1.7 | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242) | Updated schema for: campaigns, lists, templates, metadata | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | +| 0.2.2 | 2024-02-12 | [35150](https://github.com/airbytehq/airbyte/pull/35150) | Manage dependencies with Poetry. | +| 0.2.1 | 2024-01-12 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | +| 0.2.0 | 2023-09-29 | [28457](https://github.com/airbytehq/airbyte/pull/30931) | Added `userId` to `email_bounce`, `email_click`, `email_complaint`, `email_open`, `email_send` `email_send_skip`, `email_subscribe`, `email_unsubscribe`, `events` streams | +| 0.1.31 | 2023-12-06 | [33106](https://github.com/airbytehq/airbyte/pull/33106) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.1.30 | 2023-07-19 | [28457](https://github.com/airbytehq/airbyte/pull/28457) | Fixed TypeError for StreamSlice in debug mode | +| 0.1.29 | 2023-05-24 | [26459](https://github.com/airbytehq/airbyte/pull/26459) | Added requests reading timeout 300 seconds | +| 0.1.28 | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream | +| 0.1.27 | 2023-04-06 | [24962](https://github.com/airbytehq/airbyte/pull/24962) | `UserList` stream when meet `500 - Generic Error` will skip a broken slice and keep going with the next one | +| 0.1.26 | 2023-03-10 | [23938](https://github.com/airbytehq/airbyte/pull/23938) | Improve retry for `500 - Generic Error` | +| 0.1.25 | 2023-03-07 | [23821](https://github.com/airbytehq/airbyte/pull/23821) | Added retry for `500 - Generic Error`, increased max attempts number to `6` to handle `ChunkedEncodingError` | +| 0.1.24 | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification | +| 0.1.23 | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.22 | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak | +| 0.1.21 | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery | +| 0.1.20 | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors | +| 0.1.19 | 2022-10-05 | [17602](https://github.com/airbytehq/airbyte/pull/17602) | Add check for stream permissions | +| 0.1.18 | 2022-10-04 | [17573](https://github.com/airbytehq/airbyte/pull/17573) | Limit time range for SATs | +| 0.1.17 | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams | +| 0.1.16 | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header | +| 0.1.15 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.1.14 | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380) | Update `Events` stream to use `export/userEvents` endpoint | +| 0.1.13 | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091) | Adjust slice ranges for email streams | +| 0.1.12 | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780) | Split EmailSend stream into slices to fix premature connection close error | +| 0.1.11 | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619) | Bugfix type error while incrementally loading the `Templates` stream | +| 0.1.10 | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591) | Optimize export streams memory consumption for large requests | +| 0.1.9 | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Enable campaign_metrics stream | +| 0.1.8 | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Add new streams: campaign_metrics, events | +| 0.1.7 | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242) | Updated schema for: campaigns, lists, templates, metadata | diff --git a/docs/integrations/sources/jira-migrations.md b/docs/integrations/sources/jira-migrations.md new file mode 100644 index 000000000000..9dc0955b49d2 --- /dev/null +++ b/docs/integrations/sources/jira-migrations.md @@ -0,0 +1,27 @@ +# Jira Migration Guide + +## Upgrading to 1.0.0 + +Note: this change is only breaking if you are using the `Boards Issues` stream in Incremental Sync mode. + +This is a breaking change because Stream State for `Boards Issues` will be changed, so please follow the instructions below to migrate to version 1.0.0: + +1. Select **Connections** in the main navbar. +1.1 Select the connection(s) affected by the update. +2. Select the **Replication** tab. +2.1 Select **Refresh source schema**. + ```note + Any detected schema changes will be listed for your review. + ``` +2.2 Select **OK**. +3. Select **Save changes** at the bottom of the page. +3.1 Ensure the **Reset affected streams** option is checked. + ```note + Depending on destination type you may not be prompted to reset your data + ``` +4. Select **Save connection**. + ```note + This will reset the data in your destination and initiate a fresh sync. + ``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). \ No newline at end of file diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 968700e2215d..4eaf8278a348 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -124,7 +124,13 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| 0.12.0 | 2023-12-01 | [33011](https://github.com/airbytehq/airbyte/pull/33011) | Fix BoardIssues stream; increase number of retries for backoff policy to 10 | +| 1.0.2 | 2024-02-12 | [35160](https://github.com/airbytehq/airbyte/pull/35160) | Manage dependencies with Poetry. | +| 1.0.1 | 2024-01-24 | [34470](https://github.com/airbytehq/airbyte/pull/34470) | Add state checkpoint interval for all streams | +| 1.0.0 | 2024-01-01 | [33715](https://github.com/airbytehq/airbyte/pull/33715) | Save state for stream `Board Issues` per `board` | +| 0.14.1 | 2023-12-19 | [33625](https://github.com/airbytehq/airbyte/pull/33625) | Skip 404 error | +| 0.14.0 | 2023-12-15 | [33532](https://github.com/airbytehq/airbyte/pull/33532) | Add lookback window | +| 0.13.0 | 2023-12-12 | [33353](https://github.com/airbytehq/airbyte/pull/33353) | Fix check command to check access for all available streams | +| 0.12.0 | 2023-12-01 | [33011](https://github.com/airbytehq/airbyte/pull/33011) | Fix BoardIssues stream; increase number of retries for backoff policy to 10 | | 0.11.0 | 2023-11-29 | [32927](https://github.com/airbytehq/airbyte/pull/32927) | Fix incremental syncs for stream Issues | | 0.10.2 | 2023-10-26 | [31896](https://github.com/airbytehq/airbyte/pull/31896) | Provide better guidance when configuring the connector with an invalid domain | | 0.10.1 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | @@ -173,4 +179,4 @@ The Jira connector should not run into Jira API limitations under normal usage. | 0.2.6 | 2021-06-15 | [\#4113](https://github.com/airbytehq/airbyte/pull/4113) | Fixed `user` stream with the correct endpoint and query param. | | 0.2.5 | 2021-06-09 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` in base Docker image for Kubernetes support. | | 0.2.4 | | | Implementing base_read acceptance test dived by stream groups. | -| 0.2.3 | | | Implementing incremental sync. Migrated to airbyte-cdk. Adding all available entities in Jira Cloud. | \ No newline at end of file +| 0.2.3 | | | Implementing incremental sync. Migrated to airbyte-cdk. Adding all available entities in Jira Cloud. | diff --git a/docs/integrations/sources/kafka.md b/docs/integrations/sources/kafka.md index 4e963345dfdd..7eed0d3c74f2 100644 --- a/docs/integrations/sources/kafka.md +++ b/docs/integrations/sources/kafka.md @@ -50,6 +50,8 @@ The Kafka source connector supports the following [sync modes](https://docs.airb | Version | Date | Pull Request | Subject | | :------ | :-------- | :------------------------------------------------------| :---------------------------------------- | +| 0.2.4 | 2024-02-13 | [35229](https://github.com/airbytehq/airbyte/pull/35229) | Adopt CDK 0.20.4 | +| 0.2.4 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | | 0.2.3 | 2022-12-06 | [19587](https://github.com/airbytehq/airbyte/pull/19587) | Fix missing data before consumer is closed | | 0.2.2 | 2022-11-04 | [18648](https://github.com/airbytehq/airbyte/pull/18648) | Add missing record_count increment for JSON| | 0.2.1 | 2022-11-04 | This version was the same as 0.2.0 and was committed so using 0.2.2 next to keep versions in order| diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md index 56aefb605150..3f49f50eb0ca 100644 --- a/docs/integrations/sources/klaviyo.md +++ b/docs/integrations/sources/klaviyo.md @@ -62,7 +62,10 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u ## Changelog | Version | Date | Pull Request | Subject | -|:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------| +| :------- | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------- | +| `2.1.3` | 2024-02-15 | [35336](https://github.com/airbytehq/airbyte/pull/35336) | Added type transformer for the `profiles` stream. | +| `2.1.2` | 2024-02-09 | [35088](https://github.com/airbytehq/airbyte/pull/35088) | Manage dependencies with Poetry. | +| `2.1.1` | 2024-02-07 | [34998](https://github.com/airbytehq/airbyte/pull/34998) | Add missing fields to stream schemas | | `2.1.0` | 2023-12-07 | [33237](https://github.com/airbytehq/airbyte/pull/33237) | Continue syncing streams even when one of the stream fails | | `2.0.2` | 2023-12-05 | [33099](https://github.com/airbytehq/airbyte/pull/33099) | Fix filtering for archived records stream | | `2.0.1` | 2023-11-08 | [32291](https://github.com/airbytehq/airbyte/pull/32291) | Add logic to have regular checkpointing schedule | diff --git a/docs/integrations/sources/kustomer-singer.md b/docs/integrations/sources/kustomer-singer.md index 5fc12662e4e9..60cf45ce7f9a 100644 --- a/docs/integrations/sources/kustomer-singer.md +++ b/docs/integrations/sources/kustomer-singer.md @@ -1,5 +1,19 @@ # Kustomer +:::warning + +## Deprecation Notice + +The Kustomer source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + ## Sync overview The Kustomer source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. diff --git a/docs/integrations/sources/kyriba.md b/docs/integrations/sources/kyriba.md index f5c8f221a3dc..4792928ce5d9 100644 --- a/docs/integrations/sources/kyriba.md +++ b/docs/integrations/sources/kyriba.md @@ -1,24 +1,69 @@ # Kyriba + + +This page contains the setup guide and reference information for the [Kyriba](https://www.kyriba.com/) source connector. + + + ## Overview The Kyriba source retrieves data from [Kyriba](https://kyriba.com/) using their [JSON REST APIs](https://developer.kyriba.com/apiCatalog/). -## Setup Guide - -### Requirements +## Prerequisites - Kyriba domain -- Username +- Username - Password -You have to reach out to Kyriba to get these. +## Setup Guide + +### Set up the Kyriba source connector in Airbyte +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account or your Airbyte Open Source account. +2. Navigate to **Sources** in the left sidebar and click **+ New source**. in the top-right corner. +3. Choose **Kyriba** from the list of available sources. +4. For **Source name**, enter a descriptive name to help you identify this source. +5. For **Domain**, enter your Kyriba domain. +6. Input your **Username** and **Password** for basic authentication. +7. Specify the**Start Date**, from which data syncing will commence. +8. (Optional) Specify an End Date to indicate the last date up to which data will be synced. + + + +## Supported Sync Modes + +The Kyriba source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +- Full Refresh +- Incremental ## Supported Streams - [Accounts](https://developer.kyriba.com/site/global/apis/accounts/index.gsp) - [Bank Balances](https://developer.kyriba.com/site/global/apis/bank-statement-balances/index.gsp) - End of Day and Intraday -- [Cash Balances](https://developer.kyriba.com/site/global/apis/cash-balances/index.gsp) - End of Day and Intraday -- [Cash Flows](https://developer.kyriba.com/site/global/apis/cash-flows/index.gsp) (incremental) +- [Cash Balances](https://developer.kyriba.com/site/global/apis/cash-balances/index.gsp) - End of Day and Intraday +- [Cash Flows](https://developer.kyriba.com/site/global/apis/cash-flows/index.gsp) + +## Limitations & Troubleshooting + +
      + +Expand to see details about Kyriba connector limitations and troubleshooting. + + +### Connector Limitations + +#### Rate Limiting + +The Kyriba connector should not run into API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +### Troubleshooting + +* Check out common troubleshooting issues for the Stripe source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------- | +| 0.1.1 | 2024-01-30 | [34545](https://github.com/airbytehq/airbyte/pull/34545) | Updates CDK, Base image migration: remove Dockerfile and use the python-connector-base image | | 0.1.0 | 2022-07-13 | [12748](https://github.com/airbytehq/airbyte/pull/12748) | The Kyriba Source is created | + +       diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index d0d300521640..858c297f7958 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -158,7 +158,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s ## Data type map | Integration Type | Airbyte Type | Notes | -|:-----------------|:-------------|:----------------------------| +| :--------------- | :----------- | :-------------------------- | | `number` | `number` | float number | | `integer` | `integer` | whole number | | `date` | `string` | FORMAT YYYY-MM-DD | @@ -171,7 +171,12 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 0.6.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.7.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | +| 0.6.8 | 2024-02-09 | [35086](https://github.com/airbytehq/airbyte/pull/35086) | Manage dependencies with Poetry. | +| 0.6.7 | 2024-01-11 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | prepare for airbyte-lib | +| 0.6.6 | 2024-01-15 | [34222](https://github.com/airbytehq/airbyte/pull/34222) | Use stream slices for Analytics streams | +| 0.6.5 | 2023-12-15 | [33530](https://github.com/airbytehq/airbyte/pull/33530) | Fix typo in `Pivot Category` list | +| 0.6.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.6.3 | 2023-10-13 | [31396](https://github.com/airbytehq/airbyte/pull/31396) | Fix pagination for reporting | | 0.6.2 | 2023-08-23 | [31221](https://github.com/airbytehq/airbyte/pull/31221) | Increase max time between messages to 24 hours | | 0.6.1 | 2023-08-23 | [29600](https://github.com/airbytehq/airbyte/pull/29600) | Update field descriptions | @@ -198,4 +203,4 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | 0.1.3 | 2021-11-11 | [7839](https://github.com/airbytehq/airbyte/pull/7839) | Added OAuth support | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for `Campaigns/targetingCriteria` transformation, coerced `Creatives/variables/values` to string by default | -| 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte | \ No newline at end of file +| 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte | diff --git a/docs/integrations/sources/linnworks.md b/docs/integrations/sources/linnworks.md index ef1da0d98548..47f9aa6257b3 100644 --- a/docs/integrations/sources/linnworks.md +++ b/docs/integrations/sources/linnworks.md @@ -1,61 +1,79 @@ # Linnworks -## Sync overview +This page contains the setup guide and reference information for the [Linnworks](https://www.linnworks.com) source connector. -Linnworks source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +## Prerequisites -Airbyte uses [Linnworks API](https://apps.linnworks.net/Api) to fetch data from Linnworks. +- A Linnworks account -### Output schema +## Setup guide -This Source is capable of syncing the following data as streams: +### Generate Credentials in Linnworks -- [StockLocations](https://apps.linnworks.net/Api/Method/Inventory-GetStockLocations) -- [StockLocationDetails](https://apps.linnworks.net/Api/Method/Locations-GetLocation) -- [StockItems](https://apps.linnworks.net//Api/Method/Stock-GetStockItemsFull) -- [ProcessedOrders](https://apps.linnworks.net/Api/Method/ProcessedOrders-SearchProcessedOrders) -- [ProcessedOrderDetails](https://apps.linnworks.net/Api/Method/Orders-GetOrdersById) +1. The Linnworks platform has two portals: Seller and Developer. To generate the necessary credentials, log in to the [developer portal](https://developer.linnworks.com) and select **+ New App**. +2. Input a name for your application and set the **Application Type** to `System Integration`. +3. Select **Edit** for your new application. In the **General** tab, find and copy your **Application ID** and **Application Secret**. Click on the **Installation URL** to complete the installation of your app and acquire your **API Token**. + +:::tip +The value of your API Token can be viewed at any time from the main dashboard of your account, listed in your app's **Installs** table. +::: + +### Set up the connector in Airbyte + +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +2. From the Airbyte UI, click **Sources** > **+ New Source**. +3. Select **Linnworks** from the list of available sources. +4. Enter a **Name** of your choosing. +5. Enter your **Application ID**, **Application Secret** and **API Token**. +6. Enter a **Start date** using the provided datepicker. When using Incremental sync mode, only data generated after this date will be fetched. +7. Select **Set up source** and wait for the connection test to complete. + +## Supported streams and sync modes + +The Linnworks source connector supports the following streams and [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): + +| Stream Name | Full Refresh | Incremental | +| :--------------------------------------------------------------------------------------------- | :----------- | :----------- | +| [ProcessedOrders](https://apps.linnworks.net/Api/Method/ProcessedOrders-SearchProcessedOrders) | ✓ | ✓ | +| [ProcessedOrderDetails](https://apps.linnworks.net/Api/Method/Orders-GetOrdersById) | ✓ | ✓ | +| [StockItems](https://apps.linnworks.net//Api/Method/Stock-GetStockItemsFull) | ✓ | X | +| [StockLocations](https://apps.linnworks.net/Api/Method/Inventory-GetStockLocations) | ✓ | X | +| [StockLocationDetails](https://apps.linnworks.net/Api/Method/Locations-GetLocation) | ✓ | X | ### Data type mapping -| Integration Type | Airbyte Type | Notes | +| Integration Type | Airbyte Type | Example | | :--------------- | :----------- | :------------------------- | -| `number` | `number` | float number | -| `integer` | `integer` | whole number | -| `date` | `string` | FORMAT YYYY-MM-DD | -| `datetime` | `string` | FORMAT YYYY-MM-DDThh:mm:ss | -| `array` | `array` | | +| `number` | `number` | 50.23 | +| `integer` | `integer` | 50 | +| `date` | `string` | 2020-12-31 | +| `datetime` | `string` | 2020-12-31T07:30:00 | +| `array` | `array` | ["Item 1", "Item 2"] | | `boolean` | `boolean` | True/False | -| `string` | `string` | | - -### Features +| `string` | `string` | Item 3 | -| Feature | Supported?\(Yes/No\) | Notes | -| :---------------------------------------- | :------------------- | :---- | -| Full Refresh Overwrite Sync | Yes | | -| Full Refresh Append Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Append + Deduplication Sync | Yes | | -| Namespaces | No | | +## Limitations & Troubleshooting -### Performance considerations +
      + -Rate limit varies across Linnworks API endpoint. See the endpoint documentation to learn more. Rate limited requests will receive a 429 response. The Linnworks connector should not run into Linnworks API limitations under normal usage. +Expand to see details about Linnworks connector limitations and troubleshooting -## Getting started + -### Authentication +### Rate limits -Linnworks platform has two portals: seller and developer. First, to create API credentials, log in to the [developer portal](https://developer.linnworks.com) and create an application of type `System Integration`. Then click on provided Installation URL and proceed with an installation wizard. The wizard will show a token that you will need for authentication. The installed application will be present on your account on [seller portal](https://login.linnworks.net/). +Rate limits for the Linnworks API vary across endpoints. Use the [links in the **Supported Streams** table](#supported-streams-and-sync-modes) to view each endpoint's limits. Rate limited requests will receive a 429 response, but the Linnworks connector should not run into Linnworks API limitations under normal usage. -Authentication credentials can be obtained on developer portal section Applications -> _Your application name_ -> Edit -> General. And the token, if you missed it during the install, can be obtained anytime under the section Applications -> _Your application name_ -> Installs. +
      ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :-------------------------------------------------------------------------- | -| 0.1.5 | Unknown | Unknown | Bump Version | -| 0.1.4 | 2021-11-24 | [8226](https://github.com/airbytehq/airbyte/pull/8226) | Source Linnworks: improve streams ProcessedOrders and ProcessedOrderDetails | -| 0.1.3 | 2021-11-24 | [8169](https://github.com/airbytehq/airbyte/pull/8169) | Source Linnworks: refactor stream StockLocations | -| 0.1.2 | 2021-11-23 | [8177](https://github.com/airbytehq/airbyte/pull/8177) | Source Linnworks: add stream ProcessedOrderDetails | -| 0.1.0 | 2021-11-09 | [7588](https://github.com/airbytehq/airbyte/pull/7588) | New Source: Linnworks | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------- | +| 0.1.6 | 2024-01-31 | [34717](https://github.com/airbytehq/airbyte/pull/34717) | Update CDK and migrate to base image | +| 0.1.5 | 2022-11-20 | [19865](https://github.com/airbytehq/airbyte/pull/19865) | Bump Version | +| 0.1.4 | 2021-11-24 | [8226](https://github.com/airbytehq/airbyte/pull/8226) | Source Linnworks: improve streams ProcessedOrders and ProcessedOrderDetails | +| 0.1.3 | 2021-11-24 | [8169](https://github.com/airbytehq/airbyte/pull/8169) | Source Linnworks: refactor stream StockLocations | +| 0.1.2 | 2021-11-23 | [8177](https://github.com/airbytehq/airbyte/pull/8177) | Source Linnworks: add stream ProcessedOrderDetails | +| 0.1.0 | 2021-11-09 | [7588](https://github.com/airbytehq/airbyte/pull/7588) | New Source: Linnworks | diff --git a/docs/integrations/sources/mailchimp-migrations.md b/docs/integrations/sources/mailchimp-migrations.md new file mode 100644 index 000000000000..c236c549eef2 --- /dev/null +++ b/docs/integrations/sources/mailchimp-migrations.md @@ -0,0 +1,85 @@ +# Mailchimp Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 of the Source Mailchimp connector introduces a number of breaking changes to the schemas of all incremental streams. A full schema refresh and data reset are required when upgrading to this version. + +### Upgrade steps + +1. Select **Connections** in the main navbar. +2. From the list of your existing connections, select the connection(s) affected by the update. +3. Select the **Replication** tab, then select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** when you are ready to proceed. +::: + +4. At the bottom of the page, select **Save changes**. Ensure the **Reset all streams** option is checked. + +:::note +Depending on the destination type, you may not be prompted to reset your data +::: + +5. Select **Save connection**. This will reset the data in your destination (if applicable) and initiate a fresh sync. + +## Changes + +- The `._links` field, which contained non-user-relevant Mailchimp metadata, has been removed from all streams. +- All instances of datetime fields have had their type changed from `string` to airbyte-type `timestamp-with-timezone`. This change should ensure greater precision and consistency in how datetime information is represented and processed by destinations. +- The Mailchimp API returns many fields without data as empty strings. To accomodate the above changes, empty strings are now converted to null values: + +```md +{"id": "record_id", "last_opened": ""} -> {"id": "record_id", "last_opened": null} +``` + +### Updated datetime fields + +- Automations: + - `create_time` + - `send_time` + +- Campaigns: + - `create_time` + - `send_time` + - `rss_opts.last_sent` + - `ab_split_opts.send_time_a` + - `ab_split_opts.send_time_b` + - `variate_settings.send_times` (Array of datetime fields) + +- Email Activity: + - `timestamp` + +- List Members: + - `timestamp_signup` + - `timestamp_opt` + - `last_changed` + - `created_at` + +- Lists: + - `date_created` + - `stats.campaign_last_sent` + - `stats.last_sub_date` + - `stats.last_unsub_date` + +- Reports: + - `send_time` + - `rss_last_send` + - `opens.last_open` + - `clicks.last_click` + - `ab_split.a.last_open` + - `ab_split.b.last_open` + - `timewarp.last_open` + - `timeseries.timestamp` + +- Segment Members: + - `timestamp_signup` + - `timestamp_opt` + - `last_changed` + - `last_note.created_at` + +- Segments: + - `created_at` + - `updated_at` + +- Unsubscribes: + - `timestamp` diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index 4fa227cccc46..c6aed7b58e57 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -1,49 +1,82 @@ # Mailchimp -This page guides you through setting up the Mailchimp source connector. +This page guides you through setting up the [Mailchimp](https://mailchimp.com/) source connector. -## Prerequisite +## Prerequisites -You can use [OAuth](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/) or an API key to authenticate your Mailchimp account. If you choose to authenticate with OAuth, [register](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/#register-your-application) your Mailchimp account. + + +#### For Airbyte Cloud + +- Access to a valid Mailchimp account. If you are not an Owner/Admin of the account, you must be [granted Admin access](https://mailchimp.com/help/manage-user-levels-in-your-account/#Grant_account_access) by the account's Owner/Admin. + + + + + +#### For Airbyte Open Source + +- A valid Mailchimp **API Key** (recommended) or OAuth credentials: **Client ID**, **Client Secret** and **Access Token** + + + +## Setup guide + + + +### Airbyte Open Source: Generate a Mailchimp API key + +1. Navigate to the API Keys section of your Mailchimp account. +2. Click **Create New Key**, and give the key a name to help you identify it. You won't be able to see or copy the key once you finish generating it, so be sure to copy the key and store it in a secure location. + +For more information on Mailchimp API Keys, please refer to the [official Mailchimp docs](https://mailchimp.com/help/about-api-keys/#api+key+security). If you want to use OAuth authentication with Airbyte Open Source, please follow the steps laid out [here](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/) to obtain your OAuth **Client ID**, **Client Secret** and **Access Token**. + + ## Set up the Mailchimp source connector 1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. -2. Click **Sources** and then click **+ New source**. -3. On the Set up the source page, select **Mailchimp** from the Source type dropdown. +2. Click **Sources** and then click **+ New source**. +3. Find and select **Mailchimp** from the list of available sources. 4. Enter a name for your source. +5. You can use OAuth or an API key to authenticate your Mailchimp account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. + + + +- To authenticate using OAuth for Airbyte Cloud, click **Authenticate your Mailchimp account** and follow the instructions to sign in with Mailchimp and authorize your account. -6. You can use OAuth or an API key to authenticate your Mailchimp account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. - - To authenticate using OAuth for Airbyte Cloud, ensure you have [registered your Mailchimp account](#prerequisite) and then click **Authenticate your Mailchimp account** to sign in with Mailchimp and authorize your account. - - To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter the [API key](https://mailchimp.com/developer/marketing/guides/quick-start/#generate-your-api-key) for your Mailchimp account. - :::note - Check the [performance considerations](#performance-considerations) before using an API key. - ::: -7. Click **Set up source**. + -## Supported sync modes + -The Mailchimp source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): +- To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter the [API key](https://mailchimp.com/developer/marketing/guides/quick-start/#generate-your-api-key) for your Mailchimp account. +- To authenticate using OAuth credentials, select **Oauth2.0** from the dropdown and enter the **Client ID**, **Client Secret** and **Access Token** you obtained. - - Full Refresh - - Incremental + + +6. (Optional) You may optionally provide an **Incremental Sync Start Date** using the provided datepicker, or by programmatically entering a UTC date-time in the format `YYYY-MM-DDThh:mm:ss.sssZ`. If set, only data generated on or after the configured date-time will be synced. Leaving this field blank will sync all data returned from the API. +7. Click **Set up source** and wait for the tests to complete. + + ## Supported streams -The Mailchimp source connector supports the following streams: - -[Automations](https://mailchimp.com/developer/marketing/api/automation/list-automations/) -[Campaigns](https://mailchimp.com/developer/marketing/api/campaigns/get-campaign-info/) -[Email Activity](https://mailchimp.com/developer/marketing/api/email-activity-reports/list-email-activity/) -[Interests](https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/) -[Interest Categories](https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/) -[Lists](https://mailchimp.com/developer/api/marketing/lists/get-list-info) -[List Members](https://mailchimp.com/developer/marketing/api/list-members/list-members-info/) -[Reports](https://mailchimp.com/developer/marketing/api/reports/list-campaign-reports/) -[Segments](https://mailchimp.com/developer/marketing/api/list-segments/list-segments/) -[Segment Members](https://mailchimp.com/developer/marketing/api/list-segment-members/list-members-in-segment/) -[Tags](https://mailchimp.com/developer/marketing/api/lists-tags-search/search-for-tags-on-a-list-by-name/) -[Unsubscribes](https://mailchimp.com/developer/marketing/api/unsub-reports/list-unsubscribed-members/) +The Mailchimp source connector supports the following streams and [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): + +| Stream | Full Refresh | Incremental | +| :----------------------------------------------------------------------------------------------------------------- | :----------- | :---------- | +| [Automations](https://mailchimp.com/developer/marketing/api/automation/list-automations/) | ✓ | ✓ | +| [Campaigns](https://mailchimp.com/developer/marketing/api/campaigns/get-campaign-info/) | ✓ | ✓ | +| [Email Activity](https://mailchimp.com/developer/marketing/api/email-activity-reports/list-email-activity/) | ✓ | ✓ | +| [Interests](https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/) | ✓ | | +| [Interest Categories](https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/) | ✓ | | +| [Lists](https://mailchimp.com/developer/api/marketing/lists/get-list-info) | ✓ | ✓ | +| [List Members](https://mailchimp.com/developer/marketing/api/list-members/list-members-info/) | ✓ | ✓ | +| [Reports](https://mailchimp.com/developer/marketing/api/reports/list-campaign-reports/) | ✓ | ✓ | +| [Segments](https://mailchimp.com/developer/marketing/api/list-segments/list-segments/) | ✓ | ✓ | +| [Segment Members](https://mailchimp.com/developer/marketing/api/list-segment-members/list-members-in-segment/) | ✓ | ✓ | +| [Tags](https://mailchimp.com/developer/marketing/api/lists-tags-search/search-for-tags-on-a-list-by-name/) | ✓ | | +| [Unsubscribes](https://mailchimp.com/developer/marketing/api/unsub-reports/list-unsubscribed-members/) | ✓ | ✓ | ### A note on primary keys @@ -56,18 +89,30 @@ All other streams contain an `id` primary key. ## Data type mapping -| Integration Type | Airbyte Type | Notes | -|:---------------------------|:-------------|:------------------------------------------------------------------------------------| -| `array` | `array` | the type of elements in the array is determined based on the mappings in this table | -| `date`, `time`, `datetime` | `string` | | -| `int`, `float`, `number` | `number` | | -| `object` | `object` | properties within objects are mapped based on the mappings in this table | -| `string` | `string` | | +| Integration Type | Airbyte Type | Notes | +| :------------------- | :------------------------ | :---------------------------------------------------------------------------------- | +| `array` | `array` | the type of elements in the array is determined based on the mappings in this table | +| `string` | `string` | | +| `float`, `number` | `number` | | +| `integer` | `integer` | | +| `object` | `object` | properties within objects are mapped based on the mappings in this table | +| `string` (timestamp) | `timestamp_with_timezone` | Mailchimp timestamps are formatted as `YYYY-MM-DDTHH:MM:SS+00:00` | + +## Limitations & Troubleshooting -## Performance considerations +
      + + +Expand to see details about Mailchimp connector limitations and troubleshooting + + + +### Connector limitations [Mailchimp does not impose rate limits](https://mailchimp.com/developer/guides/marketing-api-conventions/#throttling) on how much data is read from its API in a single sync process. However, Mailchimp enforces a maximum of 10 simultaneous connections to its API, which means that Airbyte is unable to run more than 10 concurrent syncs from Mailchimp using API keys generated from the same account. +
      + ## Tutorials Now that you have set up the Mailchimp source connector, check out the following Mailchimp tutorial: @@ -77,7 +122,11 @@ Now that you have set up the Mailchimp source connector, check out the following ## Changelog | Version | Date | Pull Request | Subject | -|---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| +| ------- | ---------- | -------------------------------------------------------- | -------------------------------------------------------------------------- | +| 1.1.2 | 2024-02-09 | [35092](https://github.com/airbytehq/airbyte/pull/35092) | Manage dependencies with Poetry. | +| 1.1.1 | 2024-01-11 | [34157](https://github.com/airbytehq/airbyte/pull/34157) | Prepare for airbyte-lib | +| 1.1.0 | 2023-12-20 | [32852](https://github.com/airbytehq/airbyte/pull/32852) | Add optional start_date for incremental streams | +| 1.0.0 | 2023-12-19 | [32836](https://github.com/airbytehq/airbyte/pull/32836) | Add airbyte-type to `datetime` columns and remove `._links` column | | 0.10.0 | 2023-11-23 | [32782](https://github.com/airbytehq/airbyte/pull/32782) | Add SegmentMembers stream | | 0.9.0 | 2023-11-17 | [32218](https://github.com/airbytehq/airbyte/pull/32218) | Add Interests, InterestCategories, Tags streams | | 0.8.3 | 2023-11-15 | [32543](https://github.com/airbytehq/airbyte/pull/32543) | Handle empty datetime fields in Reports stream | @@ -112,3 +161,5 @@ Now that you have set up the Mailchimp source connector, check out the following | 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | | 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | + +       diff --git a/docs/integrations/sources/mailjet-mail.md b/docs/integrations/sources/mailjet-mail.md index 09709ad06495..85c89b0fda51 100644 --- a/docs/integrations/sources/mailjet-mail.md +++ b/docs/integrations/sources/mailjet-mail.md @@ -34,5 +34,6 @@ Mailjet APIs are under rate limits for the number of API calls allowed per API k | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.1 | 2022-04-19 | [#24689](https://github.com/airbytehq/airbyte/pull/24689) | Add listrecipient stream | +| 0.1.2 | 2022-12-18 | [#30924](https://github.com/airbytehq/airbyte/pull/30924) | Adds Subject field to `message` stream | +| 0.1.1 | 2022-04-19 | [#24689](https://github.com/airbytehq/airbyte/pull/24689) | Add listrecipient stream | | 0.1.0 | 2022-10-26 | [#18332](https://github.com/airbytehq/airbyte/pull/18332) | 🎉 New Source: Mailjet Mail API [low-code CDK] | diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index ee2d4782593f..b7a26c41b35d 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -106,7 +106,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Data type map | Integration Type | Airbyte Type | Notes | -|:-----------------|:-------------|:--------------------------------------------------------------------------------| +| :--------------- | :----------- | :------------------------------------------------------------------------------ | | `array` | `array` | primitive arrays are converted into arrays of the types described in this table | | `int`, `long` | `number` | | | `object` | `object` | | @@ -116,8 +116,12 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Changelog | Version | Date | Pull Request | Subject | -|:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------| -| 1.2.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------- | +| 1.2.6 | 2024-02-09 | [35078](https://github.com/airbytehq/airbyte/pull/35078) | Manage dependencies with Poetry. | +| 1.2.5 | 2024-01-15 | [34246](https://github.com/airbytehq/airbyte/pull/34246) | prepare for airbyte-lib | +| `1.2.4` | 2024-01-08 | [33999](https://github.com/airbytehq/airbyte/pull/33999) | Fix for `Export daily quota exceeded` | +| `1.2.3` | 2023-08-02 | [28999](https://github.com/airbytehq/airbyte/pull/28999) | Fix for ` _csv.Error: line contains NUL` | +| `1.2.2` | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | `1.2.1` | 2023-09-18 | [30533](https://github.com/airbytehq/airbyte/pull/30533) | Fix `json_schema` for stream `Leads` | | `1.2.0` | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | | `1.1.0` | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream | @@ -137,4 +141,4 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa | `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | | `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | | `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | -| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | \ No newline at end of file +| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | diff --git a/docs/integrations/sources/microsoft-onedrive.md b/docs/integrations/sources/microsoft-onedrive.md new file mode 100644 index 000000000000..c840f58b9517 --- /dev/null +++ b/docs/integrations/sources/microsoft-onedrive.md @@ -0,0 +1,130 @@ +# Microsoft OneDrive + +This page contains the setup guide and reference information for the Microsoft OneDrive source connector. + +### Requirements + +* Application \(client\) ID +* Directory \(tenant\) ID +* Drive name +* Folder Path +* Client secrets + +## Setup guide + + + +**For Airbyte Cloud:** + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Microsoft OneDrive** from the Source type dropdown. +4. Enter the name for the Microsoft OneDrive connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. The **OAuth2.0** authorization method is selected by default. Click **Authenticate your Microsoft OneDrive account**. Log in and authorize your Microsoft account. +8. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +9. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +10. Click **Set up source** + + + + +**For Airbyte Open Source:** + +### Step 1: Set up OneDrive application + +The Microsoft Graph API uses OAuth for authentication. Microsoft Graph exposes granular permissions that control the access that apps have to resources, like users, groups, and mail. When a user signs in to your app they, or, in some cases, an administrator, are given a chance to consent to these permissions. If the user consents, your app is given access to the resources and APIs that it has requested. For apps that don't take a signed-in user, permissions can be pre-consented to by an administrator when the app is installed. + +Microsoft Graph has two types of permissions: + +* **Delegated permissions** are used by apps that have a signed-in user present. For these apps, either the user or an administrator consents to the permissions that the app requests, and the app can act as the signed-in user when making calls to Microsoft Graph. Some delegated permissions can be consented by non-administrative users, but some higher-privileged permissions require administrator consent. +* **Application permissions** are used by apps that run without a signed-in user present; for example, apps that run as background services or daemons. Application permissions can only be consented by an administrator. + +This source requires **Application permissions**. Follow these [instructions](https://docs.microsoft.com/en-us/graph/auth-v2-service?context=graph%2Fapi%2F1.0&view=graph-rest-1.0) for creating an app in the Azure portal. This process will produce the `client_id`, `client_secret`, and `tenant_id` needed for the tap configuration file. + +1. Login to [Azure Portal](https://portal.azure.com/#home) +2. Click upper-left menu icon and select **Azure Active Directory** +3. Select **App Registrations** +4. Click **New registration** +5. Register an application + 1. Name: + 2. Supported account types: Accounts in this organizational directory only + 3. Register \(button\) +6. Record the client\_id and tenant\_id which will be used by the tap for authentication and API integration. +7. Select **Certificates & secrets** +8. Provide **Description and Expires** + 1. Description: tap-microsoft-onedrive client secret + 2. Expires: 1-year + 3. Add +9. Copy the client secret value, this will be the client\_secret +10. Select **API permissions** + 1. Click **Add a permission** +11. Select **Microsoft Graph** +12. Select **Application permissions** +13. Select the following permissions: + 1. Files + * Files.Read.All +14. Click **Add permissions** +15. Click **Grant admin consent** + +### Step 2: Set up the Microsoft OneDrive connector in Airbyte + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the **Set up** the source page, select **Microsoft OneDrive** from the Source type dropdown. +4. Enter the name for the Microsoft OneDrive connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. Switch to **Service Key Authentication** +8. For **User Practical Name**, enter the [UPN](https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls) for your user. +9. Enter **Tenant ID**, **Client ID** and **Client secret**. +10. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +11. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +12. Click **Set up source** + + + +## Sync overview + +### Data type mapping + +| Integration Type | Airbyte Type | +|:-----------------|:-------------| +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | + +### Features + +| Feature | Supported?\(Yes/No\) | +|:------------------------------|:---------------------| +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | + +### Performance considerations + +The connector is restricted by normal Microsoft Graph [requests limitation](https://docs.microsoft.com/en-us/graph/throttling). + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------| +| 0.1.6 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | +| 0.1.5 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.1.4 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.1.3 | 2024-01-24 | [34478](https://github.com/airbytehq/airbyte/pull/34478) | Fix OAuth | +| 0.1.2 | 2021-12-22 | [33745](https://github.com/airbytehq/airbyte/pull/33745) | Add ql and sl to metadata | +| 0.1.1 | 2021-12-15 | [33758](https://github.com/airbytehq/airbyte/pull/33758) | Fix for docs name | +| 0.1.0 | 2021-12-06 | [32655](https://github.com/airbytehq/airbyte/pull/32655) | New source | diff --git a/docs/integrations/sources/microsoft-sharepoint.md b/docs/integrations/sources/microsoft-sharepoint.md new file mode 100644 index 000000000000..db706d733355 --- /dev/null +++ b/docs/integrations/sources/microsoft-sharepoint.md @@ -0,0 +1,134 @@ +# Microsoft SharePoint + + +This page contains the setup guide and reference information for the Microsoft SharePoint source connector. + + +### Requirements + +* Application \(client\) ID +* Directory \(tenant\) ID +* Drive name +* Folder Path +* Client secrets + +## Setup guide + + + + +**For Airbyte Cloud:** + + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Microsoft SharePoint** from the Source type dropdown. +4. Enter the name for the Microsoft SharePoint connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. The **OAuth2.0** authorization method is selected by default. Click **Authenticate your Microsoft SharePoint account**. Log in and authorize your Microsoft account. +8. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +9. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +10. Click **Set up source** + + + + + +**For Airbyte Open Source:** + + +### Step 1: Set up SharePoint application + +The Microsoft Graph API uses OAuth for authentication. Microsoft Graph exposes granular permissions that control the access that apps have to resources, like users, groups, and mail. When a user signs in to your app, they or in some cases an administrator are given a chance to consent to these permissions. If the user consents, your app is given access to the resources and APIs that it has requested. For apps that don't take a signed-in user, permissions can be pre-consented to by an administrator when the app is installed. + +Microsoft Graph has two types of permissions: + +* **Delegated permissions** are used by apps that have a signed-in user present. For these apps, either the user or an administrator consents to the permissions that the app requests, and the app can act as the signed-in user when making calls to Microsoft Graph. Some delegated permissions can be consented by non-administrative users, but some higher-privileged permissions require administrator consent. +* **Application permissions** are used by apps that run without a signed-in user present; for example, apps that run as background services or daemons. Application permissions can only be consented by an administrator. + +This source requires **Application permissions**. Follow these [instructions](https://docs.microsoft.com/en-us/graph/auth-v2-service?context=graph%2Fapi%2F1.0&view=graph-rest-1.0) for creating an app in the Azure portal. This process will produce the `client_id`, `client_secret`, and `tenant_id` needed for the tap configuration file. + +1. Login to [Azure Portal](https://portal.azure.com/#home) +2. Click upper-left menu icon and select **Azure Active Directory** +3. Select **App Registrations** +4. Click **New registration** +5. Register an application + 1. Name: + 2. Supported account types: Accounts in this organizational directory only + 3. Register \(button\) +6. Record the client\_id and tenant\_id which will be used by the tap for authentication and API integration. +7. Select **Certificates & secrets** +8. Provide **Description and Expires** + 1. Description: tap-microsoft-teams client secret + 2. Expires: 1-year + 3. Add +9. Copy the client secret value, this will be the client\_secret +10. Select **API permissions** + 1. Click **Add a permission** +11. Select **Microsoft Graph** +12. Select **Application permissions** +13. Select the following permissions: + 1. Files + * Files.Read.All +14. Click **Add permissions** +15. Click **Grant admin consent** + +### Step 2: Set up the Microsoft SharePoint connector in Airbyte + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the **Set up** the source page, select **Microsoft SharePoint** from the Source type dropdown. +4. Enter the name for the Microsoft SharePoint connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. Switch to **Service Key Authentication** +8. For **User Practical Name**, enter the [UPN](https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls) for your user. +9. Enter **Tenant ID**, **Client ID** and **Client secret**. +10. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +11. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +12. Click **Set up source** + + + + + +## Sync overview + +### Data type mapping + +| Integration Type | Airbyte Type | +|:-----------------|:-------------| +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | + +### Features + +| Feature | Supported?\(Yes/No\) | +|:------------------------------|:---------------------| +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | + +### Performance considerations + +The connector is restricted by normal Microsoft Graph [requests limitation](https://docs.microsoft.com/en-us/graph/throttling). + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------| +| 0.1.0 | 2024-01-25 | [33537](https://github.com/airbytehq/airbyte/pull/33537) | New source | + + diff --git a/docs/integrations/sources/microsoft-teams-migrations.md b/docs/integrations/sources/microsoft-teams-migrations.md new file mode 100644 index 000000000000..5610ecd721ad --- /dev/null +++ b/docs/integrations/sources/microsoft-teams-migrations.md @@ -0,0 +1,38 @@ +# Microsoft teams Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 of the Microsoft Teams source connector introduces breaking changes to the schemas of all streams. A full schema refresh is required to ensure a seamless upgrade to this version. + +### Refresh schemas and reset data + +1. Select **Connections** in the main navbar. +2. From the list of your existing connections, select the connection(s) affected by the update. +3. Select the **Replication** tab, then select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** when you are ready to proceed. +::: + +4. At the bottom of the page, select **Save changes**. + +:::caution +Depending on your destination, you may be prompted to **Reset all streams**. Although this step is not required to proceed, it is highly recommended for users who have selected `Full Refresh | Append` sync mode, as the updated schema may lead to inconsistencies in the data structure within the destination. +::: + +5. Select **Save connection**. This will reset the data in your destination (if selected) and initiate a fresh sync. + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + +### Changes in 1.0.0 + +- The naming convention for field names in previous versions used "snake_case", which is not aligned with the "camelCase" convention used by the Microsoft Graph API. For example: + +`user_id` -> `userId` +`created_date` -> `createdDate` + +With the update to "camelCase", fields that may have been unrecognized or omitted in earlier versions will now be properly mapped and included in the data synchronization process, enhancing the accuracy and completeness of your data. + +- The `team_device_usage_report` stream contained a fatal bug that could lead to crashes during syncs. You should now be able to reliably use this stream during syncs. + +- `Date` and `date-time` fields have been typed as airbyte_type `date` and `timestamp_without_timezone`, respectively. diff --git a/docs/integrations/sources/microsoft-teams.md b/docs/integrations/sources/microsoft-teams.md index 1adde3295778..cc3846a489d3 100644 --- a/docs/integrations/sources/microsoft-teams.md +++ b/docs/integrations/sources/microsoft-teams.md @@ -29,22 +29,24 @@ Some APIs aren't supported in v1.0, e.g. channel messages and channel messages r ### Data type mapping -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| Integration Type | Airbyte Type | +| :--------------- | :--------------------------- | +| `string` | `string` | +| `number` | `number` | +| `date` | `date` | +| `datetime` | `timestamp_without_timezone` | +| `array` | `array` | +| `object` | `object` | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Coming soon | | -| Replicate Incremental Deletes | Coming soon | | -| SSL connection | Yes | | -| Namespaces | No | | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ### Performance considerations @@ -54,9 +56,9 @@ The connector is restricted by normal Microsoft Graph [requests limitation](http ### Requirements -* Application \(client\) ID +* Application \(client\) ID * Directory \(tenant\) ID -* Client secrets +* Client secrets ### Setup guide @@ -157,8 +159,9 @@ Token acquiring implemented by [instantiate](https://docs.microsoft.com/en-us/az ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- | :--- | -| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.2.4 | 2021-12-07 | [7807](https://github.com/airbytehq/airbyte/pull/7807) | Implement OAuth support | -| 0.2.3 | 2021-12-06 | [8469](https://github.com/airbytehq/airbyte/pull/8469) | Migrate to the CDK | +| Version | Date | Pull Request | Subject | +|:------- |:---------- | :------------------------------------------------------- | :----------------------------- | +| 1.0.0 | 2024-01-04 | [33959](https://github.com/airbytehq/airbyte/pull/33959) | Schema updates | +| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.2.4 | 2021-12-07 | [7807](https://github.com/airbytehq/airbyte/pull/7807) | Implement OAuth support | +| 0.2.3 | 2021-12-06 | [8469](https://github.com/airbytehq/airbyte/pull/8469) | Migrate to the CDK | diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index 5593a107647b..359318423e2b 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -55,6 +55,9 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------| +| 2.1.0 | 2024-02-13 | [35203](https://github.com/airbytehq/airbyte/pull/35203) | Update stream Funnels schema with custom_event_id and custom_event fields | +| 2.0.2 | 2024-02-12 | [35151](https://github.com/airbytehq/airbyte/pull/35151) | Manage dependencies with Poetry. | +| 2.0.1 | 2024-01-11 | [34147](https://github.com/airbytehq/airbyte/pull/34147) | prepare for airbyte-lib | | 2.0.0 | 2023-10-30 | [31955](https://github.com/airbytehq/airbyte/pull/31955) | Delete the default primary key for the Export stream | | 1.0.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 1.0.0 | 2023-09-27 | [30025](https://github.com/airbytehq/airbyte/pull/30025) | Fix type of datetime field in engage stream; fix primary key for export stream. | @@ -98,4 +101,4 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits | 0.1.3 | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings | | 0.1.2 | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate | | 0.1.1 | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region | -| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | \ No newline at end of file +| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | diff --git a/docs/integrations/sources/monday-migrations.md b/docs/integrations/sources/monday-migrations.md new file mode 100644 index 000000000000..9d095b9e127f --- /dev/null +++ b/docs/integrations/sources/monday-migrations.md @@ -0,0 +1,77 @@ +# Monday Migration Guide + +## Upgrading to 2.0.0 + +Source Monday has deprecated API version 2023-07. We have upgraded the connector to the latest API version 2024-01. In this new version, the Id field has changed from an integer to a string in the streams Boards, Items, Tags, Teams, Updates, Users and Workspaces. Please reset affected streams. + +## Connector Upgrade Guide + +### For Airbyte Open Source: Update the local connector image + +Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: + +1. Select **Settings** in the main navbar. + 1. Select **Sources**. +2. Find Monday in the list of connectors. + +:::note +You will see two versions listed, the current in-use version and the latest version available. +::: + +3. Select **Change** to update your OSS version to the latest available version. + +### Update the connector version + +1. Select **Sources** in the main navbar. +2. Select the instance of the connector you wish to upgrade. + +:::note +Each instance of the connector must be updated separately. If you have created multiple instances of a connector, updating one will not affect the others. +::: + +3. Select **Upgrade** + 1. Follow the prompt to confirm you are ready to upgrade to the new version. + + +### Refresh schemas and reset data + +1. Select **Connections** in the main navbar. +2. Select the connection(s) affected by the update. +3. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +4. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset all streams** option is checked. +5. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navb nar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +:::note +Depending on destination type you may not be prompted to reset your data. +::: +4. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + diff --git a/docs/integrations/sources/monday.md b/docs/integrations/sources/monday.md index 4f0310162d75..065bc3df1c89 100644 --- a/docs/integrations/sources/monday.md +++ b/docs/integrations/sources/monday.md @@ -1,5 +1,7 @@ # Monday +This page contains the setup guide and reference information for the [Monday](https://monday.com/) source connector. + ## Prerequisites * Monday API Token / Monday Access Token @@ -15,18 +17,20 @@ You can get the API token for Monday by going to Profile picture (bottom left co 3. On the Set up the source page, enter the name for the Monday connector and select **Monday** from the Source type dropdown. 4. Fill in your API Key or authenticate using OAuth and then click **Set up source**. -### Connect using `OAuth 2.0` option: +### Connect using `OAuth 2.0` option + 1. Select `OAuth2.0` in `Authorization Method`. 2. Click on `authenticate your Monday account`. -2. Proceed the authentication using your credentials for your Monday account. +3. Proceed with the authentication using the credentials for your Monday account. + +### Connect using `API Token` option -### Connect using `API Token` option: 1. Generate an API Token as described [here](https://developer.monday.com/api-reference/docs/authentication). 2. Use the generated `api_token` in the Airbyte connection. ## Supported sync modes -The Monday supports full refresh syncs +The Monday source connector supports the following features: | Feature | Supported? | |:------------------|:-----------| @@ -49,6 +53,7 @@ Several output streams are available from this source: * [Workspaces](https://developer.monday.com/api-reference/docs/workspaces) Important Notes: + * `Columns` are available from the `Boards` stream. By syncing the `Boards` stream you will get the `Columns` for each `Board` synced in the database The typical name of the table depends on the `destination` you use like `boards.columns`, for instance. @@ -61,16 +66,19 @@ Ids of boards and items are extracted from activity logs events and used to sele Some data may be lost if the time between incremental syncs is longer than the activity logs retention time for your plan. Check your Monday plan at https://monday.com/pricing. - ## Performance considerations The Monday connector should not run into Monday API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. - ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------| +| 2.0.3 | 2024-02-21 | [35506](https://github.com/airbytehq/airbyte/pull/35506) | Support for column values of the mirror type for the `Items` stream. | +| 2.0.2 | 2024-02-12 | [35146](https://github.com/airbytehq/airbyte/pull/35146) | Manage dependencies with Poetry. | +| 2.0.1 | 2024-02-08 | [35016](https://github.com/airbytehq/airbyte/pull/35016) | Migrated to the latest airbyte cdk | +| 2.0.0 | 2024-01-12 | [34108](https://github.com/airbytehq/airbyte/pull/34108) | Migrated to the latest API version: 2024-01 | +| 1.1.4 | 2023-12-13 | [33448](https://github.com/airbytehq/airbyte/pull/33448) | Increase test coverage and migrate to base image | | 1.1.3 | 2023-09-23 | [30248](https://github.com/airbytehq/airbyte/pull/30248) | Add new field "type" to board stream | | 1.1.2 | 2023-08-23 | [29777](https://github.com/airbytehq/airbyte/pull/29777) | Add retry for `502` error | | 1.1.1 | 2023-08-15 | [29429](https://github.com/airbytehq/airbyte/pull/29429) | Ignore `null` records in response | diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index c037dd878a17..c51d5b6d64f1 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -128,47 +128,70 @@ on discovering a self-hosted deployment connection string. To configure the Airbyte MongoDB source, use the database credentials and connection string from steps 1 and 2, respectively. The source will test the connection to the MongoDB instance upon creation. -### Upgrade From Previous Version +## Replication Methods -:::caution +The MongoDB source utilizes change data capture (CDC) as a reliable way to keep your data up to date. -The 1.0.0 version of the MongoDB V2 source connector contains breaking changes from previous versions of the connector. +### CDC -::: +Airbyte utilizes [the change streams feature](https://www.mongodb.com/docs/manual/changeStreams/) of a [MongoDB replica set](https://www.mongodb.com/docs/manual/replication/) to incrementally capture inserts, updates and deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/). -The quickest upgrade path is to click upgrade on any out-of-date connection in the UI. These connections will display -the following message banner: - -> **Action Required** -> There is a pending upgrade for **MongoDB**. -> -> **Version 1.0.0:** -> **We advise against upgrading until you have run a test upgrade as outlined [here](https://docs.airbyte.com/integrations/sources/mongodb-v2-migrations).** This version brings a host of updates to the MongoDB source connector, significantly increasing its scalability and reliability, especially for large collections. As of this version with checkpointing, [CDC incremental updates](https://docs.airbyte.com/understanding-airbyte/cdc) and improved schema discovery, this connector is also now [certified](https://docs.airbyte.com/integrations/). Selecting `Upgrade` will upgrade **all** connections using this source, require you to reconfigure the source, then run a full reset on **all** of your connections. -> -> Upgrade **MongoDB** by **Dec 1, 2023** to continue syncing with this source. For more information, see this [guide](https://docs.airbyte.com/integrations/sources/mongodb-v2). +### Schema Enforcement -After upgrading to the latest version of the MongoDB V2 source connector, users will be required to manually re-configure -existing MongoDB V2 source connector configurations. The required [configuration parameter](#configuration-parameters) values can be discovered -using the [quick start](#quick-start) steps in this documentation. +By default the MongoDB V2 source connector enforces a schema. This means that while setting up a connector it will sample a configureable number of docuemnts and will create a set of fields to sync. From that set of fields, an admin can then deselect specific fields from the Replication screen to filter them out from the sync. +When the schema enforced option is disabled, MongoDB collections are read in schema-less mode which doesn't assume documents share the same structure. +This allows for greater flexibility in reading data that is unstructured or vary a lot in between documents in a single collection. +When schema is not enforced, each document will generate a record that only contains the following top-level fields: +```json +{ + "_id": , + "data": {} +} +``` +The contents of `data` will vary according to the contents of each document read from MongoDB. +Unlike in Schema enforced mode, the same field can vary in type between document. For example field `"xyz"` may be a String on one document and a Date on another. +As a result no field will be omitted and no document will be rejected. +When Schema is not enforced there is not way to deselect fields as all fields are read for every document. -## Replication Methods +## Limitations & Troubleshooting -The MongoDB source utilizes change data capture (CDC) as a reliable way to keep your data up to date. +### MongoDB Oplog and Change Streams -### CDC +[MongoDB's Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) are based on the [Replica Set Oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/). This has retention limitations. Syncs that run less frequently than the retention period of the Oplog may encounter issues with missing data. -Airbyte utilizes [the change streams feature](https://www.mongodb.com/docs/manual/changeStreams/) of a [MongoDB replica set](https://www.mongodb.com/docs/manual/replication/) to incrementally capture inserts, updates and deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/). +We recommend adjusting the Oplog size for your MongoDB cluster to ensure it holds at least 24 hours of changes. For optimal results, we suggest expanding it to maintain a week's worth of data. To adjust your Oplog size, see the corresponding tutorials for [MongoDB Atlas](https://www.mongodb.com/docs/atlas/cluster-additional-settings/#set-oplog-size) (fully-managed) and [MongoDB shell](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/) (self-hosted). -## Limitations & Troubleshooting +If you are running into an issue similar to "invalid resume token", it may mean you need to: +1. Increase the Oplog retention period. +2. Increase the Oplog size. +3. Increase the Airbyte sync frequency. + +You can run the commands outlined [in this tutorial](https://www.mongodb.com/docs/manual/tutorial/troubleshoot-replica-sets/#check-the-size-of-the-oplog) to verify the current of your Oplog. The expect output is: + +```yaml +configured oplog size: 10.10546875MB +log length start to end: 94400 (26.22hrs) +oplog first event time: Mon Mar 19 2012 13:50:38 GMT-0400 (EDT) +oplog last event time: Wed Oct 03 2012 14:59:10 GMT-0400 (EDT) +now: Wed Oct 03 2012 15:00:21 GMT-0400 (EDT) +``` + +When importing a large MongoDB collection for the first time, the import duration might exceed the Oplog retention period. The Oplog is crucial for incremental updates, and an invalid resume token will require the MongoDB collection to be re-imported to ensure no source updates were missed. + +### Supported MongoDB Clusters * Only supports [replica set](https://www.mongodb.com/docs/manual/replication/) cluster type. -* Schema discovery uses [sampling](https://www.mongodb.com/docs/manual/reference/operator/aggregation/sample/) of the documents to collect all distinct top-level fields. This value is universally applied to all collections discovered in the target database. The approach is modelled after [MongoDB Compass sampling](https://www.mongodb.com/docs/compass/current/sampling/) and is used for efficiency. By default, 10,000 documents are sampled. This value can be increased up to 100,000 documents to increase the likelihood that all fields will be discovered. However, the trade-off is time, as a higher value will take the process longer to sample the collection. * TLS/SSL is required by this connector. TLS/SSL is enabled by default for MongoDB Atlas clusters. To enable TSL/SSL connection for a self-hosted MongoDB instance, please refer to [MongoDb Documentation](https://docs.mongodb.com/manual/tutorial/configure-ssl/). * Views, capped collections and clustered collections are not supported. * Empty collections are excluded from schema discovery. * Collections with different data types for the values in the `_id` field among the documents in a collection are not supported. All `_id` values within the collection must be the same data type. -* [MongoDB's change streams](https://www.mongodb.com/docs/manual/changeStreams/) are based on the [Replica Set Oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/), which has retention limitations. Syncs that run less frequently than the retention period of the oplog may encounter issues with missing data. +* Atlas DB cluster are only supported in a dedicated M10 tier and above. Lower tiers may fail during connection setup. + +### Schema Discovery & Enforcement + +* Schema discovery uses [sampling](https://www.mongodb.com/docs/manual/reference/operator/aggregation/sample/) of the documents to collect all distinct top-level fields. This value is universally applied to all collections discovered in the target database. The approach is modelled after [MongoDB Compass sampling](https://www.mongodb.com/docs/compass/current/sampling/) and is used for efficiency. By default, 10,000 documents are sampled. This value can be increased up to 100,000 documents to increase the likelihood that all fields will be discovered. However, the trade-off is time, as a higher value will take the process longer to sample the collection. +* When Running with Schema Enforced set to `false` there is no attempt to discover any schema. See more in [Schema Enforcement](#Schema-Enforcement). ## Configuration Parameters @@ -180,6 +203,7 @@ Airbyte utilizes [the change streams feature](https://www.mongodb.com/docs/manua | Username | The username which is used to access the database. Required for MongoDB Atlas clusters. | | Password | The password associated with this username. Required for MongoDB Atlas clusters. | | Authentication Source | (MongoDB Atlas clusters only) Specifies the database that the supplied credentials should be validated against. Defaults to `admin`. See the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/connection-string/#mongodb-urioption-urioption.authSource) for more details. | +| Schema Enforced | Controls whether schema is discovered and enforced. See discussion in [Schema Enforcement](#Schema-Enforcement). | | Initial Waiting Time in Seconds (Advanced) | The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. | | Size of the queue (Advanced) | The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful. | | Discovery Sample Size (Advanced) | The maximum number of documents to sample when attempting to discover the unique fields for a collection. Default is 10,000 with a valid range of 1,000 to 100,000. See the [MongoDB sampling method](https://www.mongodb.com/docs/compass/current/sampling/#sampling-method) for more details. | @@ -190,6 +214,22 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 1.2.13 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 1.2.12 | 2024-02-21 | [35526](https://github.com/airbytehq/airbyte/pull/35526) | Improve error handling. | +| 1.2.11 | 2024-02-20 | [35375](https://github.com/airbytehq/airbyte/pull/35375) | Add config to throw an error on invalid CDC position and enable it by default. | +| 1.2.10 | 2024-02-13 | [35036](https://github.com/airbytehq/airbyte/pull/34751) | Emit analytics message for invalid CDC cursor. | +| 1.2.9 | 2024-02-13 | [35114](https://github.com/airbytehq/airbyte/pull/35114) | Extend subsequent cdc record wait time to the duration of initial. Bug Fixes | +| 1.2.8 | 2024-02-08 | [34748](https://github.com/airbytehq/airbyte/pull/34748) | Adopt CDK 0.19.0 | +| 1.2.7 | 2024-02-01 | [34759](https://github.com/airbytehq/airbyte/pull/34759) | Fail sync if initial snapshot for any stream fails. | +| 1.2.6 | 2024-01-31 | [34594](https://github.com/airbytehq/airbyte/pull/34594) | Scope initial resume token to streams of interest. | +| 1.2.5 | 2024-01-29 | [34641](https://github.com/airbytehq/airbyte/pull/34641) | Allow resuming an initial snapshot when Id type is not of default ObjectId . | +| 1.2.4 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | +| 1.2.3 | 2024-01-18 | [34364](https://github.com/airbytehq/airbyte/pull/34364) | Add additional logging for resume token + reduce discovery size to 10. | +| 1.2.2 | 2024-01-16 | [34314](https://github.com/airbytehq/airbyte/pull/34314) | Reduce minimum document discovery size to 100. | +| 1.2.1 | 2023-12-18 | [33549](https://github.com/airbytehq/airbyte/pull/33549) | Add logging to understand op log size. | +| 1.2.0 | 2023-12-18 | [33438](https://github.com/airbytehq/airbyte/pull/33438) | Remove LEGACY state flag | +| 1.1.0 | 2023-12-14 | [32328](https://github.com/airbytehq/airbyte/pull/32328) | Schema less mode in mongodb. | +| 1.0.12 | 2023-12-13 | [33430](https://github.com/airbytehq/airbyte/pull/33430) | Add more verbose logging. | | 1.0.11 | 2023-11-28 | [33356](https://github.com/airbytehq/airbyte/pull/33356) | Support for better debugging tools. | | 1.0.10 | 2023-11-28 | [32886](https://github.com/airbytehq/airbyte/pull/32886) | Handle discover phase OOMs | | 1.0.9 | 2023-11-08 | [32285](https://github.com/airbytehq/airbyte/pull/32285) | Additional support to read UUIDs | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 22e5a63e05c4..164b087f6fdf 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -109,7 +109,7 @@ MS SQL Server provides some built-in stored procedures to enable CDC. @source_schema = N'{schema name}', @source_name = N'{table name}', @role_name = N'{role name}', [1] - @filegroup_name = N'{fiilegroup name}', [2] + @filegroup_name = N'{filegroup name}', [2] @supports_net_changes = 0 [3] GO ``` @@ -342,6 +342,18 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.7.3 | 2024-02-23 | [35596](https://github.com/airbytehq/airbyte/pull/35596) | Fix a logger issue | +| 3.7.2 | 2024-02-21 | [35368](https://github.com/airbytehq/airbyte/pull/35368) | Change query syntax to make it compatible with Azure SQL Managed Instance. | +| 3.7.1 | 2024-02-20 | [35405](https://github.com/airbytehq/airbyte/pull/35405) | Change query syntax to make it compatible with Azure Synapse. | +| 3.7.0 | 2024-01-30 | [33311](https://github.com/airbytehq/airbyte/pull/33311) | Source mssql with checkpointing initial sync. | +| 3.6.1 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | +| 3.6.0 | 2024-01-10 | [33700](https://github.com/airbytehq/airbyte/pull/33700) | Remove CDC config options for data_to_sync and snapshot isolation. | +| 3.5.1 | 2024-01-05 | [33510](https://github.com/airbytehq/airbyte/pull/33510) | Test-only changes. | +| 3.5.0 | 2023-12-19 | [33071](https://github.com/airbytehq/airbyte/pull/33071) | Fix SSL configuration parameters | +| 3.4.1 | 2024-01-02 | [33755](https://github.com/airbytehq/airbyte/pull/33755) | Encode binary to base64 format | +| 3.4.0 | 2023-12-19 | [33481](https://github.com/airbytehq/airbyte/pull/33481) | Remove LEGACY state flag | +| 3.3.2 | 2023-12-14 | [33505](https://github.com/airbytehq/airbyte/pull/33505) | Using the released CDK. | +| 3.3.1 | 2023-12-12 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | extracting MsSql specific files out of the CDK. | | 3.3.0 | 2023-12-12 | [33018](https://github.com/airbytehq/airbyte/pull/33018) | Migrate to Per-stream/Global states and away from Legacy states | | 3.2.1 | 2023-12-11 | [33330](https://github.com/airbytehq/airbyte/pull/33330) | Parse DatetimeOffset fields with the correct format when used as cursor | | 3.2.0 | 2023-12-07 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | CDC : Enable compression of schema history blob in state. | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 83fc11dbe3c5..26aafa4f12df 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -223,6 +223,18 @@ Any database or table encoding combination of charset and collation is supported | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.3.11 | 2024-02-23 | [35527](https://github.com/airbytehq/airbyte/pull/35527) | Adopt 0.23.1 and shutdown timeouts. | +| 3.3.10 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 3.3.9 | 2024-02-21 | [35525](https://github.com/airbytehq/airbyte/pull/35338) | Adopt 0.21.4 and reduce cdc state compression threshold to 1MB. | +| 3.3.8 | 2024-02-20 | [35338](https://github.com/airbytehq/airbyte/pull/35338) | Add config to throw an error on invalid CDC position. | +| 3.3.7 | 2024-02-13 | [35036](https://github.com/airbytehq/airbyte/pull/34751) | Emit analytics message for invalid CDC cursor. | +| 3.3.6 | 2024-02-13 | [34869](https://github.com/airbytehq/airbyte/pull/34573) | Don't emit state in SourceStateIterator when there is an underlying stream failure. | +| 3.3.5 | 2024-02-12 | [34580](https://github.com/airbytehq/airbyte/pull/34580) | Support special chars in db name | +| 3.3.4 | 2024-02-08 | [34750](https://github.com/airbytehq/airbyte/pull/34750) | Adopt CDK 0.19.0 | +| 3.3.3 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | +| 3.3.2 | 2024-01-08 | [33005](https://github.com/airbytehq/airbyte/pull/33005) | Adding count stats for incremental sync in AirbyteStateMessage | +| 3.3.1 | 2024-01-03 | [33312](https://github.com/airbytehq/airbyte/pull/33312) | Adding count stats in AirbyteStateMessage | +| 3.3.0 | 2023-12-19 | [33436](https://github.com/airbytehq/airbyte/pull/33436) | Remove LEGACY state flag | | 3.2.4 | 2023-12-12 | [33356](https://github.com/airbytehq/airbyte/pull/33210) | Support for better debugging tools.. | | 3.2.3 | 2023-12-08 | [33210](https://github.com/airbytehq/airbyte/pull/33210) | Update MySql driver property value for zero date handling. | | 3.2.2 | 2023-12-06 | [33082](https://github.com/airbytehq/airbyte/pull/33082) | Improvements to MySQL schema snapshot error handling. | diff --git a/docs/integrations/sources/mysql/mysql-troubleshooting.md b/docs/integrations/sources/mysql/mysql-troubleshooting.md index aee512157839..7e6265d0b867 100644 --- a/docs/integrations/sources/mysql/mysql-troubleshooting.md +++ b/docs/integrations/sources/mysql/mysql-troubleshooting.md @@ -18,6 +18,18 @@ * Amazon RDS MySQL or MariaDB connection issues: If you see the following `Cannot create a PoolableConnectionFactory` error, please add `enabledTLSProtocols=TLSv1.2` in the JDBC parameters. * Amazon RDS MySQL connection issues: If you see `Error: HikariPool-1 - Connection is not available, request timed out after 30001ms.`, many times this due to your VPC not allowing public traffic. We recommend going through [this AWS troubleshooting checklist](https://aws.amazon.com/premiumsupport/knowledge-center/rds-cannot-connect/) to ensure the correct permissions/settings have been granted to allow Airbyte to connect to your database. +### Under CDC incremental mode, there are still full refresh syncs + +Normally under the CDC mode, the MySQL source will first run a full refresh sync to read the snapshot of all the existing data, and all subsequent runs will only be incremental syncs reading from the binlogs. However, occasionally, you may see full refresh syncs after the initial run. When this happens, you will see the following log: + +> Saved offset no longer present on the server, Airbyte is going to trigger a sync from scratch + +The root causes is that the binglogs needed for the incremental sync have been removed by MySQL. This can occur under the following scenarios: + +- When there are lots of database updates resulting in more WAL files than allowed in the `pg_wal` directory, Postgres will purge or archive the WAL files. This scenario is preventable. Possible solutions include: + - Sync the data source more frequently. + - Set a higher `binlog_expire_logs_seconds`. It's recommended to set this value to a time period of 7 days. See detailed documentation [here](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#sysvar_binlog_expire_logs_seconds). The downside of this approach is that more disk space will be needed. + ### EventDataDeserializationException errors during initial snapshot When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. Airbyte doesn't acquire any table locks \(for tables defined with MyISAM engine, the tables would still be locked\) while creating the snapshot to allow writes by other database clients. But in order for the sync to work without any error/unexpected behaviour, it is assumed that no schema changes are happening while the snapshot is running. diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 83343623f217..04d96e2a9be2 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -110,42 +110,44 @@ The connector is restricted by Notion [request limits](https://developers.notion ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------- | -| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | -| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | -| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | -| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | -| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | -| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | -| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | -| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | -| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | -| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | -| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | -| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | -| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | -| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | -| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | -| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | -| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | -| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | -| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | -| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | -| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | -| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | -| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | -| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | -| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | -| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | -| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | -| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | -| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | -| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | -| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | -| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | -| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | -| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| 2.1.0 | 2024-02-19 | [35409](https://github.com/airbytehq/airbyte/pull/35409) | Update users stream schema with bot type info fields and block schema with mention type info fields. | +| 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | +| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | +| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | +| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | +| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | +| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | +| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | +| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | +| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | +| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | +| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | +| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | +| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | +| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | +| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | +| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | +| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | +| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | +| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | +| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | +| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | +| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | +| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | +| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | +| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | +| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | +| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | +| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | +| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | +| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | +| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | +| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | +| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | +| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | +| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | diff --git a/docs/integrations/sources/opsgenie.md b/docs/integrations/sources/opsgenie.md index 088d27f21988..a40bdc02de50 100644 --- a/docs/integrations/sources/opsgenie.md +++ b/docs/integrations/sources/opsgenie.md @@ -51,6 +51,7 @@ The Opsgenie connector uses the most recent API version for each source of data. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------| :--- | +| 0.3.1 | 2024-02-14 | [35269](https://github.com/airbytehq/airbyte/pull/35269) | Fix parsing of updated_at timestamps in alerts | | 0.3.0 | 2023-10-19 | [31552](https://github.com/airbytehq/airbyte/pull/31552) | Migrated to Low Code | | 0.2.0 | 2023-10-24 | [31777](https://github.com/airbytehq/airbyte/pull/31777) | Fix schema | | 0.1.0 | 2022-09-14 | [16768](https://github.com/airbytehq/airbyte/pull/16768) | Initial Release | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index e4493f950b19..61b9b69b0f43 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -131,7 +131,10 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.5.2 | 2024-02-13 | [35225](https://github.com/airbytehq/airbyte/pull/35225) | Adopt CDK 0.20.4 | +| 0.5.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.5.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | | 0.4.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.3.25 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | | 0.3.24 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index c4304a8abe0e..5de66dd17329 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -55,7 +55,7 @@ The Pardot connector should not run into Pardot API limitations under normal usa * `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce * `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. * `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -* `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. +* `is_sandbox`: Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. ## Changelog diff --git a/docs/integrations/sources/paypal-transaction-migrations.md b/docs/integrations/sources/paypal-transaction-migrations.md index 862e65763d54..abe8b5f55900 100644 --- a/docs/integrations/sources/paypal-transaction-migrations.md +++ b/docs/integrations/sources/paypal-transaction-migrations.md @@ -6,6 +6,6 @@ Version 2.1.0 changes the format of the state object. Upgrading to 2.1.0 is safe To downgrade to 2.0.0: - Edit your connection state: - - Change the the keys for the transactions and balances streams to "date" + - Change the keys for the transactions and balances streams to "date" - Change the format of the cursor to "yyyy-MM-dd'T'HH:mm:ss+00:00" Alternatively, you can also reset your connection. diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index 0abf7725b540..50df0f002d98 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -1,48 +1,60 @@ -# Paypal Transaction +# Paypal -This page contains the setup guide and reference information for the Paypal Transaction source connector. +This page contains the setup guide and reference information for the Paypal source connector. + +This connector uses [PayPal APIs](https://developer.paypal.com/api/rest/authentication/) OAuth 2.0 access token to authenticate requests. ## Prerequisites -The [Paypal Transaction API](https://developer.paypal.com/docs/api/transaction-search/v1/) is used to get the history of transactions for a PayPal account. +You will need a Paypal account, which you can get following [these steps](https://developer.paypal.com/docs/platforms/get-started/) + +In the same page, you will also find how to setup a Sandbox so you can test the connector before using it in production. ## Setup guide -### Step 1: Set up Paypal Transaction +### Step 1: Get your Paypal secrets + +After creating your account you will be able to get your `Client ID` and `Secret`. You can find them in your [Apps & Credentials page](https://developer.paypal.com/dashboard/applications/live). + + +### Step 2: Set up the Paypal Transaction connector in Airbyte + + +1. Log into your Airbyte account + - For Cloud [Log in here](https://cloud.airbyte.com/workspaces). + +2. In the left navigation bar, click **Sources**. + + a. If this is your first time creating a source, use the search bar and enter **Paypal Transaction** and select it. + + b. If you already have sources configured, go to the top-right corner and click **+new source**. Then enter **Paypal Transaction** in the searech bar and select the connector. + +3. Set the name for your source +4. Enter your `Client ID` +5. Enter your `Client secret` +6. `Start Date`: Use the provided datepicker or enter manually a UTC date and time in the format `YYYY-MM-DDTHH:MM:SSZ`. +7. Switch ON/Off the Sandbox toggle. By defaukt the toggle is OFF, meaning it work only in a produciton environment. +8. _(Optional) `Dispute Start Date Range`: Use the provided datepicker or enter manually a UTC date and time in the format `YYYY-MM-DDTHH:MM:SS.sssZ`. + - If you don't add a date and you sync the `lists_disputes stream`, it will use the default value of 180 days in the past to retrieve data + - It is mandatory to add the milliseconds is you enter a datetime. + - This option only works for `lists_disputes stream` -In order to get an `Client ID` and `Secret` please go to [this](https://developer.paypal.com/docs/platforms/get-started/) page and follow the instructions. After registration you may find your `Client ID` and `Secret` [here](https://developer.paypal.com/developer/accounts/). +9. _(Optional)`Refresh Token`:_ You can enter manually a refresh token. Right now the stream does this automatically. +10. _(Optional)`Number of days per request`:_ You can specify the days used by the connector when requesting data from the Paypal API. This helps in cases when you have a rate limit and you want to lower the window of retrieving data. + - Paypal has a 10K record limit per request. This option is useful if your sync is every week and you have more than 10K per week + - The default value is 7 + - This Max value you can enter is 31 days + +11. Click **Set up source** -:::note +:::info -Our Paypal Transactions Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Paypal Transactions is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort. +By default, syncs are run with a slice period of 7 days. If you see errors with the message `Result set size is greater than the maximum limit` or an error code like `RESULTSET_TOO_LARGE`: + +- Try lower the the size of the slice period in your optional parameters in your connection configuration. +- You can try to lower the scheduling sync window in case a day slice period is not enough. Lowering the sync period it may help avoid reaching the 10K limit. ::: -## Step 2: Set up the Paypal Transaction connector in Airbyte - - -**For Airbyte Cloud:** - -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Paypal Transaction connector and select **Paypal Transaction** from the Source type dropdown. -4. Enter your client id -5. Enter your secret -6. Choose if your account is sandbox -7. Enter the date you want your sync to start from -8. Click **Set up source**. - - - -**For Airbyte Open Source:** - -1. Navigate to the Airbyte Open Source dashboard -2. Set the name for your source -3. Enter your client id -4. Enter your secret -5. Choose if your account is sandbox -6. Enter the date you want your sync to start from -7. Click **Set up source** - ## Supported sync modes @@ -50,9 +62,10 @@ The PayPal Transaction source connector supports the following [sync modes](http | Feature | Supported? | | :------------------------ | :--------- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Namespaces | No | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | + ## Supported Streams @@ -60,36 +73,219 @@ This Source is capable of syncing the following core Streams: * [Transactions](https://developer.paypal.com/docs/api/transaction-search/v1/#transactions) * [Balances](https://developer.paypal.com/docs/api/transaction-search/v1/#balances) +* [List Products](https://developer.paypal.com/docs/api/catalog-products/v1/#products_list) +* [Show Product Details](https://developer.paypal.com/docs/api/catalog-products/v1/#products_get) +* [List Disputes](https://developer.paypal.com/docs/api/customer-disputes/v1/#disputes_list) +* [Search Invoices](https://developer.paypal.com/docs/api/invoicing/v2/#invoices_search-invoices) +* [List Payments](https://developer.paypal.com/docs/api/payments/v1/#payment_list) + + +### Transactions Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features + +| **Param/Feature** | `Transactions` | +| :-------------------------- | :------------------------ | +| `Start Date` | Timestamp with TZ (no ms) | +| `Dispute Start Date Range` | NA | +| `Refresh token` | Auto | +| `Number of days per request`| Max 31 , 7(D) | +| `Pagination Strategy` | Page Increment | +| `Page size ` | Max 500 (F) | +| `Full Refresh` | :white_check_mark: | +| `Incremental` | :white_check_mark: (D) | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +___ + +### Balances Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features + +| **Param/Feature** |`Balances` | +| :-------------------------- |:------------------------ | +| `Start Date` |Timestamp with TZ (no ms) | +| `Dispute Start Date Range` |NA | +| `Refresh token` |Auto | +| `Number of days per request`|NA | +| `Pagination Strategy` |NA | +| `Page size ` |NA | +| `Full Refresh` |:white_check_mark: | +| `Incremental` |:white_check_mark: (D) | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +___ + + +### List Products Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features + + +| **Param/Feature** |`List Products` | +| :-------------------------- |:------------------------ | +| `Start Date` |NA | +| `Dispute Start Date Range` |NA | +| `Refresh token` |Auto | +| `Number of days per request`|NA | +| `Pagination Strategy` |Page Increment | +| `Page size ` |Max 20 (F) | +| `Full Refresh` |:white_check_mark: (D) | +| `Incremental` |:x: | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +:::caution + +When configuring your stream take in consideration that the way the API works limits the speed on retreiving data. In some cases a +30K catalog retrieval could take between 10-15 minutes. + +::: + +___ + +### Show Products Stream -## Performance considerations +The below table contains the configuraiton parameters available for this connector and the default values and available features -Paypal transaction API has some [limits](https://developer.paypal.com/docs/integration/direct/transaction-search/) +| **Param/Feature** |`Show Prod. Details` | +| :-------------------------- |:------------------------ | +| `Start Date` |NA | +| `Dispute Start Date Range` |NA | +| `Refresh token` |Auto | +| `Number of days per request`|NA | +| `Pagination Strategy` |NA | +| `Page size ` |NA | +| `Full Refresh` |:white_check_mark: (D) | +| `Incremental` |:x: | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + + +:::caution + +When configuring this stream consider that the parent stream paginates with 20 number of items (Max alowed page size). The Paypal API calls are not concurrent, so the time it takes depends entirely on the server side. +This stream could take a considerable time syncing, so you should consider running the sync of this and the parent stream (`list_products`) at the end of the day. +Depending on the size of the catalog it could take several hours to sync. + +::: + +___ + +### List Disputes Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features + +| **Param/Feature** |`List Disputes` | +| :-------------------------- |:------------------------ | +| `Start Date` |NA | +| `Dispute Start Date Range` |Timestamp with TZ (w/ms) | +| `Refresh token` |Auto | +| `Number of days per request`|Max 180 , 7(D) | +| `Pagination Strategy` |Page Token | +| `Page size ` |Max 50 (F) | +| `Full Refresh` |:white_check_mark: | +| `Incremental` |:white_check_mark: (D) | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +___ + +### Search Invoices Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features + +| **Param/Feature** |`Search Invoices` | +| :-------------------------- |:------------------------ | +| `Start Date` |Timestamp with TZ (no ms) | +| `Dispute Start Date Range` |NA | +| `Refresh token` |Auto | +| `Number of days per request`|ND | +| `Pagination Strategy` |Page Number | +| `Page size ` |Max 100 (F) | +| `Full Refresh` |:white_check_mark: (D) | +| `Incremental` |:x: | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +**ND:** Not Defined in the source. + + +:::info + +The `start_end` from the configuration, is passed to the body of the request and uses the `creation_date_range.start` and `creation_date_range.end`. More information in the [Paypal Developer API documentation](https://developer.paypal.com/docs/api/invoicing/v2/#invoices_search-invoices). + +::: + + +___ + +### List Payments Stream + +The below table contains the configuraiton parameters available for this connector and the default values and available features. + +| **Param/Feature** |`List Payments` | +| :-------------------------- |:------------------------ | +| `Start Date` |Timestamp with TZ (no ms) | +| `Dispute Start Date Range` |NA | +| `Refresh token` |Auto | +| `Number of days per request`|NA , 7(D) | +| `Pagination Strategy` |Page Cursor | +| `Page size ` |Max 20 (F) | +| `Full Refresh` |:white_check_mark: | +| `Incremental` |:white_check_mark: (D) | + +**D:** Default configured Value + +**F:** Fixed Value. This means it is not configurable. + +___ + +## Performance Considerations + +* **Data Availability:** It takes a maximum of 3 hours for executed transactions to appear in the list transactions call. +* **Number of days per request:** The maximum supported date range is 31 days. +* **Historical Data:** You can't retrieve more than 3yrs of data for the `transactions` stream. For `dispute_start_date` you can only retrieve 180 days of data (see specifications per stream) +* `records_per_request`: The maximum number of records in a single request are 10K (API Server restriction) +* `page_size`: The maximum page size is 500. This has been configured by default. +* `requests_per_minute` = The maximum limit is 50 requests per minute from IP address to all endpoint (API Server restriction). -* `start_date_min` = 3 years, API call lists transaction for the previous three years. -* `start_date_max` = 1.5 days, it takes a maximum of three hours for executed transactions to appear in the list transactions call. It is set to 1.5 days by default based on experience, otherwise API throw an error. -* `stream_slice_period` = 7 day, the maximum supported date range is 31 days. -* `records_per_request` = 10000, the maximum number of records in a single request. -* `page_size` = 500, the maximum page size is 500. -* `requests_per_minute` = 30, maximum limit is 50 requests per minute from IP address to all endpoint -By default, syncs are performed with a slice period of 7 days. If you see errors with the message `Result set size is greater than the maximum limit. Change the filter criteria and try again.`, lower the size of the slice period in your connection configuration. ## Data type map | Integration Type | Airbyte Type | | :--------------- | :----------- | -| `string` | `string` | -| `number` | `number` | -| `array` | `array` | -| `object` | `object` | +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | + ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------| -| 2.2.0 | 2023-10-25 | [31852](https://github.com/airbytehq/airbyte/pull/31852) | The size of the time_window can be configured | -| 2.1.2 | 2023-10-23 | [31759](https://github.com/airbytehq/airbyte/pull/31759) | Keep transaction_id as a string and fetch data in 7-day batches -| 2.1.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.4.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | +| 2.3.0 | 2024-02-14 | [34510](https://github.com/airbytehq/airbyte/pull/34510) | Silver certified. New Streams Added | +| 2.2.2 | 2024-02-09 | [35075](https://github.com/airbytehq/airbyte/pull/35075) | Manage dependencies with Poetry. | +| 2.2.1 | 2024-01-11 | [34155](https://github.com/airbytehq/airbyte/pull/34155) | prepare for airbyte-lib | +| 2.2.0 | 2023-10-25 | [31852](https://github.com/airbytehq/airbyte/pull/31852) | The size of the time_window can be configured | +| 2.1.2 | 2023-10-23 | [31759](https://github.com/airbytehq/airbyte/pull/31759) | Keep transaction_id as a string and fetch data in 7-day batches | +| 2.1.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 2.1.0 | 2023-08-14 | [29223](https://github.com/airbytehq/airbyte/pull/29223) | Migrate Python CDK to Low Code schema | | 2.0.0 | 2023-07-05 | [27916](https://github.com/airbytehq/airbyte/pull/27916) | Update `Balances` schema | | 1.0.0 | 2023-07-03 | [27968](https://github.com/airbytehq/airbyte/pull/27968) | mark `Client ID` and `Client Secret` as required fields | diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index f58cf38cebaf..3dbd7a7b8227 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -129,6 +129,8 @@ The connector is restricted by the Pinterest [requests limitation](https://devel | Version | Date | Pull Request | Subject | |:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.2.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | +| 1.1.1 | 2024-02-12 | [35159](https://github.com/airbytehq/airbyte/pull/35159) | Manage dependencies with Poetry. | | 1.1.0 | 2023-11-22 | [32747](https://github.com/airbytehq/airbyte/pull/32747) | Update docs and spec. Add missing `placement_traffic_type` field to AdGroups stream | | 1.0.0 | 2023-11-16 | [32595](https://github.com/airbytehq/airbyte/pull/32595) | Add airbyte_type: timestamp_without_timezone to date-time fields across all streams. Rename `Advertizer*` streams to `Advertiser*` | | 0.8.2 | 2023-11-20 | [32672](https://github.com/airbytehq/airbyte/pull/32672) | Fix backoff waiting time | diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index 32a0bc0781c9..cb87f1d27fb2 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -114,6 +114,7 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 2.2.2 | 2024-01-11 | [34153](https://github.com/airbytehq/airbyte/pull/34153) | prepare for airbyte-lib | | 2.2.1 | 2023-11-06 | [31147](https://github.com/airbytehq/airbyte/pull/31147) | Bugfix: handle records with a null data field | | 2.2.0 | 2023-10-25 | [31707](https://github.com/airbytehq/airbyte/pull/31707) | Add new stream mail | | 2.1.0 | 2023-10-10 | [31184](https://github.com/airbytehq/airbyte/pull/31184) | Add new stream goals | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index e94d760b9458..56ddc87ab7bb 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -1,6 +1,7 @@ # Postgres Airbyte's certified Postgres connector offers the following features: +* Replicate data from tables, views and materilized views. Other data objects won't be replicated to the destination like indexes, permissions. * Multiple methods of keeping your data fresh, including [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) and replication using the [xmin system column](#xmin). * All available [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes), providing flexibility in how data is delivered to your destination. * Reliable replication at any table size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of database reads. @@ -291,6 +292,20 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.3.12 | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. | +| 3.3.11 | 2024-02-20 | [35304](https://github.com/airbytehq/airbyte/pull/35304) | Add config to throw an error on invalid CDC position and enable it by default. | +| 3.3.10 | 2024-02-13 | [35036](https://github.com/airbytehq/airbyte/pull/34751) | Emit analytics message for invalid CDC cursor. | +| 3.3.9 | 2024-02-13 | [35224](https://github.com/airbytehq/airbyte/pull/35224) | Adopt CDK 0.20.4 | +| 3.3.8 | 2024-02-08 | [34751](https://github.com/airbytehq/airbyte/pull/34751) | Adopt CDK 0.19.0 | +| 3.3.7 | 2024-02-08 | [34781](https://github.com/airbytehq/airbyte/pull/34781) | Add a setting in the setup page to advance the LSN. | +| 3.3.6 | 2024-02-07 | [34892](https://github.com/airbytehq/airbyte/pull/34892) | Adopt CDK v0.16.6 | +| 3.3.5 | 2024-02-07 | [34948](https://github.com/airbytehq/airbyte/pull/34948) | Adopt CDK v0.16.5 | +| 3.3.4 | 2024-01-31 | [34723](https://github.com/airbytehq/airbyte/pull/34723) | Adopt CDK v0.16.3 | +| 3.3.3 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0 | +| 3.3.2 | 2024-01-24 | [34465](https://github.com/airbytehq/airbyte/pull/34465) | Check xmin only if user selects xmin sync mode. | +| 3.3.1 | 2024-01-10 | [34119](https://github.com/airbytehq/airbyte/pull/34119) | Adopt java CDK version 0.11.5. | +| 3.3.0 | 2023-12-19 | [33437](https://github.com/airbytehq/airbyte/pull/33437) | Remove LEGACY state flag | +| 3.2.27 | 2023-12-18 | [33605](https://github.com/airbytehq/airbyte/pull/33605) | Advance Postgres LSN for PG 14 & below. | | 3.2.26 | 2023-12-11 | [33027](https://github.com/airbytehq/airbyte/pull/32961) | Support for better debugging tools. | | 3.2.25 | 2023-11-29 | [32961](https://github.com/airbytehq/airbyte/pull/32961) | Bump debezium wait time default to 20 min. | | 3.2.24 | 2023-11-28 | [32686](https://github.com/airbytehq/airbyte/pull/32686) | Better logging to understand dbz closing reason attribution. | diff --git a/docs/integrations/sources/postgres/postgres-troubleshooting.md b/docs/integrations/sources/postgres/postgres-troubleshooting.md index 556118a6c1d1..329cc2af7274 100644 --- a/docs/integrations/sources/postgres/postgres-troubleshooting.md +++ b/docs/integrations/sources/postgres/postgres-troubleshooting.md @@ -78,7 +78,7 @@ Normally under the CDC mode, the Postgres source will first run a full refresh s The root causes is that the WALs needed for the incremental sync has been removed by Postgres. This can occur under the following scenarios: - When there are lots of database updates resulting in more WAL files than allowed in the `pg_wal` directory, Postgres will purge or archive the WAL files. This scenario is preventable. Possible solutions include: - - Sync the data source more frequently. The downside is that more computation resources will be consumed, leading to a higher Airbyte bill. + - Sync the data source more frequently. - Set a higher `wal_keep_size`. If no unit is provided, it is in megabytes, and the default is `0`. See detailed documentation [here](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-KEEP-SIZE). The downside of this approach is that more disk space will be needed. - When the Postgres connector successfully reads the WAL and acknowledges it to Postgres, but the destination connector fails to consume the data, the Postgres connector will try to read the same WAL again, which may have been removed by Postgres, since the WAL record is already acknowledged. This scenario is rare, because it can happen, and currently there is no way to prevent it. The correct behavior is to perform a full refresh. @@ -110,4 +110,14 @@ The connector waits for the default initial wait time of 5 minutes (300 seconds) If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. +### (Advanced) WAL disk consumption and heartbeat action query +In certain situations, WAL disk consumption increases. This can occur when there are a large volume of changes, but only a small percentage of them are being made to the databases, schemas and tables configured for capture. + +A workaround for this situation is to artificially add events to a heartbeat table that the Airbyte use has write access to. This will ensure that Airbyte can process the WAL and prevent disk space to spike. To configure this: +1. Create a table (e.g. `airbyte_heartbeat`) in the database and schema being tracked. +2. Add this table to the airbyte publication. +3. Configure the `heartbeat_action_query` property while setting up the source-postgres connector. This query will be periodically executed by Airbyte on the `airbyte_heartbeat` table. For example, this param can be set to a query like `INSERT INTO airbyte_heartbeat (text) VALUES ('heartbeat')`. + + +See detailed documentation [here](https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-wal-disk-space). diff --git a/docs/integrations/sources/quickbooks.md b/docs/integrations/sources/quickbooks.md index caf2dfde4b05..8e74d5a6081f 100644 --- a/docs/integrations/sources/quickbooks.md +++ b/docs/integrations/sources/quickbooks.md @@ -105,6 +105,8 @@ This Source is capable of syncing the following [Streams](https://developer.intu | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------- | +| `3.0.2` | 2024-02-20 | [32236](https://github.com/airbytehq/airbyte/pull/32236) | Small typo in spec correction | +| `3.0.1` | 2023-11-06 | [32236](https://github.com/airbytehq/airbyte/pull/32236) | Upgrade to `airbyte-cdk>=0.52.10` to resolve refresh token issues | | `3.0.0` | 2023-09-26 | [30770](https://github.com/airbytehq/airbyte/pull/30770) | Update schema to use `number` instead of `integer` | | `2.0.5` | 2023-09-26 | [30766](https://github.com/airbytehq/airbyte/pull/30766) | Fix improperly named keyword argument | | `2.0.4` | 2023-06-28 | [27803](https://github.com/airbytehq/airbyte/pull/27803) | Update following state breaking changes | diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 7eb6f1a7f031..fa784da249da 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,6 +76,9 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| +| 1.1.5 | 2024-02-12 | [35182](https://github.com/airbytehq/airbyte/pull/35182) | Manage dependencies with Poetry. | +| 1.1.4 | 2024-02-02 | [34772](https://github.com/airbytehq/airbyte/pull/34772) | Fix airbyte-lib distribution | +| 1.1.3 | 2024-01-31 | [34707](https://github.com/airbytehq/airbyte/pull/34707) | Added the UI toggle `Use 'Orders' Deprecated API` to switch between `deprecated` and `modern` api versions for `Orders` stream | | 1.1.2 | 2023-11-03 | [32132](https://github.com/airbytehq/airbyte/pull/32132) | Reduced `period in days` value for `Subscriptions` stream, to avoid `504 - Gateway TimeOut` error | | 1.1.1 | 2023-09-26 | [30782](https://github.com/airbytehq/airbyte/pull/30782) | For the new style pagination, pass only limit along with cursor | | 1.1.0 | 2023-09-26 | [30756](https://github.com/airbytehq/airbyte/pull/30756) | Fix pagination and slicing | diff --git a/docs/integrations/sources/recurly.md b/docs/integrations/sources/recurly.md index fe5560d9bc5d..2fac238fb528 100644 --- a/docs/integrations/sources/recurly.md +++ b/docs/integrations/sources/recurly.md @@ -64,6 +64,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | |:--------|:-----------| :--------------------------------------------------------| :--------------------------------------------------------------------------------------- | +| 0.5.0 | 2024-02-22 | [34622](https://github.com/airbytehq/airbyte/pull/34622) | Republish connector using base image/Poetry, update schemas | | 0.4.1 | 2022-06-10 | [13685](https://github.com/airbytehq/airbyte/pull/13685) | Add state_checkpoint_interval to Recurly stream | | 0.4.0 | 2022-01-28 | [9866](https://github.com/airbytehq/airbyte/pull/9866) | Revamp Recurly Schema and add more resources | | 0.3.2 | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index f7d84b6e06d2..4594602feb2b 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -55,7 +55,10 @@ All Redshift connections are encrypted using SSL ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.5.2 | 2024-02-13 | [35223](https://github.com/airbytehq/airbyte/pull/35223) | Adopt CDK 0.20.4 | +| 0.5.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.5.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | | (none) | 2023-11-17 | [32616](https://github.com/airbytehq/airbyte/pull/32616) | Improve timestamptz handling | | 0.4.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.3.17 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 2437d6f496be..1586348a10d8 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -17,6 +17,8 @@ Please note that using cloud storage may incur egress costs. Egress refers to da **If you are syncing from a private bucket**, you will need to provide both an `AWS Access Key ID` and `AWS Secret Access Key` to authenticate the connection. The IAM user associated with the credentials must be granted `read` and `list` permissions for the bucket and its objects. If you are unfamiliar with configuring AWS permissions, you can follow these steps to obtain the necessary permissions and credentials: +#### Create a Policy + 1. Log in to your Amazon AWS account and open the [IAM console](https://console.aws.amazon.com/iam/home#home). 2. In the IAM dashboard, select **Policies**, then click **Create Policy**. 3. Select the **JSON** tab, then paste the following JSON into the Policy editor (be sure to substitute in your bucket name): @@ -49,7 +51,7 @@ At this time, object-level permissions alone are not sufficient to successfully 6. If you are using an _existing_ IAM user, click the **Add permissions** dropdown menu and select **Add permissions**. If you are creating a _new_ user, you will be taken to the Permissions screen after selecting a name. 7. Select **Attach policies directly**, then find and check the box for your new policy. Click **Next**, then **Add permissions**. 8. After successfully creating your user, select the **Security credentials** tab and click **Create access key**. You will be prompted to select a use case and add optional tags to your access key. Click **Create access key** to generate the keys. - + :::caution Your `Secret Access Key` will only be visible once upon creation. Be sure to copy and store it securely for future use. ::: @@ -71,6 +73,8 @@ For more information on managing your access keys, please refer to the 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. 6. **If you are syncing from a private bucket**, you must fill the **AWS Access Key ID** and **AWS Secret Access Key** fields with the appropriate credentials to authenticate the connection. All other fields are optional and can be left empty. Refer to the [S3 Provider Settings section](#s3-provider-settings) below for more information on each field. +All other fields are optional and can be left empty. Refer to the [S3 Provider Settings section](#s3-provider-settings) below for more information on each field. + ## Supported sync modes The Amazon S3 source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -84,6 +88,10 @@ The Amazon S3 source connector supports the following [sync modes](https://docs. | Replicate Multiple Streams \(distinct tables\) | Yes | | Namespaces | No | +## Supported streams + +There is no predefined streams. The streams are based on content of your bucket. + ## File Compressions | Compression | Supported? | @@ -245,7 +253,7 @@ There are currently no options for JSONL parsing. The Document File Type Format is currently an experimental feature and not subject to SLAs. Use at your own risk. ::: -The Document File Type Format is a special format that allows you to extract text from Markdown, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. +The Document File Type Format is a special format that allows you to extract text from Markdown, TXT, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. @@ -256,6 +264,18 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.5.7 | 2024-02-23 | [34895](https://github.com/airbytehq/airbyte/pull/34895) | Run incremental syncs with concurrency | +| 4.5.6 | 2024-02-21 | [35246](https://github.com/airbytehq/airbyte/pull/35246) | Fixes bug that occurred when creating CSV streams with tab delimiter. | +| 4.5.5 | 2024-02-18 | [35392](https://github.com/airbytehq/airbyte/pull/35392) | Add support filtering by start date | +| 4.5.4 | 2024-02-15 | [35055](https://github.com/airbytehq/airbyte/pull/35055) | Temporarily revert concurrency | +| 4.5.3 | 2024-02-12 | [35164](https://github.com/airbytehq/airbyte/pull/35164) | Manage dependencies with Poetry. | +| 4.5.2 | 2024-02-06 | [34930](https://github.com/airbytehq/airbyte/pull/34930) | Bump CDK version to fix issue when SyncMode is missing from catalog | +| 4.5.1 | 2024-02-02 | [31701](https://github.com/airbytehq/airbyte/pull/31701) | Add `region` support | +| 4.5.0 | 2024-02-01 | [34591](https://github.com/airbytehq/airbyte/pull/34591) | Run full refresh syncs concurrently | +| 4.4.1 | 2024-01-30 | [34665](https://github.com/airbytehq/airbyte/pull/34665) | Pin moto & CDK version | +| 4.4.0 | 2024-01-12 | [33818](https://github.com/airbytehq/airbyte/pull/33818) | Add IAM Role Authentication | +| 4.3.1 | 2024-01-04 | [33937](https://github.com/airbytehq/airbyte/pull/33937) | Prepare for airbyte-lib | +| 4.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | | 4.2.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 4.2.3 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 4.2.2 | 2023-11-20 | [32677](https://github.com/airbytehq/airbyte/pull/32677) | Only read files with ".zip" extension as zipped files | @@ -333,4 +353,4 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | 0.1.3 | 2021-08-04 | [5197](https://github.com/airbytehq/airbyte/pull/5197) | Fixed bug where sync could hang indefinitely on schema inference | | 0.1.2 | 2021-08-02 | [5135](https://github.com/airbytehq/airbyte/pull/5135) | Fixed bug in spec so it displays in UI correctly | | 0.1.1 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990/commits/ff5f70662c5f84eabc03526cddfcc9d73c58c0f4) | Fixed documentation url in source definition | -| 0.1.0 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990) | Created S3 source connector | \ No newline at end of file +| 0.1.0 | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990) | Created S3 source connector | diff --git a/docs/integrations/sources/salesforce.inapp.md b/docs/integrations/sources/salesforce.inapp.md deleted file mode 100644 index 591744bbb4b4..000000000000 --- a/docs/integrations/sources/salesforce.inapp.md +++ /dev/null @@ -1,88 +0,0 @@ -## Prerequisites - -- [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased -- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) - -- (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials - - -## Setup guide - -### Step 1: (Optional, Recommended) Create a read-only Salesforce user - -While you can set up the Salesforce connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the data Airbyte can read. - -To create a dedicated read only Salesforce user: - -1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account. -2. On the top right of the screen, click the gear icon and then click **Setup**. -3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**. -4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**. -5. Click **Save**. The Profiles page is displayed. Click **Edit**. -6. Scroll down to the **Standard Object Permissions** and **Custom Object Permissions** and enable the **Read** checkbox for objects that you want to replicate via Airbyte. -7. Scroll to the top and click **Save**. -8. On the left side, under Administration, click **Users** > **Users**. The All Users page is displayed. Click **New User**. -9. Fill out the required fields: - 1. For License, select **Salesforce**. - 2. For Profile, select **Airbyte Read Only User**. - 3. For Email, make sure to use an email address that you can access. -10. Click **Save**. -11. Copy the Username and keep it accessible. -12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible. - - - -### For Airbyte Open Source only: Obtain Salesforce OAuth credentials - -If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: - -- Client ID -- Client Secret -- Refresh Token - -To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications: - - 1. If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`. - 2. When running a curl command, run it with the `-L` option to follow any redirects. - 3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens. - - - -### Step 2: Set up the Salesforce connector in Airbyte - -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. -3. Find and select **Salesforce** from the list of available sources. -4. Enter a **Source name** of your choosing to help you identify this source. -5. To authenticate: - -**For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. - - -**For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. - -6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. -7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). -8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. -9. Click **Set up source** and wait for the tests to complete. - -### Supported Objects - -The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm). - -Airbyte fetches and handles all the possible and available streams dynamically based on: - -* If the authenticated Salesforce user has the Role and Permissions to read and fetch objects - -* If the object has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with. - -### Incremental Deletes - -The Salesforce connector supports retrieving deleted records from the Salesforce recycle bin. For the streams which support it, a deleted record will be marked with `isDeleted=true`. To find out more about how Salesforce manages records in the recycle bin, please visit their [docs](https://help.salesforce.com/s/articleView?id=sf.home_delete.htm&type=5). - -### Syncing Formula Fields - -The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream to pull in all the updated values of the field. - -For detailed information on supported sync modes, supported streams and performance considerations, refer to the -[full documentation for Salesforce](https://docs.airbyte.com/integrations/sources/salesforce). diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 34e2179c5aeb..54edec588742 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -1,6 +1,10 @@ # Salesforce -This page contains the setup guide and reference information for the Salesforce source connector. + + +This page contains the setup guide and reference information for the [Salesforce](https://www.salesforce.com/) source connector. + + ## Prerequisites @@ -43,7 +47,7 @@ To create a dedicated read only Salesforce user: -### For Airbyte Open Source only: Obtain Salesforce OAuth credentials +### For Airbyte Open Source: Obtain Salesforce OAuth credentials If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: @@ -61,22 +65,42 @@ To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmme ### Step 2: Set up the Salesforce connector in Airbyte -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. + + +**For Airbyte Cloud:** + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Salesforce** from the list of available sources. 4. Enter a **Source name** of your choosing to help you identify this source. 5. To authenticate: - **For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. - - +6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. +7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). +8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. +9. Click **Set up source** and wait for the tests to complete. + + + + + +**For Airbyte Open Source:** + +1. Navigate to your Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Salesforce** from the list of available sources. +4. Enter a **Source name** of your choosing to help you identify this source. +5. To authenticate: **For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. - 6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. 7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). 8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. 9. Click **Set up source** and wait for the tests to complete. + + + + ## Supported sync modes The Salesforce source connector supports the following sync modes: @@ -85,8 +109,9 @@ The Salesforce source connector supports the following sync modes: - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- (Recommended)[ Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) -## Supported Objects +## Supported Streams The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm). @@ -95,26 +120,35 @@ Airbyte allows exporting all available Salesforce objects dynamically based on: - If the authenticated Salesforce user has the Role and Permissions to read and fetch objects - If the salesforce object has the queryable property set to true. Airbyte can only fetch objects which are queryable. If you don’t see an object available via Airbyte, and it is queryable, check if it is API-accessible to the Salesforce user you authenticated with. -## Syncing Formula Fields -The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream and sync a historical backfill to pull in all the updated values of the field. +## Limitations & Troubleshooting -## Syncing Deletes - -The Salesforce connector supports retrieving deleted records from the Salesforce recycle bin. For the streams which support it, a deleted record will be marked with `isDeleted=true`. To find out more about how Salesforce manages records in the recycle bin, please visit their [docs](https://help.salesforce.com/s/articleView?id=sf.home_delete.htm&type=5). +
      + +Expand to see details about Salesforce connector limitations and troubleshooting. + +### Connector limitations -## Performance considerations +#### Rate limiting The Salesforce connector is restricted by Salesforce’s [Daily Rate Limits](https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_api.htm). The connector syncs data until it hits the daily rate limit, then ends the sync early with success status, and starts the next sync from where it left off. Note that picking up from where it ends will work only for incremental sync, which is why we recommend using the [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) sync mode. +#### A note on the BULK API vs REST API and their limitations +## Syncing Formula Fields + +The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream and sync a historical backfill to pull in all the updated values of the field. + +## Syncing Deletes + +The Salesforce connector supports retrieving deleted records from the Salesforce recycle bin. For the streams which support it, a deleted record will be marked with `isDeleted=true`. To find out more about how Salesforce manages records in the recycle bin, please visit their [docs](https://help.salesforce.com/s/articleView?id=sf.home_delete.htm&type=5). ## Usage of the BULK API vs REST API Salesforce allows extracting data using either the [BULK API](https://developer.salesforce.com/docs/atlas.en-us.236.0.api_asynch.meta/api_asynch/asynch_api_intro.htm) or [REST API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm). To achieve fast performance, Salesforce recommends using the BULK API for extracting larger amounts of data (more than 2,000 records). For this reason, the Salesforce connector uses the BULK API by default to extract any Salesforce objects, unless any of the following conditions are met: - The Salesforce object has columns which are unsupported by the BULK API, like columns with a `base64` or `complexvalue` type -- The Salesforce object is not supported by BULK API. In this case we sync the objects via the REST API which will occasionally cost more of your API quota. This includes the following objects: +- The Salesforce object is not supported by BULK API. In this case we sync the objects via the REST API which will occasionally cost more of your API quota. This includes the following objects: - AcceptedEventRelation - Attachment - CaseStatus @@ -142,18 +176,29 @@ More information on the differences between various Salesforce APIs can be found If you set the `Force Use Bulk API` option to `true`, the connector will ignore unsupported properties and sync Stream using BULK API. ::: +### Troubleshooting -## Tutorials +#### Tutorials Now that you have set up the Salesforce source connector, check out the following Salesforce tutorials: - [Replicate Salesforce data to BigQuery](https://airbyte.com/tutorials/replicate-salesforce-data-to-bigquery) - [Replicate Salesforce and Zendesk data to Keen for unified analytics](https://airbyte.com/tutorials/salesforce-zendesk-analytics) +* Check out common troubleshooting issues for the Salesforce source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| 2.3.2 | 2024-02-19 | [35421](https://github.com/airbytehq/airbyte/pull/35421) | Add Stream Slice Step option to specification | +| 2.3.1 | 2024-02-12 | [35147](https://github.com/airbytehq/airbyte/pull/35147) | Manage dependencies with Poetry. | +| 2.3.0 | 2023-12-15 | [33522](https://github.com/airbytehq/airbyte/pull/33522) | Sync streams concurrently in all sync modes | +| 2.2.2 | 2024-01-04 | [33936](https://github.com/airbytehq/airbyte/pull/33936) | Prepare for airbyte-lib | +| 2.2.1 | 2023-12-12 | [33342](https://github.com/airbytehq/airbyte/pull/33342) | Added new ContentDocumentLink stream | +| 2.2.0 | 2023-12-12 | [33350](https://github.com/airbytehq/airbyte/pull/33350) | Sync streams concurrently on full refresh | | 2.1.6 | 2023-11-28 | [32535](https://github.com/airbytehq/airbyte/pull/32535) | Run full refresh syncs concurrently | | 2.1.5 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | | 2.1.4 | 2023-08-17 | [29538](https://github.com/airbytehq/airbyte/pull/29538) | Fix encoding guess | @@ -230,4 +275,6 @@ Now that you have set up the Salesforce source connector, check out the followin | 0.1.3 | 2021-11-06 | [7592](https://github.com/airbytehq/airbyte/pull/7592) | Fix getting `anyType` fields using BULK API | | 0.1.2 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | | 0.1.1 | 2021-09-21 | [6209](https://github.com/airbytehq/airbyte/pull/6209) | Fix bug with pagination for BULK API | -| 0.1.0 | 2021-09-08 | [5619](https://github.com/airbytehq/airbyte/pull/5619) | Salesforce Aitbyte-Native Connector | \ No newline at end of file +| 0.1.0 | 2021-09-08 | [5619](https://github.com/airbytehq/airbyte/pull/5619) | Salesforce Aitbyte-Native Connector | + +       diff --git a/docs/integrations/sources/search-metrics.md b/docs/integrations/sources/search-metrics.md index 5d49b4bdb967..19ae17af87ba 100644 --- a/docs/integrations/sources/search-metrics.md +++ b/docs/integrations/sources/search-metrics.md @@ -1,5 +1,20 @@ # SearchMetrics +:::warning + +## Deprecation Notice + +The SearchMetrics source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + + ## Overview The SearchMetrics source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index 78bca8f45dcf..b3851579bbf2 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -84,6 +84,8 @@ The connector is restricted by normal Sendgrid [requests limitation](https://doc | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.3 | 2024-02-21 | [35181](https://github.com/airbytehq/airbyte/pull/35343) | Handle uncompressed contacts downloads. | +| 0.4.2 | 2024-02-12 | [35181](https://github.com/airbytehq/airbyte/pull/35181) | Manage dependencies with Poetry. | | 0.4.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.4.0 | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream | 0.3.1 | 2023-01-27 | [21939](https://github.com/airbytehq/airbyte/pull/21939) | Fix contacts missing records; Remove Messages stream | @@ -100,4 +102,4 @@ The connector is restricted by normal Sendgrid [requests limitation](https://doc | 0.2.7 | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910) | Add Single Sends Stats stream | | 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | - \ No newline at end of file + diff --git a/docs/integrations/sources/sendinblue.md b/docs/integrations/sources/sendinblue.md index 62d6be403eaa..0e56d9bbc2c2 100644 --- a/docs/integrations/sources/sendinblue.md +++ b/docs/integrations/sources/sendinblue.md @@ -2,20 +2,20 @@ ## Sync overview -This source can sync data from the [Sendinblue API](https://developers.sendinblue.com/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. +This source can sync data from the [Sendinblue API](https://developers.sendinblue.com/). ## This Source Supports the Following Streams -* contacts -* campaigns -* templates +* [contacts](https://developers.brevo.com/reference/getcontacts-1) *(Incremental Sync)* +* [campaigns](https://developers.brevo.com/reference/getemailcampaigns-1) +* [templates](https://developers.brevo.com/reference/getsmtptemplates) ### Features | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | -| Incremental Sync | No | | +| Incremental Sync | Yes | | ### Performance considerations @@ -31,4 +31,5 @@ Sendinblue APIs are under rate limits for the number of API calls allowed per AP | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.0 | 2022-11-01 | [#18771](https://github.com/airbytehq/airbyte/pull/18771) | 🎉 New Source: Sendinblue API [low-code CDK] | \ No newline at end of file +| 0.1.1 | 2022-08-31 | [#30022](https://github.com/airbytehq/airbyte/pull/30022) | ✨ Source SendInBlue: Add incremental sync to contacts stream | +| 0.1.0 | 2022-11-01 | [#18771](https://github.com/airbytehq/airbyte/pull/18771) | 🎉 New Source: Sendinblue API [low-code CDK] | diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index cb6749812ff0..b51fd4f341d3 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -29,7 +29,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air ## Supported Streams -- [Events](https://docs.sentry.io/api/events/list-a-projects-events/) +- [Events](https://docs.sentry.io/api/events/list-a-projects-error-events/) - [Issues](https://docs.sentry.io/api/events/list-a-projects-issues/) - [Projects](https://docs.sentry.io/api/projects/list-your-projects/) - [Releases](https://docs.sentry.io/api/releases/list-an-organizations-releases/) @@ -47,6 +47,8 @@ The Sentry source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.4.1 | 2024-02-12 | [35145](https://github.com/airbytehq/airbyte/pull/35145) | Manage dependencies with Poetry. | +| 0.4.0 | 2024-01-05 | [32957](https://github.com/airbytehq/airbyte/pull/32957) | Added undeclared fields to schema, Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.0 | 2023-09-05 | [30192](https://github.com/airbytehq/airbyte/pull/30192) | Added undeclared fields to schema | | 0.2.4 | 2023-08-14 | [29401](https://github.com/airbytehq/airbyte/pull/29401) | Fix `null` value in stream state | | 0.2.3 | 2023-08-03 | [29023](https://github.com/airbytehq/airbyte/pull/29023) | Add incremental for `issues` stream | diff --git a/docs/integrations/sources/sftp.md b/docs/integrations/sources/sftp.md index 8d6d84e942d9..13cd4a0979fc 100644 --- a/docs/integrations/sources/sftp.md +++ b/docs/integrations/sources/sftp.md @@ -107,6 +107,9 @@ More formats \(e.g. Apache Avro\) will be supported in the future. ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------| +| 0.2.2 | 2024-02-13 | [35221](https://github.com/airbytehq/airbyte/pull/35221) | Adopt CDK 0.20.4 | +| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.2.0 | 2024-01-15 | [34265](https://github.com/airbytehq/airbyte/pull/34265) | Remove LEGACY state flag | | 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.0 | 2021-24-05 | | Initial version | diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index a616ff2c502f..5a5aa4b43b91 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -111,6 +111,7 @@ This source can sync data for the [Shopify REST API](https://shopify.dev/api/adm - [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts#top) - [Articles](https://shopify.dev/api/admin-rest/2022-01/resources/article) +- [Balance Transactions](https://shopify.dev/docs/api/admin-rest/2023-10/resources/transactions) - [Blogs](https://shopify.dev/api/admin-rest/2022-01/resources/blog) - [Collects](https://shopify.dev/api/admin-rest/2022-01/resources/collect#top) - [Collections](https://shopify.dev/api/admin-rest/2022-01/resources/collection) @@ -210,7 +211,11 @@ If a child stream is synced independently of its parent stream, a full sync will | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------ | -| 1.1.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.8 | 2024-02-12 | [35166](https://github.com/airbytehq/airbyte/pull/35166) | Manage dependencies with Poetry. | +| 1.1.7 | 2024-01-19 | [33804](https://github.com/airbytehq/airbyte/pull/33804) | Updated documentation with list of all supported streams | +| 1.1.6 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 1.1.5 | 2023-12-28 | [33827](https://github.com/airbytehq/airbyte/pull/33827) | Fix GraphQL query | +| 1.1.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 1.1.3 | 2023-10-17 | [31500](https://github.com/airbytehq/airbyte/pull/31500) | Fixed the issue caused by the `missing access token` while setup the new source and not yet authenticated | | 1.1.2 | 2023-10-13 | [31381](https://github.com/airbytehq/airbyte/pull/31381) | Fixed the issue caused by the `state` presence while fetching the `deleted events` with pagination | | 1.1.1 | 2023-09-18 | [30560](https://github.com/airbytehq/airbyte/pull/30560) | Performance testing - include socat binary in docker image | diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 6baf8f9953a2..a5a99979b523 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -118,14 +118,13 @@ The Slack source connector supports the following [sync modes](https://docs.airb ## Supported Streams +For most of the streams, the Slack source connector uses the [Conversations API](https://api.slack.com/docs/conversations-api) under the hood. + * [Channels \(Conversations\)](https://api.slack.com/methods/conversations.list) * [Channel Members \(Conversation Members\)](https://api.slack.com/methods/conversations.members) * [Messages \(Conversation History\)](https://api.slack.com/methods/conversations.history) It will only replicate messages from non-archive, public channels that the Slack App is a member of. * [Users](https://api.slack.com/methods/users.list) * [Threads \(Conversation Replies\)](https://api.slack.com/methods/conversations.replies) -* [User Groups](https://api.slack.com/methods/usergroups.list) -* [Files](https://api.slack.com/methods/files.list) -* [Remote Files](https://api.slack.com/methods/files.remote.list) ## Performance considerations @@ -164,9 +163,13 @@ Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------| -| 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.9 | 2024-02-12 | [35157](https://github.com/airbytehq/airbyte/pull/35157) | Manage dependencies with Poetry. | +| 0.3.8 | 2024-02-09 | [35131](https://github.com/airbytehq/airbyte/pull/35131) | Fixed the issue when `schema discovery` fails with `502` due to the platform timeout | +| 0.3.7 | 2024-01-10 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | +| 0.3.6 | 2023-11-21 | [32707](https://github.com/airbytehq/airbyte/pull/32707) | Threads: do not use client-side record filtering | +| 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.4 | 2023-10-06 | [31134](https://github.com/airbytehq/airbyte/pull/31134) | Update CDK and remove non iterable return from records | -| 0.3.3 | 2023-09-28 | [30580](https://github.com/airbytehq/airbyte/pull/30580) | Add `bot_id` field to threads schema | +| 0.3.3 | 2023-09-28 | [30580](https://github.com/airbytehq/airbyte/pull/30580) | Add `bot_id` field to threads schema | | 0.3.2 | 2023-09-20 | [30613](https://github.com/airbytehq/airbyte/pull/30613) | Set default value for channel_filters during discover | | 0.3.1 | 2023-09-19 | [30570](https://github.com/airbytehq/airbyte/pull/30570) | Use default availability strategy | | 0.3.0 | 2023-09-18 | [30521](https://github.com/airbytehq/airbyte/pull/30521) | Add unexpected fields to streams `channel_messages`, `channels`, `threads`, `users` | @@ -192,4 +195,4 @@ Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). | 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | | 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector | - \ No newline at end of file + diff --git a/docs/integrations/sources/smartsheets.md b/docs/integrations/sources/smartsheets.md index 4d9e62d85997..52359b70e88e 100644 --- a/docs/integrations/sources/smartsheets.md +++ b/docs/integrations/sources/smartsheets.md @@ -110,6 +110,7 @@ The remaining column datatypes supported by Smartsheets are more complex types ( | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------| +| 1.1.2 | 2024-01-08 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | | 1.1.1 | 2023-06-06 | [27096](https://github.com/airbytehq/airbyte/pull/27096) | Fix error when optional metadata fields are not set | | 1.1.0 | 2023-06-02 | [22382](https://github.com/airbytehq/airbyte/pull/22382) | Add support for ingesting metadata fields | | 1.0.2 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Fix dependencies conflict | diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index 5f12ba912116..a11dd398f260 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -113,6 +113,7 @@ Snapchat Marketing API has limitations to 1000 items per page. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------| +| 0.3.2 | 2024-02-12 | [35171](https://github.com/airbytehq/airbyte/pull/35171) | Manage dependencies with Poetry. | | 0.3.0 | 2023-05-22 | [26358](https://github.com/airbytehq/airbyte/pull/26358) | Remove deprecated authSpecification in favour of advancedAuth | | 0.2.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema | | 0.1.16 | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema | @@ -130,4 +131,4 @@ Snapchat Marketing API has limitations to 1000 items per page. | 0.1.3 | 2021-11-10 | [7811](https://github.com/airbytehq/airbyte/pull/7811) | Add oauth2.0, fix stream_state | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-07-29 | [5072](https://github.com/airbytehq/airbyte/pull/5072) | Fix bug with incorrect stream\_state value | -| 0.1.0 | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843) | Initial release supporting the Snapchat Marketing API | \ No newline at end of file +| 0.1.0 | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843) | Initial release supporting the Snapchat Marketing API | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 5556fad94bc2..7032ff83d72a 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -125,43 +125,46 @@ To read more please check official [Snowflake documentation](https://docs.snowfl ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.2 | 2023-10-20 | [31613](https://github.com/airbytehq/airbyte/pull/31613) | Fixed handling of TIMESTAMP_TZ columns. upgrade | -| 0.2.1 | 2023-10-11 | [31252](https://github.com/airbytehq/airbyte/pull/31252) | Snowflake JDBC version upgrade | -| 0.2.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | -| 0.1.36 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | -| 0.1.35 | 2023-06-14 | [27335](https://github.com/airbytehq/airbyte/pull/27335) | Remove noisy debug logs | -| 0.1.34 | 2023-03-30 | [24693](https://github.com/airbytehq/airbyte/pull/24693) | Fix failure with TIMESTAMP_WITH_TIMEZONE column being used as cursor | -| 0.1.33 | 2023-03-29 | [24667](https://github.com/airbytehq/airbyte/pull/24667) | Fix bug which wont allow TIMESTAMP_WITH_TIMEZONE column to be used as a cursor | -| 0.1.32 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.31 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.1.30 | 2023-02-21 | [22358](https://github.com/airbytehq/airbyte/pull/22358) | Improved handling of big integer cursor type values. | -| 0.1.29 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources. | -| 0.1.28 | 2023-01-06 | [20465](https://github.com/airbytehq/airbyte/pull/20465) | Improve the schema config field to only discover tables from the specified scehma and make the field optional | -| 0.1.27 | 2022-12-14 | [20407](https://github.com/airbytehq/airbyte/pull/20407) | Fix an issue with integer values converted to floats during replication | -| 0.1.26 | 2022-11-10 | [19314](https://github.com/airbytehq/airbyte/pull/19314) | Set application id in JDBC URL params based on OSS/Cloud environment | -| 0.1.25 | 2022-11-10 | [15535](https://github.com/airbytehq/airbyte/pull/15535) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.24 | 2022-09-26 | [17144](https://github.com/airbytehq/airbyte/pull/17144) | Fixed bug with incorrect date-time datatypes handling | -| 0.1.23 | 2022-09-26 | [17116](https://github.com/airbytehq/airbyte/pull/17116) | added connection string identifier | -| 0.1.22 | 2022-09-21 | [16766](https://github.com/airbytehq/airbyte/pull/16766) | Update JDBC Driver version to 3.13.22 | -| 0.1.21 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | -| 0.1.20 | 2022-09-01 | [16258](https://github.com/airbytehq/airbyte/pull/16258) | Emit state messages more frequently | -| 0.1.19 | 2022-08-19 | [15797](https://github.com/airbytehq/airbyte/pull/15797) | Allow using role during oauth | -| 0.1.18 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.17 | 2022-08-09 | [15314](https://github.com/airbytehq/airbyte/pull/15314) | Discover integer columns as integers rather than floats | -| 0.1.16 | 2022-08-04 | [15314](https://github.com/airbytehq/airbyte/pull/15314) | (broken, do not use) Discover integer columns as integers rather than floats | -| 0.1.15 | 2022-07-22 | [14828](https://github.com/airbytehq/airbyte/pull/14828) | Source Snowflake: Source/Destination doesn't respect DATE data type | -| 0.1.14 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.11 | 2022-04-27 | [10953](https://github.com/airbytehq/airbyte/pull/10953) | Implement OAuth flow | -| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.6 | 2022-01-25 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | -| 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive | -| 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description | -| 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver | -| 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types | -| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.1 | 2024-02-13 | [35220](https://github.com/airbytehq/airbyte/pull/35220) | Adopt CDK 0.20.4 | +| 0.3.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.3.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | +| 0.2.2 | 2023-10-20 | [31613](https://github.com/airbytehq/airbyte/pull/31613) | Fixed handling of TIMESTAMP_TZ columns. upgrade | +| 0.2.1 | 2023-10-11 | [31252](https://github.com/airbytehq/airbyte/pull/31252) | Snowflake JDBC version upgrade | +| 0.2.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | +| 0.1.36 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | +| 0.1.35 | 2023-06-14 | [27335](https://github.com/airbytehq/airbyte/pull/27335) | Remove noisy debug logs | +| 0.1.34 | 2023-03-30 | [24693](https://github.com/airbytehq/airbyte/pull/24693) | Fix failure with TIMESTAMP_WITH_TIMEZONE column being used as cursor | +| 0.1.33 | 2023-03-29 | [24667](https://github.com/airbytehq/airbyte/pull/24667) | Fix bug which wont allow TIMESTAMP_WITH_TIMEZONE column to be used as a cursor | +| 0.1.32 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.31 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.30 | 2023-02-21 | [22358](https://github.com/airbytehq/airbyte/pull/22358) | Improved handling of big integer cursor type values. | +| 0.1.29 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources. | +| 0.1.28 | 2023-01-06 | [20465](https://github.com/airbytehq/airbyte/pull/20465) | Improve the schema config field to only discover tables from the specified scehma and make the field optional | +| 0.1.27 | 2022-12-14 | [20407](https://github.com/airbytehq/airbyte/pull/20407) | Fix an issue with integer values converted to floats during replication | +| 0.1.26 | 2022-11-10 | [19314](https://github.com/airbytehq/airbyte/pull/19314) | Set application id in JDBC URL params based on OSS/Cloud environment | +| 0.1.25 | 2022-11-10 | [15535](https://github.com/airbytehq/airbyte/pull/15535) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | +| 0.1.24 | 2022-09-26 | [17144](https://github.com/airbytehq/airbyte/pull/17144) | Fixed bug with incorrect date-time datatypes handling | +| 0.1.23 | 2022-09-26 | [17116](https://github.com/airbytehq/airbyte/pull/17116) | added connection string identifier | +| 0.1.22 | 2022-09-21 | [16766](https://github.com/airbytehq/airbyte/pull/16766) | Update JDBC Driver version to 3.13.22 | +| 0.1.21 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | +| 0.1.20 | 2022-09-01 | [16258](https://github.com/airbytehq/airbyte/pull/16258) | Emit state messages more frequently | +| 0.1.19 | 2022-08-19 | [15797](https://github.com/airbytehq/airbyte/pull/15797) | Allow using role during oauth | +| 0.1.18 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.17 | 2022-08-09 | [15314](https://github.com/airbytehq/airbyte/pull/15314) | Discover integer columns as integers rather than floats | +| 0.1.16 | 2022-08-04 | [15314](https://github.com/airbytehq/airbyte/pull/15314) | (broken, do not use) Discover integer columns as integers rather than floats | +| 0.1.15 | 2022-07-22 | [14828](https://github.com/airbytehq/airbyte/pull/14828) | Source Snowflake: Source/Destination doesn't respect DATE data type | +| 0.1.14 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.11 | 2022-04-27 | [10953](https://github.com/airbytehq/airbyte/pull/10953) | Implement OAuth flow | +| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.6 | 2022-01-25 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | +| 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive | +| 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description | +| 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver | +| 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types | +| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 0f76e3b166a2..2e5bb5e957c3 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -155,6 +155,13 @@ However, not all the entities are supported by the Events API, so the Stripe con - `Shipping Rates` On the other hand, the following streams use the `updated` field value as a cursor: + +:::note + +`updated` is an artificial cursor field introduced by Airbyte for the Incremental sync option. + +::: + - `Application Fees` - `Application Fee Refunds` - `Authorizations` @@ -214,93 +221,101 @@ Each record is marked with `is_deleted` flag when the appropriate event happens ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 5.1.0 | 2023-12-11 | [32908](https://github.com/airbytehq/airbyte/pull/32908/) | Read full refresh streams concurrently | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 5.2.4 | 2024-02-12 | [35137](https://github.com/airbytehq/airbyte/pull/35137) | Fix license in `pyproject.toml` | +| 5.2.3 | 2024-02-09 | [35068](https://github.com/airbytehq/airbyte/pull/35068) | Manage dependencies with Poetry. | +| 5.2.2 | 2024-01-31 | [34619](https://github.com/airbytehq/airbyte/pull/34619) | Events stream concurrent on incremental syncs | +| 5.2.1 | 2024-01-18 | [34495](https://github.com/airbytehq/airbyte/pull/34495) | Fix deadlock issue | +| 5.2.0 | 2024-01-18 | [34347](https://github.com/airbytehq/airbyte/pull//34347) | Add new fields invoices and subscription streams. Upgrade the CDK for better memory usage. | +| 5.1.3 | 2023-12-18 | [33306](https://github.com/airbytehq/airbyte/pull/33306/) | Adding integration tests | +| 5.1.2 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 5.1.1 | 2024-01-04 | [33926](https://github.com/airbytehq/airbyte/pull/33926/) | Update endpoint for `bank_accounts` stream | +| 5.1.0 | 2023-12-11 | [32908](https://github.com/airbytehq/airbyte/pull/32908/) | Read full refresh streams concurrently | | 5.0.2 | 2023-12-01 | [33038](https://github.com/airbytehq/airbyte/pull/33038) | Add stream slice logging for SubStream | -| 5.0.1 | 2023-11-17 | [32638](https://github.com/airbytehq/airbyte/pull/32638/) | Availability stretegy: check availability of both endpoints (if applicable) - common API + events API | -| 5.0.0 | 2023-11-16 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | -| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | -| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | -| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | -| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | -| 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | -| 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | -| 4.4.0 | 2023-10-04 | [31046](https://github.com/airbytehq/airbyte/pull/31046) | Added margins field to invoice_line_items stream. | -| 4.3.1 | 2023-09-27 | [30800](https://github.com/airbytehq/airbyte/pull/30800) | Handle permission issues a non breaking | -| 4.3.0 | 2023-09-26 | [30752](https://github.com/airbytehq/airbyte/pull/30752) | Do not sync upcoming invoices, extend stream schemas | -| 4.2.0 | 2023-09-21 | [30660](https://github.com/airbytehq/airbyte/pull/30660) | Fix updated state for the incremental syncs | -| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | -| 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | -| 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | -| 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update | -| 3.17.4 | 2023-08-15 | [29425](https://github.com/airbytehq/airbyte/pull/29425) | Revert 3.17.3 | -| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | -| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | -| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | -| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | -| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | -| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | -| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | -| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | -| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | -| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | -| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | -| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | -| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | -| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | -| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | -| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | -| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | -| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | -| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | -| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | -| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | -| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | -| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | -| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | -| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | -| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | -| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | -| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | -| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | -| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | -| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | -| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | -| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | -| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | -| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | -| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | -| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | -| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | -| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | -| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | -| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | -| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | -| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | -| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | -| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | -| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | -| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | -| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | -| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | -| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | -| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | -| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | -| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | -| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | -| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | -| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | -| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | -| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | -| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | -| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | -| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | -| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | -| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | -| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | +| 5.0.1 | 2023-11-17 | [32638](https://github.com/airbytehq/airbyte/pull/32638/) | Availability stretegy: check availability of both endpoints (if applicable) - common API + events API | +| 5.0.0 | 2023-11-16 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | +| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | +| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | +| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | +| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | +| 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | +| 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | +| 4.4.0 | 2023-10-04 | [31046](https://github.com/airbytehq/airbyte/pull/31046) | Added margins field to invoice_line_items stream. | +| 4.3.1 | 2023-09-27 | [30800](https://github.com/airbytehq/airbyte/pull/30800) | Handle permission issues a non breaking | +| 4.3.0 | 2023-09-26 | [30752](https://github.com/airbytehq/airbyte/pull/30752) | Do not sync upcoming invoices, extend stream schemas | +| 4.2.0 | 2023-09-21 | [30660](https://github.com/airbytehq/airbyte/pull/30660) | Fix updated state for the incremental syncs | +| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | +| 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | +| 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | +| 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update | +| 3.17.4 | 2023-08-15 | [29425](https://github.com/airbytehq/airbyte/pull/29425) | Revert 3.17.3 | +| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | +| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | +| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | +| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | +| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | +| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | +| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | +| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | +| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | +| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | +| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | +| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | +| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | +| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | +| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | +| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | +| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | +| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | +| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | +| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | +| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | +| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | +| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | +| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | +| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | +| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | +| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | +| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | +| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | +| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | +| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | +| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | +| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | +| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | +| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | +| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | +| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | +| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | +| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | +| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | +| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | +| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | +| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | +| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | +| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | +| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | +| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | +| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | +| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | +| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | +| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | +| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | +| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | +| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | +| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | +| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | +| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | +| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | +| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | +| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | +| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | +| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | +| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | +| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index 92dbe0162518..561df7c2dd40 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -66,6 +66,7 @@ To cover more data from this source we use caching. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------| +| 0.2.4 | 2024-02-12 | [35168](https://github.com/airbytehq/airbyte/pull/35168) | Manage dependencies with Poetry. | | 0.2.3 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.2.2 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Fix dependencies conflict | | 0.2.1 | 2023-04-27 | [25109](https://github.com/airbytehq/airbyte/pull/25109) | Fix add missing params to stream `SurveyResponses` | @@ -86,4 +87,4 @@ To cover more data from this source we use caching. | 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | | 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | | 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | -| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | \ No newline at end of file +| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | diff --git a/docs/integrations/sources/talkdesk-explore.md b/docs/integrations/sources/talkdesk-explore.md index b9c8ba54d1c6..74b3f89df4ab 100644 --- a/docs/integrations/sources/talkdesk-explore.md +++ b/docs/integrations/sources/talkdesk-explore.md @@ -1,5 +1,20 @@ # Talkdesk Explore +:::warning + +## Deprecation Notice + +The Talkdesk Explore source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + + ## Overview Talkdesk is a software for contact center operations. diff --git a/docs/integrations/sources/teradata.md b/docs/integrations/sources/teradata.md index a8c83ab16789..39dc85b0ad9e 100644 --- a/docs/integrations/sources/teradata.md +++ b/docs/integrations/sources/teradata.md @@ -63,4 +63,7 @@ You need a Teradata user which has read permissions on the database | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------|:----------------------------| -| 0.1.0 | 2022-03-27 | https://github.com/airbytehq/airbyte/pull/24221 | New Source Teradata Vantage | \ No newline at end of file +| 0.2.2 | 2024-02-13 | [35219](https://github.com/airbytehq/airbyte/pull/35219) | Adopt CDK 0.20.4 | +| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.2.0 | 2023-12-18 | https://github.com/airbytehq/airbyte/pull/33485 | Remove LEGACY state | +| 0.1.0 | 2022-03-27 | https://github.com/airbytehq/airbyte/pull/24221 | New Source Teradata Vantage | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 3007c8b0de44..197673c5e8cb 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -126,18 +126,21 @@ Now that you have set up the TiDB source connector, check out the following TiDB ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :----------- | ------- | -| 0.2.5 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | -| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :----------- |-------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.2 | 2024-02-13 | [35218](https://github.com/airbytehq/airbyte/pull/35218) | Adopt CDK 0.20.4 | +| 0.3.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version | +| 0.3.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.2.5 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | +| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.1.5 | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec | -| 0.1.4 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.1.3 | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder | -| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | +| 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.1.5 | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec | +| 0.1.4 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.1.3 | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 4a6b430a52a2..574c081d1421 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -123,6 +123,7 @@ The connector is restricted by [requests limitation](https://business-api.tiktok | Version | Date | Pull Request | Subject | |:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------| +| 3.9.3 | 2024-02-12 | [35161](https://github.com/airbytehq/airbyte/pull/35161) | Manage dependencies with Poetry. | | 3.9.2 | 2023-11-02 | [32091](https://github.com/airbytehq/airbyte/pull/32091) | Fix incremental syncs; update docs; fix field type of `preview_url_expire_time` to `date-time`. | | 3.9.1 | 2023-10-25 | [31812](https://github.com/airbytehq/airbyte/pull/31812) | Update `support level` in `metadata`, removed duplicated `tracking_pixel_id` field from `Ads` stream schema | | 3.9.0 | 2023-10-23 | [31623](https://github.com/airbytehq/airbyte/pull/31623) | Add AdsAudienceReportsByProvince stream and expand base report metrics | diff --git a/docs/integrations/sources/todoist.md b/docs/integrations/sources/todoist.md index 77935691155c..34578169dabc 100644 --- a/docs/integrations/sources/todoist.md +++ b/docs/integrations/sources/todoist.md @@ -44,4 +44,5 @@ List of available streams: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------| -| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | +| 0.2.0 | 2023-12-19 | [32690](https://github.com/airbytehq/airbyte/pull/32690) | Migrate to low-code | +| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 5b7a7679cb92..c337bf8ed193 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -95,6 +95,7 @@ For more information, see [the Twilio docs for rate limitations](https://support | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| +| 0.10.2 | 2024-02-12 | [35153](https://github.com/airbytehq/airbyte/pull/35153) | Manage dependencies with Poetry. | | 0.10.1 | 2023-11-21 | [32718](https://github.com/airbytehq/airbyte/pull/32718) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.10.0 | 2023-07-28 | [27323](https://github.com/airbytehq/airbyte/pull/27323) | Add new stream `Step` | | 0.9.0 | 2023-06-27 | [27221](https://github.com/airbytehq/airbyte/pull/27221) | Add new stream `UserConversations` with parent `Users` | @@ -121,4 +122,4 @@ For more information, see [the Twilio docs for rate limitations](https://support | 0.1.3 | 2022-04-20 | [12183](https://github.com/airbytehq/airbyte/pull/12183) | Add new subresource on the call stream + declare a valid primary key for conference_participants stream | | 0.1.2 | 2021-12-23 | [9092](https://github.com/airbytehq/airbyte/pull/9092) | Correct specification doc URL | | 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema | -| 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented | \ No newline at end of file +| 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented | diff --git a/docs/integrations/sources/typeform.md b/docs/integrations/sources/typeform.md index 5b5cc950ee66..826e822190f0 100644 --- a/docs/integrations/sources/typeform.md +++ b/docs/integrations/sources/typeform.md @@ -90,6 +90,10 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 1.2.5 | 2024-02-12 | [35152](https://github.com/airbytehq/airbyte/pull/35152) | Manage dependencies with Poetry. | +| 1.2.4 | 2024-01-24 | [34484](https://github.com/airbytehq/airbyte/pull/34484) | Fix pagination stop condition | +| 1.2.3 | 2024-01-11 | [34145](https://github.com/airbytehq/airbyte/pull/34145) | prepare for airbyte-lib | +| 1.2.2 | 2023-12-12 | [33345](https://github.com/airbytehq/airbyte/pull/33345) | Fix single use refresh token authentication | | 1.2.1 | 2023-12-04 | [32775](https://github.com/airbytehq/airbyte/pull/32775) | Add 499 status code handling | | 1.2.0 | 2023-11-29 | [32745](https://github.com/airbytehq/airbyte/pull/32745) | Add `response_type` field to `responses` schema | | 1.1.2 | 2023-10-27 | [31914](https://github.com/airbytehq/airbyte/pull/31914) | Fix pagination for stream Responses | diff --git a/docs/integrations/sources/us-census.md b/docs/integrations/sources/us-census.md index 2242f3fceda5..374c9ac16a19 100644 --- a/docs/integrations/sources/us-census.md +++ b/docs/integrations/sources/us-census.md @@ -36,8 +36,9 @@ In addition, to understand how to configure the dataset path and query parameter ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :------------------------------------------------ | -| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.3 | 2024-01-03 | [33890](https://github.com/airbytehq/airbyte/pull/33890) | Allow additional properties in connector spec | +| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | diff --git a/docs/integrations/sources/webflow.md b/docs/integrations/sources/webflow.md index 7a47e7158af9..12b971cade38 100644 --- a/docs/integrations/sources/webflow.md +++ b/docs/integrations/sources/webflow.md @@ -12,7 +12,7 @@ This connector dynamically figures out which collections are available, creates # Webflow credentials -You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). +You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). The Webflow connector uses the Webflow API v1 and therefore will require a legacy v1 API key. Once you have the `API Key`/`API token`, you can confirm a [list of available sites](https://developers.webflow.com/#sites) and get their `_id` by executing the following: @@ -28,8 +28,8 @@ Which should respond with something similar to: [{"_id":"","createdOn":"2021-03-26T15:46:04.032Z","name":"Airbyte","shortName":"airbyte-dev","lastPublished":"2022-06-09T12:55:52.533Z","previewUrl":"https://screenshots.webflow.com/sites/","timezone":"America/Los_Angeles","database":""}] ``` -You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. - +You will need to provide the `Site ID` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. + # Related tutorial If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). @@ -38,8 +38,7 @@ If you are interested in learning more about the Webflow API and implementation | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------- | -| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow add ids to streams | -| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | -| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | - - \ No newline at end of file +| 0.1.3 | 2022-12-11 | [33315](https://github.com/airbytehq/airbyte/pull/33315) | Updates CDK to latest version and adds additional properties to schema | +| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow added IDs to streams | +| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Updates Spec Documentation URL | +| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | \ No newline at end of file diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index 738378e3e68e..1e049713fcf7 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -104,6 +104,7 @@ The connector is restricted by Xero [API rate limits](https://developer.xero.com | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------| +| 0.2.5 | 2024-01-11 | [34154](https://github.com/airbytehq/airbyte/pull/34154) | prepare for airbyte-lib | | 0.2.4 | 2023-11-24 | [32837](https://github.com/airbytehq/airbyte/pull/32837) | Handle 403 error | | 0.2.3 | 2023-06-19 | [27471](https://github.com/airbytehq/airbyte/pull/27471) | Update CDK to 0.40 | | 0.2.2 | 2023-06-06 | [27007](https://github.com/airbytehq/airbyte/pull/27007) | Update CDK | diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index d1cd8ac9adb7..1baf88415519 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,8 +80,9 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | -| 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | -| 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | +| 0.2.2 | 2024-02-12 | [35185](https://github.com/airbytehq/airbyte/pull/35185) | Manage dependencies with Poetry. | +| 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | +| 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | | 0.1.14 | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream | | 0.1.13 | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification | | 0.1.12 | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None` | @@ -96,4 +97,4 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | 0.1.3 | 2021-10-21 | [7210](https://github.com/airbytehq/airbyte/pull/7210) | Chats stream is only getting data from first page | | 0.1.2 | 2021-08-17 | [5476](https://github.com/airbytehq/airbyte/pull/5476) | Correct field unread to boolean type | | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.0 | 2021-05-03 | [3088](https://github.com/airbytehq/airbyte/pull/3088) | Initial release | \ No newline at end of file +| 0.1.0 | 2021-05-03 | [3088](https://github.com/airbytehq/airbyte/pull/3088) | Initial release | diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index d38d7733a103..c955743f3613 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -126,7 +126,7 @@ The Zendesk Support source connector supports the following streams: The Zendesk Support connector fetches deleted records in the following streams: | Stream | Deletion indicator field | -|:-------------------------|:-------------------------| +| :----------------------- | :----------------------- | | **Brands** | `is_deleted` | | **Groups** | `deleted` | | **Organizations** | `deleted_at` | @@ -157,7 +157,13 @@ The Zendesk connector ideally should not run into Zendesk API limitations under ## Changelog | Version | Date | Pull Request | Subject | -|:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| :------- | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.2.8 | 2024-02-09 | [35083](https://github.com/airbytehq/airbyte/pull/35083) | Manage dependencies with Poetry. | +| `2.2.7` | 2024-02-05 | [34840](https://github.com/airbytehq/airbyte/pull/34840) | Fix missing fields in schema | +| `2.2.6` | 2024-01-11 | [34064](https://github.com/airbytehq/airbyte/pull/34064) | Skip 504 Error for stream `Ticket Audits` | +| `2.2.5` | 2024-01-08 | [34010](https://github.com/airbytehq/airbyte/pull/34010) | prepare for airbyte-lib | +| `2.2.4` | 2023-12-20 | [33680](https://github.com/airbytehq/airbyte/pull/33680) | Fix pagination issue for streams related to incremental export sync | +| `2.2.3` | 2023-12-14 | [33435](https://github.com/airbytehq/airbyte/pull/33435) | Fix 504 Error for stream Ticket Audits | | `2.2.2` | 2023-12-01 | [33012](https://github.com/airbytehq/airbyte/pull/33012) | Increase number of retries for backoff policy to 10 | | `2.2.1` | 2023-11-10 | [32440](https://github.com/airbytehq/airbyte/pull/32440) | Made refactoring to improve code maintainability | | `2.2.0` | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extended the `CustomRoles` stream schema | @@ -238,4 +244,4 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | `0.1.1` | 2021-09-02 | [5787](https://github.com/airbytehq/airbyte/pull/5787) | Fixed incremental logic for the ticket_comments stream | | `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector | - \ No newline at end of file + diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md index 16b202874c4e..6bb00f8d4f0a 100644 --- a/docs/integrations/sources/zendesk-talk.md +++ b/docs/integrations/sources/zendesk-talk.md @@ -70,16 +70,17 @@ The Zendesk connector should not run into Zendesk API limitations under normal u | `array` | `array` | | | `object` | `object` | | - ## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :----- |:----------------------------------| -| `0.1.9` | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| `0.1.8` | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| `0.1.7` | 2023-02-10 | [22815](https://github.com/airbytehq/airbyte/pull/22815) | Specified date formatting in specification | -| `0.1.6` | 2023-01-27 | [22028](https://github.com/airbytehq/airbyte/pull/22028) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| `0.1.5` | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version | -| `0.1.4` | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | -| `0.1.3` | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------| +| 0.1.12 | 2024-02-12 | [35156](https://github.com/airbytehq/airbyte/pull/35156) | Manage dependencies with Poetry. | +| 0.1.11 | 2024-01-12 | [34204](https://github.com/airbytehq/airbyte/pull/34204) | prepare for airbyte-lib | +| 0.1.10 | 2023-12-04 | [33030](https://github.com/airbytehq/airbyte/pull/33030) | Base image migration: remove Dockerfile and use python-connector-base image | +| 0.1.9 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 0.1.8 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 0.1.7 | 2023-02-10 | [22815](https://github.com/airbytehq/airbyte/pull/22815) | Specified date formatting in specification | +| 0.1.6 | 2023-01-27 | [22028](https://github.com/airbytehq/airbyte/pull/22028) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.5 | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version | +| 0.1.4 | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | +| 0.1.3 | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | diff --git a/docs/integrations/sources/zuora.md b/docs/integrations/sources/zuora.md index 86ce852107fc..b0c5f019d967 100644 --- a/docs/integrations/sources/zuora.md +++ b/docs/integrations/sources/zuora.md @@ -1,5 +1,19 @@ # Zuora +:::warning + +## Deprecation Notice + +The Zuora source connector is scheduled for deprecation on March 5th, 2024 due to incompatibility with upcoming platform updates as we prepare to launch Airbyte 1.0. This means it will no longer be supported or available for use in Airbyte. + +This connector does not support new per-stream features which are vital for ensuring data integrity in Airbyte's synchronization processes. Without these capabilities, we cannot enforce our standards of reliability and correctness for data syncing operations. + +### Recommended Actions + +Users who still wish to sync data from this connector are advised to explore creating a custom connector as an alternative to continue their data synchronization needs. For guidance, please visit our [Custom Connector documentation](https://docs.airbyte.com/connector-development/). + +::: + ## Sync overview The Zuora source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. diff --git a/docs/operating-airbyte/security.md b/docs/operating-airbyte/security.md index 7f1b10973bd6..ae224b3ad75a 100644 --- a/docs/operating-airbyte/security.md +++ b/docs/operating-airbyte/security.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Security Airbyte is committed to keeping your data safe by following industry-standard practices for securing physical deployments, setting access policies, and leveraging the security features of leading Cloud providers. @@ -84,14 +88,6 @@ Note that this process is not reversible. Once you have converted to a secret st Most Airbyte Open Source connectors support encryption-in-transit (SSL or HTTPS). We recommend configuring your connectors to use the encryption option whenever available. -### Telemetry - -Airbyte does send anonymized data to our services to improve the product (especially connector reliability and scale). To disable telemetry, modify the .env file and define the following environment variable: - -``` -TRACKING_STRATEGY=logging -``` - ## Securing Airbyte Cloud Airbyte Cloud leverages the security features of leading Cloud providers and sets least-privilege access policies to ensure data security. diff --git a/docs/operator-guides/browsing-output-logs.md b/docs/operator-guides/browsing-output-logs.md index 19de2cdcb6b6..d4afd258c227 100644 --- a/docs/operator-guides/browsing-output-logs.md +++ b/docs/operator-guides/browsing-output-logs.md @@ -1,6 +1,8 @@ -# Browsing Logs +--- +products: all +--- -## Overview +# Browsing logs Airbyte records the full logs as a part of each sync. These logs can be used to understand the underlying operations Airbyte performs to read data from the source and write to the destination as a part of the [Airbyte Protocol](/understanding-airbyte/airbyte-protocol.md). The logs includes many details, including any errors that can be helpful when troubleshooting sync errors. diff --git a/docs/operator-guides/collecting-metrics.md b/docs/operator-guides/collecting-metrics.md index 9db6198e23b4..a1203fc5191e 100644 --- a/docs/operator-guides/collecting-metrics.md +++ b/docs/operator-guides/collecting-metrics.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Monitoring Airbyte diff --git a/docs/operator-guides/configuring-airbyte-db.md b/docs/operator-guides/configuring-airbyte-db.md index 1b57f2ae9073..9adc4c881b56 100644 --- a/docs/operator-guides/configuring-airbyte-db.md +++ b/docs/operator-guides/configuring-airbyte-db.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring the Airbyte Database Airbyte uses different objects to store internal state and metadata. This data is stored and manipulated by the various Airbyte components, but you have the ability to manage the deployment of this database in the following two ways: @@ -21,6 +25,10 @@ If you need to interact with it, for example, to make back-ups or perform some c ## Connecting to an External Postgres database +:::info +Currently, Airbyte requires connection to a Postgres 13 instance. +::: + Let's walk through what is required to use a Postgres instance that is not managed by Airbyte. First, for the sake of the tutorial, we will run a new instance of Postgres in its own docker container with the command below. If you already have Postgres running elsewhere, you can skip this step and use the credentials for that in future steps. ```bash diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 8f234a915090..0618613a3b76 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring Airbyte This section covers how to configure Airbyte, and the various configuration Airbyte accepts. @@ -53,6 +57,8 @@ The following variables are relevant to both Docker and Kubernetes. 7. `VAULT_AUTH_METHOD` - How vault will preform authentication. Currently, only supports Token auth. Defaults to token. Alpha Support. 8. `AWS_ACCESS_KEY` - Defines the aws_access_key_id from the AWS credentials to use for AWS Secret Manager. 9. `AWS_SECRET_ACCESS_KEY`- Defines aws_secret_access_key to use for the AWS Secret Manager. +10. `AWS_KMS_KEY_ARN` - Optional param that defines the KMS Encryption key used for the AWS Secret Manager. +11. `AWS_SECRET_MANAGER_SECRET_TAGS` - Defines the tags that will be included to all writes to the AWS Secret Manager. The format should be "key1=value1,key2=value2". #### Database diff --git a/docs/operator-guides/configuring-connector-resources.md b/docs/operator-guides/configuring-connector-resources.md index 9fc0df16325c..20c03a8dc9bb 100644 --- a/docs/operator-guides/configuring-connector-resources.md +++ b/docs/operator-guides/configuring-connector-resources.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring Connector Resources As noted in [Workers & Jobs](../understanding-airbyte/jobs.md), there are four different types of jobs. diff --git a/docs/operator-guides/reset.md b/docs/operator-guides/reset.md index de8080165e0f..e68bbbc877be 100644 --- a/docs/operator-guides/reset.md +++ b/docs/operator-guides/reset.md @@ -1,4 +1,8 @@ -# Resetting Your Data +--- +products: all +--- + +# Resetting your data Resetting your data allows you to drop all previously synced data so that any ensuing sync can start syncing fresh. This is useful if you don't require the data replicated to your destination to be saved permanently or are just testing Airbyte. @@ -20,13 +24,7 @@ You will also automatically be prompted to reset affected streams if you edit an Similarly to a sync job, a reset can be completed as successful, failed, or cancelled. To resolve a failed reset, you should manually drop the tables in the destination so that Airbyte can continue syncing accurately into the destination. ## Reset behavior -When a reset is successfully completed, all the records are deleted from your destination tables (and files, if using local JSON or local CSV as the destination). - -:::info -If you are using destinations that are on the [Destinations v2](/release_notes/upgrading_to_destinations_v2.md) framework, only raw tables will be cleared of their data. Final tables will retain all records from the last sync. -::: - -A reset **DOES NOT** delete any destination tables when using a data warehouse, data lake, database. The schema is retained but will not contain any rows. +When a reset is successfully completed, all the records are deleted from your destination tables (and files, if using local JSON or local CSV as the destination), and then the next sync will begin. :::tip If you have any orphaned tables or files that are no longer being synced to, they should be cleaned up separately, as Airbyte will not clean them up for you. This can occur when the `Destination Namespace` or `Stream Prefix` connection configuration is changed for an existing connection. diff --git a/docs/operator-guides/scaling-airbyte.md b/docs/operator-guides/scaling-airbyte.md index 062cbd33d715..9c80cdbff378 100644 --- a/docs/operator-guides/scaling-airbyte.md +++ b/docs/operator-guides/scaling-airbyte.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Scaling Airbyte As depicted in our [High-Level View](../understanding-airbyte/high-level-view.md), Airbyte is made up of several components under the hood: 1. Scheduler 2. Server 3. Temporal 4. Webapp 5. Database diff --git a/docs/operator-guides/telemetry.md b/docs/operator-guides/telemetry.md new file mode 100644 index 000000000000..71352d7c8b47 --- /dev/null +++ b/docs/operator-guides/telemetry.md @@ -0,0 +1,30 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Telemetry + +Airbyte collects telemetry data in the UI and the servers to help us understand users and their use-cases better to improve the product. + +Also check our [privacy policy](https://airbyte.com/privacy-policy) for more details. + + + + To disable telemetry for your instance, modify the `.env` file and define the following environment variable: + + ``` + TRACKING_STRATEGY=logging + ``` + + + When visiting the webapp or our homepage the first time, you'll be asked for your consent to + telemetry collection depending on the legal requirements of your location. + + To change this later go to **Settings** > **User Settings** > **Cookie Preferences** or **Cookie Preferences** in the footer of our [homepage](https://airbyte.com). + + Server side telemetry collection can't be changed using Airbyte Cloud. + + \ No newline at end of file diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md b/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md index 1f0175b392d8..30fa2c4051e6 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md @@ -1,6 +1,14 @@ +--- +products: oss-* +--- + # Transformations with Airbyte (Part 3/3) -## Overview +:::warning +Normalization and Custom Transformation are deprecated features. +Destinations using Normalization will be replaced by [Typing and Deduping](/using-airbyte/core-concepts/typing-deduping.md). +Custom Transformation will be removed on March 31. For more information, visit [here](https://github.com/airbytehq/airbyte/discussions/34860). +::: This tutorial will describe how to push a custom dbt transformation project back to Airbyte to use during syncs. diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md b/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md index e7ea6b4158bb..bbb7987d0b1b 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md @@ -1,6 +1,14 @@ +--- +products: oss-* +--- + # Transformations with dbt (Part 2/3) -## Overview +:::warning +Normalization and Custom Transformation are deprecated features. +Destinations using Normalization will be replaced by [Typing and Deduping](/using-airbyte/core-concepts/typing-deduping.md). +Custom Transformation will be removed on March 31. For more information, visit [here](https://github.com/airbytehq/airbyte/discussions/34860). +::: This tutorial will describe how to integrate SQL based transformations with Airbyte syncs using specialized transformation tool: dbt. diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md b/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md index 4e29e15fe167..361b26c657a5 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md @@ -1,8 +1,14 @@ -# Transformations with SQL (Part 1/3) +--- +products: oss-* +--- -## Transformations with SQL \(Part 1/3\) +# Transformations with SQL (Part 1/3) -### Overview +:::warning +Normalization and Custom Transformation are deprecated features. +Destinations using Normalization will be replaced by [Typing and Deduping](/using-airbyte/core-concepts/typing-deduping.md). +Custom Transformation will be removed on March 31. For more information, visit [here](https://github.com/airbytehq/airbyte/discussions/34860). +::: This tutorial will describe how to integrate SQL based transformations with Airbyte syncs using plain SQL queries. diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 956ab977521c..5a4da98d9904 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Upgrading Airbyte :::info @@ -80,91 +84,22 @@ This will completely reset your Airbyte deployment back to scratch and you will ::: -## Upgrading on K8s (0.27.0-alpha and above) - -If you are upgrading from (i.e. your current version of Airbyte is) Airbyte version **0.27.0-alpha or above** on Kubernetes : - -1. In a terminal, on the host where Airbyte is running, turn off Airbyte. - - ```bash - kubectl delete deployments airbyte-db airbyte-worker airbyte-server airbyte-temporal airbyte-webapp --namespace= - ``` - -2. Upgrade the kube deployment to new version. - - i. If you are running Airbyte from a cloned version of the Airbyte GitHub repo and want to use the current most recent stable version, just `git pull`. - -3. Bring Airbyte back online. - - ```bash - kubectl apply -k kube/overlays/stable - ``` - - After 2-5 minutes, `kubectl get pods | grep airbyte` should show `Running` as the status for all the core Airbyte pods. This may take longer on Kubernetes clusters with slow internet connections. - - Run `kubectl port-forward svc/airbyte-webapp-svc 8000:80` to allow access to the UI/API. - -## Upgrading on K8s (0.26.4-alpha and below) - -If you are upgrading from (i.e. your current version of Airbyte is) Airbyte version **before 0.27.0-alpha** on Kubernetes we **do not** support automatic migration. Please follow the following steps to upgrade your Airbyte Kubernetes deployment. - -1. Switching over to your browser, navigate to the Admin page in the UI. Then go to the Configuration Tab. Click Export. This will download a compressed back-up archive \(gzipped tarball\) of all of your Airbyte configuration data and sync history locally. - - _Note: Any secrets that you have entered into Airbyte will be in this archive, so you should treat it as a secret._ - -2. Back to the terminal, migrate the local archive to the new version using the Migration App (packaged in a docker container). - - ```bash - docker run --rm -v :/config airbyte/migration: --\ - --input /config/airbyte_archive.tar.gz\ - --output \ - [ --target-version ] - ``` - - Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. - - ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.50.37 --\ - --input /config/airbyte_archive.tar.gz\ - --output /config/airbyte_archive_migrated.tar.gz - ``` - -3. Turn off Airbyte fully and **(see warning)** delete the existing Airbyte Kubernetes volumes. - - _WARNING: Make sure you have already exported your data \(step 1\). This command is going to delete your data in Kubernetes, you may lose your airbyte configurations!_ - - This is where all airbyte configurations are saved. Those configuration files need to be upgraded and restored with the proper version in the following steps. - - ```bash - # Careful, this is deleting data! - kubectl delete -k kube/overlays/stable - ``` +## Upgrading on K8s using Helm -4. Follow **Step 2** in the `Upgrading on Docker` section to check out the most recent version of Airbyte. Although it is possible to migrate by changing the `.env` file in the kube overlay directory, this is not recommended as it does not capture any changes to the Kubernetes manifests. -5. Bring Airbyte back up. +The instructions below are for users using custom deployment and have a `values.yaml`. If you're not using a `values.yaml` to deploy Airbyte using Helm can jump directly to step `4.`. +1. Access [Airbyte ArtifactHub](https://artifacthub.io/packages/helm/airbyte/airbyte) and select the version you want to upgrade. +2. You can click in `Default Values` and compare the value file between the new version and version you're running. You can run `helm list -n ` to check the CHART version you're using. +3. Update your `values.yaml` file if necessary. +4. Upgrade the Helm app running: ```bash - kubectl apply -k kube/overlays/stable + helm upgrade --install airbyte/airbyte --values --version ``` -6. Switching over to your browser, navigate to the Admin page in the UI. Then go to the Configuration Tab and click on Import. Upload your migrated archive. - -If you prefer to import and export your data via API instead the UI, follow these instructions: - -1. Instead of Step 3 above use the following curl command to export the archive: - + After 2-5 minutes, Helm will print a message showing how to port-forward Airbyte. This may take longer on Kubernetes clusters with slow internet connections. In general the message is the following: ```bash - curl -H "Content-Type: application/json" -X POST localhost:8000/api/v1/deployment/export --output /tmp/airbyte_archive.tar.gz - ``` - -2. Instead of Step X above user the following curl command to import the migrated archive: - - ```bash - curl -H "Content-Type: application/x-gzip" -X POST localhost:8000/api/v1/deployment/import --data-binary @ - ``` - -Here is an example of what this request might look like assuming that the migrated archive is called `airbyte_archive_migrated.tar.gz` and is in the `/tmp` directory. - -```bash -curl -H "Content-Type: application/x-gzip" -X POST localhost:8000/api/v1/deployment/import --data-binary @/tmp/airbyte_archive_migrated.tar.gz -``` + export POD_NAME=$(kubectl get pods -l "app.kubernetes.io/name=webapp" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:$CONTAINER_PORT + ``` diff --git a/docs/operator-guides/using-custom-connectors.md b/docs/operator-guides/using-custom-connectors.md index 04be26cf889e..6597dc7ad88a 100644 --- a/docs/operator-guides/using-custom-connectors.md +++ b/docs/operator-guides/using-custom-connectors.md @@ -1,4 +1,9 @@ -# Using custom connectors +--- +products: oss-* +sidebar_label: Uploading custom connectors +--- + +# Uploading Docker-based custom connectors :::info This guide walks through the setup of a Docker-based custom connector. To understand how to use our low-code connector builder, read our guide [here](/connector-development/connector-builder-ui/overview.md). diff --git a/docs/operator-guides/using-dagster-integration.md b/docs/operator-guides/using-dagster-integration.md index ac5f60834bcf..03dd051118dc 100644 --- a/docs/operator-guides/using-dagster-integration.md +++ b/docs/operator-guides/using-dagster-integration.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Dagster in minutes +products: oss-* --- # Using the Dagster Integration @@ -67,3 +68,7 @@ Don't be fooled by our simple example of only one Dagster Flow. Airbyte is a pow We love to hear any questions or feedback on our [Slack](https://slack.airbyte.io/). We're still in alpha, so if you see any rough edges or want to request a connector, feel free to create an issue on our [Github](https://github.com/airbytehq/airbyte) or thumbs up an existing issue. +## Related articles and guides +For additional information about using Dagster and Airbyte together, see the following: + +- [Build an e-commerce analytics stack with Airbyte, dbt, Dagster and BigQuery](https://github.com/airbytehq/quickstarts/tree/main/ecommerce_analytics_bigquery) diff --git a/docs/operator-guides/using-kestra-plugin.md b/docs/operator-guides/using-kestra-plugin.md index 3c27e8797c2d..0a8da24761a3 100644 --- a/docs/operator-guides/using-kestra-plugin.md +++ b/docs/operator-guides/using-kestra-plugin.md @@ -1,5 +1,6 @@ --- description: Using the Kestra Plugin to Orchestrate Airbyte +products: oss-* --- # Using the Kestra Plugin diff --git a/docs/operator-guides/using-prefect-task.md b/docs/operator-guides/using-prefect-task.md index d0b462e10f23..c7339306356d 100644 --- a/docs/operator-guides/using-prefect-task.md +++ b/docs/operator-guides/using-prefect-task.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Prefect in minutes +products: oss-* --- # Using the Prefect Airbyte Task @@ -90,3 +91,7 @@ Don't be fooled by our simple example of only one Prefect Flow. Airbyte is a pow We love to hear any questions or feedback on our [Slack](https://slack.airbyte.io/). We're still in alpha, so if you see any rough edges or want to request a connector, feel free to create an issue on our [Github](https://github.com/airbytehq/airbyte) or thumbs up an existing issue. +## Related articles and guides +For additional information about using Prefect and Airbyte together, see the following: + +- [Build an e-commerce analytics stack with Airbyte, dbt, Prefect and BigQuery](https://github.com/airbytehq/quickstarts/tree/main/airbyte_dbt_prefect_bigquery) diff --git a/docs/operator-guides/using-the-airflow-airbyte-operator.md b/docs/operator-guides/using-the-airflow-airbyte-operator.md index 97c73eb37f91..84831527f014 100644 --- a/docs/operator-guides/using-the-airflow-airbyte-operator.md +++ b/docs/operator-guides/using-the-airflow-airbyte-operator.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Apache Airflow in minutes +products: oss-* --- # Using the Airbyte Operator to orchestrate Airbyte OSS @@ -137,11 +138,12 @@ Don't be fooled by our simple example of only one Airflow task. Airbyte is a pow We love to hear any questions or feedback on our [Slack](https://slack.airbyte.io/). We're still in alpha, so if you see any rough edges or want to request a connector, feel free to create an issue on our [Github](https://github.com/airbytehq/airbyte) or thumbs up an existing issue. -## Related articles +## Related articles and guides For additional information about using the Airflow and Airbyte together, see the following: - [Using the new Airbyte API to orchestrate Airbyte Cloud with Airflow](https://airbyte.com/blog/orchestrating-airbyte-api-airbyte-cloud-airflow) - [A step-by-step guide to setting up and configuring Airbyte and Airflow to work together](https://airbyte.com/tutorials/how-to-use-airflow-and-airbyte-together) +- [Build an e-commerce Analytics Stack with Airbyte, dbt, Airflow (ADA) and BigQuery](https://github.com/airbytehq/quickstarts/tree/main/airbyte_dbt_airflow_bigquery) - [The difference between Airbyte and Airflow](https://airbyte.com/blog/airbyte-vs-airflow) - [ETL Pipelines with Airflow: the Good, the Bad and the Ugly](https://airbyte.com/blog/airflow-etl-pipelines) - [Automate your Data Scraping with Apache Airflow and Beautiful Soup](https://airbyte.com/tutorials/data-scraping-with-airflow-and-beautiful-soup) diff --git a/docs/release_notes/december_2023.md b/docs/release_notes/december_2023.md new file mode 100644 index 000000000000..3aa0ba4df2cf --- /dev/null +++ b/docs/release_notes/december_2023.md @@ -0,0 +1,19 @@ +# December 2023 +## airbyte v0.50.36 to v0.50.40 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +Airbyte introduced a new schemaless mode for our MongoDB source connector to improve our ability to sync data from collections with varying fields for each document. This enhancement not only streamlines connector configuration, but also ensures reliable data propagation, even when upstream teams modify the fields uploaded to new MongoDB documents in your collection. + +## Connector Improvements + +In addition to our schemaless mode for MongoDB, we have also: + + - Enhanced our [Bing Ads](https://github.com/airbytehq/airbyte/pull/33095) source by allowing for account-specific filtering and improved error handling. + - Enabled per-stream state for [MS SQL](https://github.com/airbytehq/airbyte/pull/33018) source to increase resiliency to stream changes. + - Published a new [OneDrive](https://github.com/airbytehq/airbyte/pull/32655) source connector to support additional unstructured data in files. + - Added streams for our [Hubspot](https://github.com/airbytehq/airbyte/pull/33266) source to add `property_history` for Companies and Deals. We also added incremental syncing for all property history streams for increased sync reliability. + - Improved our [Klaviyo](https://github.com/airbytehq/airbyte/pull/33099) source connector to account for rate-limiting and gracefully handle stream-specific errors to continue syncing other streams + diff --git a/docs/release_notes/destinations_v2.js b/docs/release_notes/destinations_v2.js index f8c8494d8f19..d880e3b5fdf2 100644 --- a/docs/release_notes/destinations_v2.js +++ b/docs/release_notes/destinations_v2.js @@ -1,5 +1,5 @@ -import React, {useState} from 'react'; -import CodeBlock from '@theme/CodeBlock'; +import React, { useState } from "react"; +import CodeBlock from "@theme/CodeBlock"; function concatenateRawTableName(namespace, name) { let plainConcat = namespace + name; @@ -8,18 +8,21 @@ function concatenateRawTableName(namespace, name) { for (let i = 0; i < plainConcat.length; i++) { // If we've found an underscore, count the number of consecutive underscores let underscoreRun = 0; - while (i < plainConcat.length && plainConcat.charAt(i) === '_') { - underscoreRun++; - i++; + while (i < plainConcat.length && plainConcat.charAt(i) === "_") { + underscoreRun++; + i++; } longestUnderscoreRun = Math.max(longestUnderscoreRun, underscoreRun); } - return namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name; + return ( + namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name + ); } // Taken from StandardNameTransformer function convertStreamName(str) { - return str.normalize('NFKD') + return str + .normalize("NFKD") .replaceAll(/\p{M}/gu, "") .replaceAll(/\s+/g, "_") .replaceAll(/[^A-Za-z0-9_]/g, "_"); @@ -44,9 +47,12 @@ export const BigQueryMigrationGenerator = () => { } function generateSql(og_namespace, new_namespace, name, raw_dataset) { - let v2RawTableName = '`' + bigqueryConvertStreamName(concatenateRawTableName(new_namespace, name)) + '`'; - let v1namespace = '`' + escapeNamespace(og_namespace) + '`'; - let v1name = '`' + bigqueryConvertStreamName("_airbyte_raw_" + name) + '`'; + let v2RawTableName = + "`" + + bigqueryConvertStreamName(concatenateRawTableName(new_namespace, name)) + + "`"; + let v1namespace = "`" + escapeNamespace(og_namespace) + "`"; + let v1name = "`" + bigqueryConvertStreamName("_airbyte_raw_" + name) + "`"; return `CREATE SCHEMA IF NOT EXISTS ${raw_dataset}; CREATE OR REPLACE TABLE \`${raw_dataset}\`.${v2RawTableName} ( _airbyte_raw_id STRING, @@ -66,9 +72,9 @@ AS ( } return ( - + ); -} +}; export const SnowflakeMigrationGenerator = () => { // See SnowflakeSQLNameTransformer @@ -81,7 +87,8 @@ export const SnowflakeMigrationGenerator = () => { } } function generateSql(og_namespace, new_namespace, name, raw_schema) { - let v2RawTableName = '"' + concatenateRawTableName(new_namespace, name) + '"'; + let v2RawTableName = + '"' + concatenateRawTableName(new_namespace, name) + '"'; let v1namespace = snowflakeConvertStreamName(og_namespace); let v1name = snowflakeConvertStreamName("_airbyte_raw_" + name); return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}"; @@ -100,36 +107,115 @@ AS ( )`; } return ( - + ); -} +}; -export const MigrationGenerator = ({destination, generateSql}) => { - const defaultMessage = -`Enter your stream's name and namespace to see the SQL output. +export const RedshiftMigrationGenerator = () => { + // See RedshiftSQLNameTransformer + function redshiftConvertStreamName(str) { + str = convertStreamName(str); + if (str.charAt(0).match(/[A-Za-z_]/)) { + return str; + } else { + return "_" + str; + } + } + function generateSql(og_namespace, new_namespace, name, raw_schema) { + let v2RawTableName = + '"' + concatenateRawTableName(new_namespace, name) + '"'; + let v1namespace = redshiftConvertStreamName(og_namespace); + let v1name = redshiftConvertStreamName("_airbyte_raw_" + name); + return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}"; +DROP TABLE IF EXISTS "${raw_schema}".${v2RawTableName}; +CREATE TABLE "${raw_schema}".${v2RawTableName} ( + "_airbyte_raw_id" VARCHAR(36) NOT NULL PRIMARY KEY + , "_airbyte_extracted_at" TIMESTAMPTZ DEFAULT NOW() + , "_airbyte_loaded_at" TIMESTAMPTZ + , "_airbyte_data" SUPER +); +INSERT INTO "${raw_schema}".${v2RawTableName} ( + SELECT + _airbyte_ab_id AS "_airbyte_raw_id", + _airbyte_emitted_at AS "_airbyte_extracted_at", + CAST(NULL AS TIMESTAMPTZ) AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM ${v1namespace}.${v1name} +);`; + } + return ( + + ); +}; + +export const PostgresMigrationGenerator = () => { + // StandardNameTransformer + identifier should start with a letter or an underscore + function postgresConvertStreamName(str) { + str = convertStreamName(str); + if (str.charAt(0).match(/[A-Za-z_]/)) { + return str; + } else { + return "_" + str; + } + } + function generateSql(og_namespace, new_namespace, name, raw_schema) { + let v2RawTableName = + concatenateRawTableName(new_namespace, name).toLowerCase(); + let v1namespace = postgresConvertStreamName(og_namespace); + let v1name = postgresConvertStreamName("_airbyte_raw_" + name).toLowerCase(); + return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}"; +DROP TABLE IF EXISTS "${raw_schema}".${v2RawTableName}; +CREATE TABLE "${raw_schema}".${v2RawTableName} ( + "_airbyte_raw_id" VARCHAR(36) NOT NULL PRIMARY KEY + , "_airbyte_extracted_at" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + , "_airbyte_loaded_at" TIMESTAMP WITH TIME ZONE DEFAULT NULL + , "_airbyte_data" JSONB +); +INSERT INTO "${raw_schema}".${v2RawTableName} ( + SELECT + _airbyte_ab_id AS "_airbyte_raw_id", + _airbyte_emitted_at AS "_airbyte_extracted_at", + CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM ${v1namespace}.${v1name} +);`; + } + return ( + + ); +}; + +export const MigrationGenerator = ({ destination, generateSql }) => { + const defaultMessage = `Enter your stream's name and namespace to see the SQL output. If your stream has no namespace, take the default value from the destination connector's settings.`; const [message, updateMessage] = useState({ - 'message': defaultMessage, - 'language': 'text' + message: defaultMessage, + language: "text", }); function updateSql(event) { - let og_namespace = document.getElementById("og_stream_namespace_" + destination).value; - let new_namespace = document.getElementById("new_stream_namespace_" + destination).value; + let og_namespace = document.getElementById( + "og_stream_namespace_" + destination + ).value; + let new_namespace = document.getElementById( + "new_stream_namespace_" + destination + ).value; let name = document.getElementById("stream_name_" + destination).value; - var raw_dataset = document.getElementById("raw_dataset_" + destination).value; - if (raw_dataset === '') { - raw_dataset = 'airbyte_internal'; + var raw_dataset = document.getElementById( + "raw_dataset_" + destination + ).value; + if (raw_dataset === "") { + raw_dataset = "airbyte_internal"; } let sql = generateSql(og_namespace, new_namespace, name, raw_dataset); - if ([og_namespace, new_namespace, name].every(text => text != "")) { + if ([og_namespace, new_namespace, name].every((text) => text != "")) { updateMessage({ - 'message': sql, - 'language': 'sql' + message: sql, + language: "sql", }); } else { updateMessage({ - 'message': defaultMessage, - 'language': 'text' + message: defaultMessage, + language: "text", }); } } @@ -137,16 +223,41 @@ If your stream has no namespace, take the default value from the destination con return (
      -
      + +
      -
      + +
      -
      - -
      - - { message['message'] } + +
      + + +
      + + {message["message"]}
      ); -} +}; diff --git a/docs/release_notes/january_2024.md b/docs/release_notes/january_2024.md new file mode 100644 index 000000000000..95be1d2b5c70 --- /dev/null +++ b/docs/release_notes/january_2024.md @@ -0,0 +1,18 @@ +# January 2024 +## airbyte v0.50.41 to v0.50.45 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +Airbyte migrated our [Redshift destination](https://github.com/airbytehq/airbyte/pull/34077) on the [Destinations V2](./upgrading_to_destinations_v2) framework. This enables you to map tables one-to-one with your source, experience better error handling, and deliver data incrementally. + +## Connector Improvements + +In addition to our Redshift V2 destination, we also released a few notable Connector improvements: + + - Our S3 Source now supports [IAM role-based authentication](https://github.com/airbytehq/airbyte/pull/33818), allowing users to utilize IAM roles for more granular control over permissions and to eliminate the need for managing static access keys. + - Our [Salesforce](https://github.com/airbytehq/airbyte/issues/30819) source now supports syncing the object ContentDocumentLink, which enables reporting for files within Content Documents. + - [OneDrive](https://docs.airbyte.com/integrations/sources/microsoft-onedrive) and [Sharepoint](https://github.com/airbytehq/airbyte/pull/33537) are now offered as a source from which to connect your files. + - Stripe and Salesforce are enabled to run [concurrently](https://github.com/airbytehq/airbyte/pull/34454) with full refresh with 4x speed + diff --git a/docs/release_notes/november_2023.md b/docs/release_notes/november_2023.md new file mode 100644 index 000000000000..67323252e8c5 --- /dev/null +++ b/docs/release_notes/november_2023.md @@ -0,0 +1,24 @@ +# November 2023 +## airbyte v0.50.34 to v0.50.35 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +Airbyte now supports extracting text content from PDF, Docx, and Pptx files from S3, Azure Blob Storage, and the newly introduced [Google Drive](/integrations/sources/google-drive.md) source. This is an important part of supporting LLM use cases that rely on unstructured data in files. + +SSO and RBAC (admin roles only) are now available in Airbyte Cloud! Read more below. + +## Platform Releases +- **SSO and RBAC** You can now use SSO in Airbyte Cloud to administer permissions in Airbyte. This is currently only available through Okta, with plans to support Active Directory next. We also now offer **RBAC** (admin roles only) to ensure a high level of security when managing you workspace. For access to this feature, reach out to our [Sales team](https://www.airbyte.com/company/talk-to-sales). +- **Continuous heartbeat checks** We're continually monitoring syncs to verify they continue making progress, and have added functionality in the background to ensure that we continue receiving updated ["heartbeat" messages](/understanding-airbyte/heartbeats.md) from our connectors. This will ensure that we continue delivering data and avoid any timeouts. + +## Connector Improvements + +In addition to being able to extract text content from unstructured data sources, we have also: + + - Revamped core Marketing connectors Pinterest, Instagram and Klaviyo to significantly improve the setup experience and ensure resiliency and reliability. + - [Added incremenetal sync](https://github.com/airbytehq/airbyte/pull/32473) functionality for Hubspot's stream `property_history`, which improves sync time and reliability. + - [Added new streams](https://github.com/airbytehq/airbyte/pull/32738) for Amazon Seller Partner: `get_vendor_net_pure_product_margin_report`,`get_vendor_readl_time_inventory_report`, and `get_vendor_traffic_report` to enable additional reporting. + - Released our first connector, Stripe, that can perform [concurrent syncs](https://github.com/airbytehq/airbyte/pull/32473) where streams sync in parallel when syncing in Full Refresh mode. + diff --git a/docs/release_notes/upgrading_to_destinations_v2.md b/docs/release_notes/upgrading_to_destinations_v2.md index 15d9d173d2d2..e0f27a394e64 100644 --- a/docs/release_notes/upgrading_to_destinations_v2.md +++ b/docs/release_notes/upgrading_to_destinations_v2.md @@ -1,6 +1,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import {SnowflakeMigrationGenerator, BigQueryMigrationGenerator} from './destinations_v2.js' +import {SnowflakeMigrationGenerator, BigQueryMigrationGenerator, RedshiftMigrationGenerator, PostgresMigrationGenerator} from './destinations_v2.js' # Upgrading to Destinations V2 @@ -32,7 +32,7 @@ Note that Destinations V2 also removes the option to _only_ replicate raw data. The following table details the delivered data modified by Destinations V2: | Current Normalization Setting | Source Type | Impacted Data (Breaking Changes) | -|-------------------------------|---------------------------------------|----------------------------------------------------------| +| ----------------------------- | ------------------------------------- | -------------------------------------------------------- | | Raw JSON | All | `_airbyte` metadata columns, raw table location | | Normalized tabular data | API Source | Unnested tables, `_airbyte` metadata columns, SCD tables | | Normalized tabular data | Tabular Source (database, file, etc.) | `_airbyte` metadata columns, SCD tables | @@ -43,7 +43,7 @@ Whenever possible, we've taken this opportunity to use the best data type for st ## Quick Start to Upgrading -**The quickest path to upgrading is to click upgrade on any out-of-date connection in the UI**. The advanced options later in this document will allow you to test out the upgrade in more detail if you choose. +**The quickest path to upgrading is to click upgrade on any out-of-date connection in the UI**. The advanced options later in this document will allow you to test out the upgrade in more detail if you choose. :::caution @@ -51,9 +51,19 @@ Whenever possible, we've taken this opportunity to use the best data type for st ::: - ![Upgrade Path](./assets/airbyte_destinations_v2_upgrade_prompt.png) +:::caution Upgrade Warning + +* The upgrading process entails hydrating the v2 format raw table by querying the v1 raw table through a standard query, such as "INSERT INTO v2_raw_table SELECT * FROM v1_raw_table." +The duration of this process can vary significantly based on the data size and may encounter failures contingent on the Destination's capacity to execute the query. +In some cases, creating a new Airbyte connection, rather than migrating your existing connection, may be faster. Note that in these cases, all data will be re-imported. +* Following the successful migration of v1 raw tables to v2, the v1 raw tables will be dropped. However, it is essential to note that if there are any derived objects (materialized views) or referential +constraints (foreign keys) linked to the old raw table, this operation may encounter failure, resulting in an unsuccessful upgrade or broken derived objects (like materialized views etc). + +If any of the above concerns are applicable to your existing setup, we recommend [Upgrading Connections One by One with Dual-Writing](#upgrading-connections-one-by-one-with-dual-writing) for a more controlled upgrade process +::: + After upgrading the out-of-date destination to a [Destinations V2 compatible version](#destinations-v2-effective-versions), the following will occur at the next sync **for each connection** sending data to the updated destination: 1. Existing raw tables replicated to this destination will be copied to a new `airbyte_internal` schema. @@ -73,7 +83,7 @@ Versions are tied to the destination. When you update the destination, **all con - [Testing Destinations V2 on a Single Connection](#testing-destinations-v2-for-a-single-connection) - [Upgrading Connections One by One Using CDC](#upgrade-paths-for-connections-using-cdc) - [Upgrading as a User of Raw Tables](#upgrading-as-a-user-of-raw-tables) -- [Rolling back to Legacy Normalization](#oss-only-rolling-back-to-legacy-normalization) +- [Rolling back to Legacy Normalization](#open-source-only-rolling-back-to-legacy-normalization) ## Advanced Upgrade Paths @@ -107,6 +117,12 @@ These steps allow you to dual-write for connections incrementally syncing data w + + + + + + 2. Navigate to the existing connection you are duplicating, and navigate to the `Settings` tab. Open the `Advanced` settings to see the connection state (which manages incremental syncs). Copy the state to your clipboard. @@ -131,8 +147,8 @@ When you are done testing, you can disable or delete this testing connection, an If you have written downstream transformations directly from the output of raw tables, or use the "Raw JSON" normalization setting, you should know that: - Multiple column names are being updated (from `airbyte_ab_id` to `airbyte_raw_id`, and `airbyte_emitted_at` to `airbyte_extracted_at`). -- The location of raw tables will from now on default to an `airbyte` schema in your destination. -- When you upgrade to a [Destinations V2 compatible version](#destinations-v2-effective-versions) of your destination, we will leave a copy of your existing raw tables as they are, and new syncs will work from a new copy we make in the new `airbyte_internal` schema. Although existing downstream dashboards will go stale, they will not be broken. +- The location of raw tables will from now on default to an `airbyte_internal` schema in your destination. +- When you upgrade to a [Destinations V2 compatible version](#destinations-v2-effective-versions) of your destination, we will leave a copy of your existing raw tables as they are, and new syncs will work from a new copy we make in the new `airbyte_internal` schema. Although existing downstream dashboards will go stale, they will not be broken. - You can dual write by following the [steps above](#upgrading-connections-one-by-one-with-dual-writing) and copying your raw data to the schema of your newly created connection. We may make further changes to raw tables in the future, as these tables are intended to be a staging ground for Airbyte to optimize the performance of your syncs. We cannot guarantee the same level of stability as for final tables in your destination schema, nor will features like error handling be implemented in the raw tables. @@ -156,8 +172,8 @@ For each destination connector, Destinations V2 is effective as of the following |-----------------------|-----------------------|----------------------------|--------------------------| | BigQuery | 1.10.2 | 2.0.6+ | November 7, 2023 | | Snowflake | 2.1.7 | 3.1.0+ | November 7, 2023 | -| Redshift | 0.6.11 | [coming soon] 2.0.0+ | [coming soon] early 2024 | -| Postgres | 0.4.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | +| Redshift | 0.8.0 | 2.0.0+ | March 15, 2024 | +| Postgres | 0.6.3 | 2.0.0+ | May 31, 2024 | | MySQL | 0.2.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | Note that legacy normalization will be deprecated for ClickHouse, DuckDB, MSSQL, TiDB, and Oracle DB in early 2024. If you wish to add Destinations V2 capability to these destinations, please reference our implementation guide (coming soon). @@ -184,8 +200,38 @@ In addition to the changes which apply for all destinations described above, the ### BigQuery -1. [Object and array properties](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) are properly stored as JSON columns. Previously, we had used TEXT, which made querying sub-properties more difficult. - - In certain cases, numbers within sub-properties with long decimal values will need to be converted to float representations due to a _quirk_ of Bigquery. Learn more [here](https://github.com/airbytehq/airbyte/issues/29594). +#### [Object and array properties](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) are properly stored as JSON columns +Previously, we had used TEXT, which made querying sub-properties more difficult. +In certain cases, numbers within sub-properties with long decimal values will need to be converted to float representations due to a _quirk_ of Bigquery. Learn more [here](https://github.com/airbytehq/airbyte/issues/29594). + +### Snowflake + +#### Explicitly uppercase column names in Final Tables +Snowflake will implicitly uppercase column names if they are not quoted. Airbyte needs to quote the column names because a variety of sources have column/field names which contain special characters that require quoting in Snowflake. +However, when you quote a column name in Snowflake, it also preserves lowercase naming. During the Snowflake V2 beta, most customers found this behavior unexpected and expected column selection to be case-insensitive for columns without special characters. +As a result of this feedback, we decided to explicitly uppercase column names in the final tables, which does mean that columns which previous required quoting, now also require you to convert to the upper case version. + +For example: + +```sql +-- Snowflake will implicitly uppercase column names which are not quoted +-- These three queries are equivalent +SELECT my_column from my_table; +SELECT MY_COLUMN from MY_TABLE; +SELECT "MY_COLUMN" from MY_TABLE; + +-- However, this query is different, and requires a lowercase column name +SELECT "my_column" from my_table; + +-- Because we are explicitly upper-casing column names, column names containing special characters (like a space) +-- should now also be uppercase + +-- Before v2 +SELECT "my column" from my_table; +-- After v2 +SELECT "MY COLUMN" from my_table; +``` + ## Updating Downstream Transformations diff --git a/docs/terraform-documentation.md b/docs/terraform-documentation.md index e9e52ff6e396..dc4b40588841 100644 --- a/docs/terraform-documentation.md +++ b/docs/terraform-documentation.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Terraform Documentation Airbyte's Terraform provider enables you to automate & version-control your Airbyte configuration as code. Save time managing Airbyte and collaborate on Airbyte configuration changes with your teammates. Airbyte's Terraform provider is built off our [Airbyte API](https://api.airbyte.com). @@ -5,3 +9,5 @@ Airbyte's Terraform provider enables you to automate & version-control your Airb The Terraform provider is available for users on Airbyte Cloud, OSS & Self-Managed Enterprise. Check out our guide for [getting started with Airbyte's Terraform provider](https://reference.airbyte.com/reference/using-the-terraform-provider). + +Additionally, you can find examples of data stacks using the Terraform provider in our [quickstarts repository](https://github.com/airbytehq/quickstarts). diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index e436b24eada6..19f59a160b2f 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -26,18 +26,26 @@ Each of these concepts is described in greater depth in their respective section The Airbyte Protocol is versioned independently of the Airbyte Platform, and the version number is used to determine the compatibility between connectors and the Airbyte Platform. -| Version | Date of Change | Pull Request(s) | Subject | -|:---------|:---------------|:--------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------| -| `v1.0.0` | 2022-11-28 | [17486](https://github.com/airbytehq/airbyte/pull/17486) & [19846](https://github.com/airbytehq/airbyte/pull/19846) | Well known data types added | -| `v0.3.2` | 2022-10-28 | [18875](https://github.com/airbytehq/airbyte/pull/18875) | `AirbyteEstimateTraceMessage` added | -| `v0.3.1` | 2022-10-12 | [17907](https://github.com/airbytehq/airbyte/pull/17907) | `AirbyteControlMessage.ConnectorConfig` added | -| `v0.3.0` | 2022-09-09 | [16479](https://github.com/airbytehq/airbyte/pull/16479) | `AirbyteLogMessage.stack_trace` added | -| `v0.2.0` | 2022-06-10 | [13573](https://github.com/airbytehq/airbyte/pull/13573) & [12586](https://github.com/airbytehq/airbyte/pull/12586) | `STREAM` and `GLOBAL` STATE messages | -| `v0.1.1` | 2022-06-06 | [13356](https://github.com/airbytehq/airbyte/pull/13356) | Add a namespace in association with the stream name | -| `v0.1.0` | 2022-05-03 | [12458](https://github.com/airbytehq/airbyte/pull/12458) & [12581](https://github.com/airbytehq/airbyte/pull/12581) | `AirbyteTraceMessage` added to allow connectors to better communicate exceptions | -| `v0.0.2` | 2021-11-15 | [7798](https://github.com/airbytehq/airbyte/pull/7798) | Support oAuth Connectors (internal) | -| `v0.0.1` | 2021-11-19 | [1021](https://github.com/airbytehq/airbyte/pull/1021) | Remove sub-JSON Schemas | -| `v0.0.0` | 2020-11-18 | [998](https://github.com/airbytehq/airbyte/pull/998) | Initial version described via JSON Schema | +| Version | Date of Change | Pull Request(s) | Subject | +|:---------|:---------------|:--------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------| +| `v0.5.2` | 2023-12-26 | [58](https://github.com/airbytehq/airbyte-protocol/pull/58) | Remove unused V1. | +| `v0.5.1` | 2023-04-12 | [53](https://github.com/airbytehq/airbyte-protocol/pull/53) | Modify various helper libraries. | +| `v0.5.0` | 2023-11-13 | [49](https://github.com/airbytehq/airbyte-protocol/pull/49) | `AirbyteStateStatsMessage` added. | +| `v0.4.2` | 2023-04-12 | [46](https://github.com/airbytehq/airbyte-protocol/pull/46) | `AirbyteAnalyticsTraceMessage` added. | +| `v0.4.1` | 2023-08-14 | [41](https://github.com/airbytehq/airbyte-protocol/pull/41) & [44](https://github.com/airbytehq/airbyte-protocol/pull/44) | Various bug fixes. | +| `v0.3.6` | 2023-04-21 | [34](https://github.com/airbytehq/airbyte-protocol/pull/34) | Add explicit `AirbyteStreamStatus` statue values. | +| `v0.3.5` | 2023-04-13 | [30](https://github.com/airbytehq/airbyte-protocol/pull/30) | Fix indentation. | +| `v0.3.4` | 2023-04-13 | [28](https://github.com/airbytehq/airbyte-protocol/pull/28) | Fix Indentation. | +| `v0.3.3` | 2023-04-12 | [18](https://github.com/airbytehq/airbyte-protocol/pull/18) | `AirbyteStreamStatusMessage` added. | +| `v0.3.2` | 2022-10-28 | [18875](https://github.com/airbytehq/airbyte/pull/18875) | `AirbyteEstimateTraceMessage` added. | +| `v0.3.1` | 2022-10-12 | [17907](https://github.com/airbytehq/airbyte/pull/17907) | `AirbyteControlMessage.ConnectorConfig` added. | +| `v0.3.0` | 2022-09-09 | [16479](https://github.com/airbytehq/airbyte/pull/16479) | `AirbyteLogMessage.stack_trace` added. | +| `v0.2.0` | 2022-06-10 | [13573](https://github.com/airbytehq/airbyte/pull/13573) & [12586](https://github.com/airbytehq/airbyte/pull/12586) | `STREAM` and `GLOBAL` STATE messages. | +| `v0.1.1` | 2022-06-06 | [13356](https://github.com/airbytehq/airbyte/pull/13356) | Add a namespace in association with the stream name. | +| `v0.1.0` | 2022-05-03 | [12458](https://github.com/airbytehq/airbyte/pull/12458) & [12581](https://github.com/airbytehq/airbyte/pull/12581) | `AirbyteTraceMessage` added to allow connectors to better communicate exceptions. | +| `v0.0.2` | 2021-11-15 | [7798](https://github.com/airbytehq/airbyte/pull/7798) | Support oAuth Connectors (internal). | +| `v0.0.1` | 2021-11-19 | [1021](https://github.com/airbytehq/airbyte/pull/1021) | Remove sub-JSON Schemas. | +| `v0.0.0` | 2020-11-18 | [998](https://github.com/airbytehq/airbyte/pull/998) | Initial version described via JSON Schema. | ## Actor Interface @@ -100,7 +108,12 @@ The `spec` command allows an actor to broadcast information about itself and how check(Config) -> AirbyteConnectionStatus ``` -The `check` command validates that, given a configuration, that the Actor is able to connect and access all resources that it needs in order to operate. e.g. Given some Postgres credentials, it determines whether it can connect to the Postgres database. If it can, it will return a success response. If it fails (perhaps the password is incorrect), it will return a failed response and (when possible) a helpful error message. If an actor's `check` command succeeds, it is expected that all subsequent methods in the sync will also succeed. +The `check` command validates that, given a configuration, that the Actor is able to connect and access all resources that it needs in order to operate. e.g. Given some Postgres credentials, it determines whether it can connect to the Postgres database. The output will be as follows: +- If it can, the `check` command will return a success response. +- If `check` fails because of a configuration issue (perhaps the password is incorrect), it will return a failed response and (when possible) a helpful error message. A failed response will be considered as a config error, i.e. user error. Outputting a trace message detailing the config error is optional, but allows for more detailed debugging of the error. +- If it fails because of a connector issue, the `check` command should output a trace message detailing the failure. It is not expected to receive an `AirbyteConnectionStatus` in this failure case. + +If an actor's `check` command succeeds, it is expected that all subsequent methods in the sync will also succeed. #### Input: @@ -171,7 +184,7 @@ For the sake of brevity, we will not re-describe `spec` and `check`. They are ex #### Input: 1. `config` - A configuration JSON object that has been validated using the `ConnectorSpecification`. -2. `catalog` - An `AirbyteCatalog`. This `catalog` should be a subset of the `catalog` returned by the `discover` command. Any `AirbyteRecordMessages`s that the destination receives that do _not_ match the structure described in the `catalog` will fail. +2. `configured catalog` - A [`ConfiguredAirbyteCatalog`](https://docs.airbyte.com/understanding-airbyte/beginners-guide-to-catalog#configuredairbytecatalog). This is a modified version of the `catalog` returned by the `discover` command. Any `AirbyteRecordMessages`s that the destination receives that do _not_ match the structure described in the `catalog` will fail. 3. `message stream` - \(this stream is consumed on stdin--it is not passed as an arg\). It will receive a stream of JSON-serialized `AirbyteMesssage`. #### Output: @@ -449,6 +462,8 @@ This concept enables incremental syncs--syncs that only replicate data that is n State also enables Partial Success. In the case where during a sync there is a failure before all data has been extracted and committed, if all records up to a certain state are committed, then the next time the sync happens, it can start from that state as opposed to going back to the beginning. Partial Success is powerful, because especially in the case of high data volumes and long syncs, being able to pick up from wherever the failure occurred can costly re-syncing of data that has already been replicated. +The state for an actor is emitted as a black box by the Source. When emitted it is wrapped in the [AirbyteStateMessage](#airbytestatemessage). The contents of the `data` field is what is passed to the Source on start up. It is up to the Source to interpret the state object. Nothing outside the Source can make any inference about the state of the object EXCEPT, if it is null, it can be concluded that there is no state and the Source will start at the beginning. + ### State & Source This section will step through how state is used to allow a Source to pick up where it left off. A Source takes state as an input. A Source should be able to take that input and use it to determine where it left off the last time. The contents of the Source is a black box to the Protocol. The Protocol provides an envelope for the Source to put its state in and then passes the state back in that envelope. The Protocol never needs to know anything about the contents of the state. Thus, the Source can track state however makes most sense to it. @@ -478,19 +493,14 @@ The normal success case (T3, not depicted) would be that all the records would m -- [link](https://whimsical.com/state-TYX5bSCVtVF4BU1JbUwfpZ) to source image -### V1 - -The state for an actor is emitted as a complete black box. When emitted it is wrapped in the [AirbyteStateMessage](#airbytestatemessage-v1). The contents of the `data` field is what is passed to the Source on start up. This gives the Source lead to decide how to track the state of each stream. That being said, a common pattern is a `Map`. Nothing outside the source can make any inference about the state of the object EXCEPT, if it is null, it can be concluded that there is no state and the Source will start at the beginning. - -### V2 (coming soon!) - -In addition to allowing a Source to checkpoint data replication, the state object is structure to allow for the ability to configure and reset streams in isolation from each other. For example, if adding or removing a stream, it is possible to do so without affecting the state of any other stream in the Source. +### State Types +In addition to allowing a Source to checkpoint data replication, the state object allows for the ability to configure and reset streams in isolation from each other. For example, if adding or removing a stream, it is possible to do so without affecting the state of any other stream in the Source. There are 3 types of state: Stream, Global, and Legacy. - **Stream** represents Sources where there is complete isolation between stream states. In these cases, the state for each stream will be emitted in its own state message. In other words, if there are 3 streams replicated during a sync, the Source would emit at least 3 state message (1 per stream). The state of the Source is the sum of all the stream states. - **Global** represents Sources where this shared state across streams. In these cases each state message contains the whole state for the connection. The `shared_state` field is where any information that is shared across streams must go. The `stream_states` field contains a list of objects that contain a Stream Descriptor and the state information for that stream that is stream-specific. There are drawbacks to this state type, so it should only be used in cases where a shared state between streams is unavoidable. -- **Legacy** exists for backwards compatibility. In this state type, the state object is totally a black box. The only inference tha can be drawn from the state object is that if it is null, then there is no state for the entire Source. All current legacy cases can be ported to stream or global. Once they are, it will be removed. +- **Legacy** exists for backwards compatibility. In this state type, the state object is totally a black box. The only inference tha can be drawn from the state object is that if it is null, then there is no state for the entire Source. **All current legacy cases are being ported to stream or global. Once they are, it will be removed.** This table breaks down attributes of these state types. @@ -500,11 +510,45 @@ This table breaks down attributes of these state types. | Stream-Level Replication Isolation | X | | | | Single state message describes full state for Source | | X | X | -- **Protocol Version** simply connotes which versions of the Protocol have support for these State types. The new state message is backwards compatible with the V1 message. This allows old versions of connectors and platforms to interact with the new message. - **Stream-Level Configuration / Reset** was mentioned above. The drawback of the old state struct was that it was not possible to configure or reset the state for a single stream without doing it for all of them. Thus, new state types support this, but the legacy one cannot. - **Stream-Level Replication Isolation** means that a Source could be run in parallel by splitting up its streams across running instances. This is only possible for Stream state types, because they are the only state type that can update its current state completely on a per-stream basis. This is one of the main drawbacks of Sources that use Global state; it is not possible to increase their throughput through parallelization. - **Single state message describes full state for Source** means that any state message contains the full state information for a Source. Stream does not meet this condition because each state message is scoped by stream. This means that in order to build a full picture of the state for the Source, the state messages for each configured stream must be gathered. +### State Principles +The following are principles Airbyte recommends Sources/Destinations adhere to with State. Airbyte enforces these principles via our CDK. + +These principles are intended to produce simple overall system behavior, and move Airbyte towards a world of shorter-lived jobs. The goal is reliable data movement with minimal data loss windows on errors. + +1. **New Sources must use per-stream/global State**. + + Per-stream/Global state unlocks more granular State operations e.g. per-stream resets, per-stream parallelisation etc. No new Connectors should be created using Legacy state. + +2. **Sources always emit State, regardless of sync mode.** + + This simplifies how the Platform treats jobs and means all Syncs are resumable. This also enables checkpointing on full refreshes in the future. This rule does not appear to Sources that do not support cursors. + However: + 1. If the source stream has no records, an empty state should still be emitted. This supports state-based counts/checksums. It is recommended for the emitted state to have unique and non-null content. + 2. If the stream is unsorted, and therefore non-resumable, it is recommended to still send a state message, even with bogus resumability, to indicate progress in the sync. + +3. **Sources do not emit sequential duplicate States with interleaved records.** + + Duplicate States make it challenging to debug state-related operations. E.g. Is this a duplicate or did we fail to properly update state? Is this a duplicate log? Sync will fail if this rule is violated. + +4. **Sources should emit state whenever it is meaningful to resume a failed sync. Platform reserves the right to discard too frequent State emission per internal platform rules.** + + Sources should strive to emit state as fast as it’s useful. Platform can discard this state if this leads to undesirable downstream behavior e.g. out of memory. This is fine as there is increasingly lower marginal value to emitting States at higher frequencies. + +5. **Platform & Destinations treat state as a black box.** + + Sources are the sole producer/consumer of a State messages’ contents. Precisely, this refers to the state fields within the various State messages. Modifying risks corrupting our data sync cursor, which is a strict no-no. + +6. **Destinations return state in the order it was received.** + + Order is used by the Platform to determine if a State message was dropped. Out-of-order State messages throw errors, as do skipped state messages. Every state message the destination recieved must be returned back to the platform, in order. + + Order-ness is determined by the type of State message. Per-stream state messages require order per-stream. Global state messages require global ordering. + + ## Messages ### Common @@ -615,26 +659,7 @@ AirbyteRecordMessage: type: integer ``` -### AirbyteStateMessage (V1) - -The state message enables the Source to emit checkpoints while replicating data. These checkpoints mean that if replication fails before completion, the next sync is able to start from the last checkpoint instead of returning to the beginning of the previous sync. The details of this process are described in [State & Checkpointing](#state--checkpointing). - -The state message is a wrapper around the state that a Source emits. The state that the Source emits is treated as a black box by the protocol--it is modeled as a JSON blob. - -```yaml -AirbyteStateMessage: - type: object - additionalProperties: true - required: - - data - properties: - data: - description: "the state data" - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode -``` - -### AirbyteStateMessage (V2 -- coming soon!) +### AirbyteStateMessage The state message enables the Source to emit checkpoints while replicating data. These checkpoints mean that if replication fails before completion, the next sync is able to start from the last checkpoint instead of returning to the beginning of the previous sync. The details of this process are described in [State & Checkpointing](#state--checkpointing). diff --git a/docs/understanding-airbyte/cdc.md b/docs/understanding-airbyte/cdc.md index ee42b9a460b8..fa012c6d5366 100644 --- a/docs/understanding-airbyte/cdc.md +++ b/docs/understanding-airbyte/cdc.md @@ -33,7 +33,7 @@ We add some metadata columns for CDC sources: * [Postgres](../integrations/sources/postgres.md) \(For a quick video overview of CDC on Postgres, click [here](https://www.youtube.com/watch?v=NMODvLgZvuE&ab_channel=Airbyte)\) * [MySQL](../integrations/sources/mysql.md) * [Microsoft SQL Server / MSSQL](../integrations/sources/mssql.md) - +* [MongoDB](../integrations/sources/mongodb-v2.md) \(More information on [Mongodb CDC: How to Sync in Near Real-Time](https://airbyte.com/data-engineering-resources/mongodb-cdc)\) ## Coming Soon * Oracle DB diff --git a/docs/understanding-airbyte/high-level-view.md b/docs/understanding-airbyte/high-level-view.md index ce45613163db..19cb5291da76 100644 --- a/docs/understanding-airbyte/high-level-view.md +++ b/docs/understanding-airbyte/high-level-view.md @@ -11,16 +11,35 @@ The platform provides all the horizontal services required to configure and run Connectors are independent modules which push/pull data to/from sources and destinations. Connectors are built in accordance with the [Airbyte Specification](./airbyte-protocol.md), which describes the interface with which data can be moved between a source and a destination using Airbyte. Connectors are packaged as Docker images, which allows total flexibility over the technologies used to implement them. A more concrete diagram can be seen below: - -![3.048-Kilometer view](../.gitbook/assets/understanding_airbyte_high_level_architecture.png) - -* `UI`: An easy-to-use graphical interface for interacting with the Airbyte API. -* `WebApp Server`: Handles connection between UI and API. -* `Config Store`: Stores all the connections information \(credentials, frequency...\). -* `Scheduler Store`: Stores statuses and job information for the scheduler bookkeeping. -* `Config API`: Airbyte's main control plane. All operations in Airbyte such as creating sources, destinations, connections, managing configurations, etc.. are configured and invoked from the API. -* `Scheduler`: The scheduler takes work requests from the API and sends them to the Temporal service to parallelize. It is responsible for tracking success/failure and for triggering syncs based on the configured frequency. -* `Temporal Service`: Manages the task queue and workflows for the Scheduler. -* `Worker`: The worker connects to a source connector, pulls the data and writes it to a destination. -* `Temporary Storage`: A storage that workers can use whenever they need to spill data on a disk. - +```mermaid +--- +title: Architecture Overview +config: + theme: neutral +--- +flowchart LR + W[fa:fa-display WebApp/UI] + S[fa:fa-server Server/Config API] + D[(fa:fa-table Config & Jobs)] + T(fa:fa-calendar Temporal) + W2[1..n Airbyte Workers] + W -->|sends API requests| S + S -->|store data| D + S -->|create workflow| T + T -->|launch task| W2 + W2 -->|return job| T + W2 -->|launches| Source + W2 -->|launches| Destination +``` + +* **Web App/UI** [`airbyte-webapp`, `airbyte-proxy`]: An easy-to-use graphical interface for interacting with the Airbyte API. +* **Server/Config API** [`airbyte-server`, `airbyte-server-api`]: Handles connection between UI and API. Airbyte's main control plane. All operations in Airbyte such as creating sources, destinations, connections, managing configurations, etc.. are configured and invoked from the API. +* **Database Config & Jobs** [`airbyte-db`]: Stores all the connections information \(credentials, frequency...\). +* **Temporal Service** [`airbyte-temporal`]: Manages the task queue and workflows. +* **Worker** [`airbyte-worker`]: The worker connects to a source connector, pulls the data and writes it to a destination. + +The diagram shows the steady-state operation of Airbyte, there are components not described you'll see in your deployment: +* **Cron** [`airbyte-cron`]: Clean the server and sync logs (when using local logs) +* **Bootloader** [`airbyte-bootloader`]: Upgrade and Migrate the Database tables and confirm the enviroment is ready to work. + +This is a holistic high-level description of each component. For Airbyte deployed in Kubernetes the structure is very similar with a few changes. diff --git a/docs/understanding-airbyte/schemaless-sources-and-destinations.md b/docs/understanding-airbyte/schemaless-sources-and-destinations.md new file mode 100644 index 000000000000..edd4051ce2ca --- /dev/null +++ b/docs/understanding-airbyte/schemaless-sources-and-destinations.md @@ -0,0 +1,62 @@ +# "Schemaless" Sources and Destinations +In order to run a sync, Airbyte requires a [catalog](/understanding-airbyte/airbyte-protocol#catalog), which includes a data schema describing the shape of data being emitted by the source. +This schema will be used to prepare the destination to populate the data during the sync. + +While having a [strongly-typed](/understanding-airbyte/supported-data-types) catalog/schema is possible for most sources, some won't have a reasonably static schema. This document describes the options available for the subset of sources that do not have a strict schema, aka "schemaless sources". + +## What is a Schemaless Source? +Schemaless sources are sources for which there is no requirement or expectation that records will conform to a particular pattern. +For example, in a MongoDB database, there's no requirement that the fields in one document are the same as the fields in the next, or that the type of value in one field is the same as the type for that field in a separate document. +Similarly, for a file-based source such as S3, the files that are present in your source may not all have the same schema. + +Although the sources themselves may not conform to an obvious schema, Airbyte still needs to know the shape of the data in order to prepare the destination for the records. +For these sources, during the [`discover`](/understanding-airbyte/airbyte-protocol#discover) method, Airbyte offers two options to create the schema: + +1. Dynamic schema inference. +2. A hardcoded "schemaless" schema. + +### Dynamic schema inference +If this option is selected, Airbyte will infer the schema dynamically based on the contents of the source. +If your source's content is homogenous, we recommend this option, as the data in your destination will be typed and you can make use of schema evolution features, column selection, and similar Airbyte features which operate against the source's schema. + +For MongoDB, you can configure the number of documents that will be used for schema inference (from 1,000 to 10,000 documents; by default, this is set to 10,000). +Airbyte will read in the requested number of documents (sampled randomly) and infer the schema from them. +For file-based sources, we look at up to 10 files (reading up to 1MB per file) and infer the schema based on the contents of those files. + +In both cases, as the contents of the source change, the schema can change too. + +The schema that's produced from the inference procedure will include all the top-level fields that were observed in the sampled records. +The type assigned to each field will be the widest type observed for that field in any of the sampled data. +So if we observe that a field has an integer type in one record and a string in another, the schema will identify the field as a string. + +There are a few drawbacks to be aware of: +- If your dataset is very large, the `discover` process can be very time-consuming. +- Because we may not use 100% of the available data to create the schema, your schema may not contain every field present in your records. + Airbyte only syncs fields that are in the schema, so you may end up with incomplete data in the destination. + +If your data set is very large or you anticipate that it will change often, we recommend using the "schemaless" schema to avoid these issues. + +_Note: For MongoDB, knowing how variable your dataset is can help you choose an appropriate value for the number of documents to use for schema inference. +If your data is uniform across all or most records, you can set this to a lower value, providing better performance on discover and during the sync. +If your data varies but you cannot use the Schemaless option, you can set it to a larger value to ensure that as many fields as possible are accounted for._ + +### Schemaless schema +If this option is selected, the schema will always be `{"data": object}`, regardless of the contents of the data. +During the sync, we "wrap" each record behind a key named `data`. +This means that the destination receives the data with one top-level field only, and the value of the field is the entire record. +This option avoids a time-consuming or inaccurate `discover` phase and guarantees that everything ends up in your destination, at the expense of Airbyte being able to structure the data into different columns. + +## Future Enhancements + +### File-based Sources: configurable amount of data read for schema inference +Currently, Airbyte chooses the amount of data that we'll use to infer the schema for file-based sources. +We will be surfacing a config option for users to choose how much data to read to infer the schema. + +This option is already available for the MongoDB source. + +### Unwrapping the data at schemaless Destinations +MongoDB and file storage systems also don't require a schema at the destination. +For this reason, if you are syncing data from a schemaless source to a schemaless destination and chose the "schemaless" schema option, Airbyte will offer the ability to "unwrap" the data at the destination so that it is not nested under the "data" key. + +### Column exclusion for schemaless schemas +We are planning to offer a way to exclude fields from being synced when the schemaless option is selected, as column selection is not applicable. diff --git a/docs/understanding-airbyte/tech-stack.md b/docs/understanding-airbyte/tech-stack.md index c829f8b7a81b..c21aea1b8dd8 100644 --- a/docs/understanding-airbyte/tech-stack.md +++ b/docs/understanding-airbyte/tech-stack.md @@ -13,7 +13,7 @@ Connectors can be written in any language. However the most common languages are: -* Python 3.9.0 +* Python 3.9 or higher * [Java 17](https://jdk.java.net/archive/) ## **Frontend** diff --git a/docs/using-airbyte/airbyte-lib/getting-started.mdx b/docs/using-airbyte/airbyte-lib/getting-started.mdx new file mode 100644 index 000000000000..50baa2320b10 --- /dev/null +++ b/docs/using-airbyte/airbyte-lib/getting-started.mdx @@ -0,0 +1,69 @@ +import AirbyteLibConnectors from '@site/src/components/AirbyteLibConnectors'; + +# Getting Started with PyAirbyte (Beta) + +PyAirbyte is a library that provides a set of utilities to use Airbyte connectors in Python. It is meant to be used in situations where setting up an Airbyte server or cloud account is not possible or desirable, for example in a Jupyter notebook or when iterating on early prototypes on a developer's workstation. + +## Installation + +```bash +pip install airbyte +``` + +Or during the beta, you may want to install the latest from from source with: + +```bash +pip install 'git+https://github.com/airbytehq/PyAirbyte.git' +``` + +## Usage + +Data can be extracted from sources and loaded into caches: + +      Try with Colab + +```python +import airbyte as ab + +source = ab.get_source( + "source-faker", + config={"count": 5_000}, + install_if_missing=True, +) +source.check() +source.select_all_streams() +result = source.read() + +for name, records in result.streams.items(): + print(f"Stream {name}: {len(list(records))} records") +``` + +## Quickstarts + +* [CoinAPI](https://github.com/airbytehq/quickstarts/blob/main/airbyte_lib_notebooks/AirbyteLib_CoinAPI_Demo.ipynb) +* [GA4](https://github.com/airbytehq/quickstarts/blob/main/airbyte_lib_notebooks/AirbyteLib_GA4_Demo.ipynb) +* [Shopify](https://github.com/airbytehq/quickstarts/blob/main/airbyte_lib_notebooks/PyAirbyte_Shopify_Demo.ipynb) +* [GitHub](https://github.com/airbytehq/quickstarts/blob/main/airbyte_lib_notebooks/AirbyteLib_Github_Incremental_Demo.ipynb) +* [Postgres (cache)](https://github.com/airbytehq/quickstarts/blob/main/airbyte_lib_notebooks/PyAirbyte_Postgres_Custom_Cache_Demo.ipynb) + + +## API Reference + +For details on specific classes and methods, please refer to our [PyAirbyte API Reference](https://airbytehq.github.io/PyAirbyte/index.html). + +## Architecture + +[comment]: <> (Edit under https://docs.google.com/drawings/d/1M7ti2D4ha6cEtPnk04RLp1SSh3au4dRJsLupnGPigHQ/edit?usp=sharing) + +![Architecture](../../assets/docs/airbyte-lib-high-level-architecture.svg) + +PyAirbyte is a python library that can be run in any context that supports Python >=3.9. It contains the following main components: +* **Source**: A source object is using a Python connector and includes a configuration object. The configuration object is a dictionary that contains the configuration of the connector, like authentication or connection modalities. The source object is used to read data from the connector. +* **Cache**: Data can be read directly from the source object. However, it is recommended to use a cache object to store the data. The cache object allows to temporarily store records from the source in a SQL database like a local DuckDB file or a Postgres or Snowflake instance. +* **Result**: An object holding the records from a read operation on a source. It allows quick access to the records of each synced stream via the used cache object. Data can be accessed as a list of records, a Pandas DataFrame or via SQLAlchemy queries. + +## Available connectors + +The following connectors are available: + + diff --git a/docs/using-airbyte/core-concepts/basic-normalization.md b/docs/using-airbyte/core-concepts/basic-normalization.md index b76d4759de54..16de09002ecc 100644 --- a/docs/using-airbyte/core-concepts/basic-normalization.md +++ b/docs/using-airbyte/core-concepts/basic-normalization.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Basic Normalization :::danger diff --git a/docs/using-airbyte/core-concepts/namespaces.md b/docs/using-airbyte/core-concepts/namespaces.md index fb1e2b895bd7..ce7c8532d91e 100644 --- a/docs/using-airbyte/core-concepts/namespaces.md +++ b/docs/using-airbyte/core-concepts/namespaces.md @@ -1,14 +1,16 @@ -# Namespaces +--- +products: all +--- -## High-Level Overview +# Namespaces Namespaces are used to generally organize data, separate tests and production data, and enforce permissions. In most cases, namespaces are schemas in the database you're replicating to. As a part of connection setup, you select where in the destination you want to write your data. Note: The default configuration is **Destination default**. -| Destination Namepsace | Description | +| Destination Namespace | Description | | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| Destination default | All streams will be replicated to the single default namespace defined by the Destination. | +| Destination default | All streams will be replicated to the single default namespace defined in the Destination's settings. | | Mirror source structure | Some sources (for example, databases) provide namespace information for a stream. If a source provides namespace information, the destination will mirror the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. | | Custom format | All streams will be replicated to a single user-defined namespace. See Custom format for more details | @@ -54,13 +56,17 @@ When replicating multiple sources into the same destination, you may create tabl For example, a Github source can be replicated into a `github` schema. However, you may have multiple connections writing from different GitHub repositories \(common in multi-tenant scenarios\). :::tip -To keep the same table names, Airbyte recommends writing the connections to unique namespaces to avoid mixing data from the different GitHub repositories. +To write more than 1 table with the same name to your destination, Airbyte recommends writing the connections to unique namespaces to avoid mixing data from the different GitHub repositories. ::: You can enter plain text (most common) or additionally add a dynamic parameter `${SOURCE_NAMESPACE}`, which uses the namespace provided by the source if available. ### Examples +:::info +If the Source does not support namespaces, the data will be replicated into the Destination's default namespace. If the Destination does not support namespaces, any preference set in the connection is ignored. +::: + The following table summarises how this works. In this example, we're looking at the replication configuration between a Postgres Source and Snowflake Destination \(with settings of schema = "my\_schema"\): | Namespace Configuration | Source Namespace | Source Table Name | Destination Namespace | Destination Table Name | @@ -74,21 +80,15 @@ The following table summarises how this works. In this example, we're looking at | Custom format = `"my\_${SOURCE\_NAMESPACE}\_schema"` | public | my\_table | my\_public\_schema | my\_table | | Custom format = " " | public | my\_table | my\_schema | my\_table | -## Syncing Details - -If the Source does not support namespaces, the data will be replicated into the Destination's default namespace. For databases, the default namespace is the schema provided in the destination configuration. - -If the Destination does not support namespaces, any preference set in the connection is ignored. - ## Using Namespaces with Basic Normalization -As part of the connections sync settings, it is possible to configure the namespace used by: 1. destination connectors: to store the `_airbyte_raw_*` tables. 2. basic normalization: to store the final normalized tables. +As part of the connection settings, it is possible to configure the namespace used by: 1. destination connectors: to store the `_airbyte_raw_*` tables. 2. basic normalization: to store the final normalized tables. -:::info When basic normalization is enabled, this is the location that both your normalized and raw data will get written to. Your raw data will show up with the prefix `_airbyte_raw_` in the namespace you define. If you don't enable basic normalization, you will only receive the raw tables. -:::note +:::note Note custom transformation outputs are not affected by the namespace settings from Airbyte: It is up to the configuration of the custom dbt project, and how it is written to handle its [custom schemas](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-schemas). The default target schema for dbt in this case, will always be the destination namespace. +::: ## Requirements diff --git a/docs/using-airbyte/core-concepts/readme.md b/docs/using-airbyte/core-concepts/readme.md index e8c336dba5d0..09398b25a618 100644 --- a/docs/using-airbyte/core-concepts/readme.md +++ b/docs/using-airbyte/core-concepts/readme.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Core Concepts Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, and how the data is written to in the destination. @@ -20,12 +24,12 @@ An Airbyte component which pulls data from a source or pushes data to a destinat A connection is an automated data pipeline that replicates data from a source to a destination. Setting up a connection enables configuration of the following parameters: -| Concept | Description | -|---------------------|---------------------------------------------------------------------------------------------------------------------| -| [Replication Frequency](/using-airbyte/core-concepts/sync-schedules.md) | When should a data sync be triggered? | -| [Destination Namespace and Stream Prefix](/using-airbyte/core-concepts/namespaces.md) | Where should the replicated data be written? | -| [Sync Mode](/using-airbyte/core-concepts/sync-modes/README.md) | How should the streams be replicated (read and written)? | -| [Schema Propagation](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How should Airbyte handle schema drift in sources? | +| Concept | Description | +|-----------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| [Replication Frequency](/using-airbyte/core-concepts/sync-schedules.md) | When should a data sync be triggered? | +| [Destination Namespace and Stream Prefix](/using-airbyte/core-concepts/namespaces.md) | Where should the replicated data be written? | +| [Sync Mode](/using-airbyte/core-concepts/sync-modes/README.md) | How should the streams be replicated (read and written)? | +| [Schema Propagation](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How should Airbyte handle schema drift in sources? | | [Catalog Selection](/cloud/managing-airbyte-cloud/configuring-connections.md#modify-streams-in-your-connection) | What data should be replicated from the source to the destination? | ## Stream @@ -79,7 +83,7 @@ Typing and deduping ensures the data emitted from sources is written into the co Typing and Deduping is the default method of transforming datasets within data warehouse and database destinations after they've been replicated. We are retaining documentation about normalization to support legacy destinations. ::: -For more details, see our [Typing & Deduping documentation](/understanding-airbyte/typing-deduping). +For more details, see our [Typing & Deduping documentation](/using-airbyte/core-concepts/typing-deduping). ## Basic Normalization diff --git a/docs/using-airbyte/core-concepts/sync-modes/README.md b/docs/using-airbyte/core-concepts/sync-modes/README.md index a561506a1f73..be548d4d04d2 100644 --- a/docs/using-airbyte/core-concepts/sync-modes/README.md +++ b/docs/using-airbyte/core-concepts/sync-modes/README.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Sync Modes A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. To minimize confusion, a mode's behavior is reflected in its name. The easiest way to understand Airbyte's sync modes is to understand how the modes are named. diff --git a/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md index ccdd7951bbe5..1bdd03f8ddee 100644 --- a/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md +++ b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Full Refresh - Append ## Overview diff --git a/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md index 6de7d266c9ce..17204cafcd59 100644 --- a/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md +++ b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md @@ -1,10 +1,14 @@ +--- +products: all +--- + # Full Refresh - Overwrite ## Overview The **Full Refresh** modes are the simplest methods that Airbyte uses to sync data, as they always retrieve all available information requested from the source, regardless of whether it has been synced before. This contrasts with [**Incremental sync**](./incremental-append.md), which does not sync data that has already been synced before. -In the **Overwrite** variant, new syncs will destroy all data in the existing destination table and then pull the new data in. Therefore, data that has been removed from the source after an old sync will be deleted in the destination table. +In the **Overwrite** variant, new syncs will destroy all data in the existing destination table and then pull the new data in. Therefore, data that has been removed from the source after an old sync will be deleted in the destination table. ## Example Behavior @@ -18,8 +22,9 @@ data in the destination _before_ the sync: | :--- | | Python | | Java | +| Bash| -new data: +new data in the source: | Languages | | :--- | @@ -27,7 +32,7 @@ new data: | Java | | Ruby | -data in the destination _after_ the sync: +data in the destination _after_ the sync (note how the old value of "bash" is no longer present): | Languages | | :--- | @@ -35,11 +40,9 @@ data in the destination _after_ the sync: | Java | | Ruby | -Note: This is how Singer target-bigquery does it. - -## In the future +## Destination-specific mechinisims for full refresh -We will consider making other flavors of full refresh configurable as first-class citizens in Airbyte. e.g. On new data, copy old data to a new table with a timestamp, and then replace the original table with the new data. As always, we will focus on adding these options in such a way that the behavior of each connector is both well documented and predictable. +The mechinisim by which a destination connector acomplishes the full refresh will vary wildly from destination to destinaton. For our certified database and data warehouse destinations, we will be recreating the final table each sync. This allows us leave the previous sync's data viewable by writing to a "final-table-tmp" location as the sync is running, and at the end dropping the olf "final" table, and renaming the new one into place. That said, this may not possible for all destinations, and we may need to erase the existing data at the start of each full-refresh sync. ## Related information diff --git a/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md b/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md index 6fa0272fda6e..7e3ad86a1b98 100644 --- a/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md +++ b/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Incremental Sync - Append + Deduped ## High-Level Context diff --git a/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md b/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md index c9facb4711f3..3a9c01859714 100644 --- a/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md +++ b/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Incremental Sync - Append ## Overview diff --git a/docs/using-airbyte/core-concepts/sync-schedules.md b/docs/using-airbyte/core-concepts/sync-schedules.md index a0d6c22fbee9..c4514d941396 100644 --- a/docs/using-airbyte/core-concepts/sync-schedules.md +++ b/docs/using-airbyte/core-concepts/sync-schedules.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Sync Schedules For each connection, you can select between three options that allow a sync to run. The three options for `Replication Frequency` are: @@ -10,7 +14,11 @@ For each connection, you can select between three options that allow a sync to r * Only one sync per connection can run at a time. * If a sync is scheduled to run before the previous sync finishes, the scheduled sync will start after the completion of the previous sync. -* Syncs can run at most every 60 minutes. Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour. +* Syncs can run at most every 60 minutes in Airbyte Cloud. Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour. + +:::note +For Scheduled or cron scheduled syncs, Airbyte guarantees syncs will initiate with a schedule accuracy of +/- 30 minutes. +::: ## Scheduled syncs When a scheduled connection is first created, a sync is executed immediately after creation. After that, a sync is run once the time since the last sync \(whether it was triggered manually or due to a schedule\) has exceeded the schedule interval. For example: @@ -23,17 +31,21 @@ When a scheduled connection is first created, a sync is executed immediately aft - **October 3rd, 5:01pm:** It has been more than 24 hours since the last sync, so a sync is run ## Cron Scheduling -If you prefer more flexibility in scheduling your sync, you can also use CRON scheduling to set a precise time of day or month. +If you prefer more precision in scheduling your sync, you can also use CRON scheduling to set a specific time of day or month. -Airbyte uses the CRON scheduler from [Quartz](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). We recommend reading their [documentation](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) to learn more about how to +Airbyte uses the CRON scheduler from [Quartz](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). We recommend reading their [documentation](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) to understand the required formatting. You can also refer to these examples: -When setting up the cron extpression, you will also be asked to choose a time zone the sync will run in. - -:::note -For Scheduled or cron scheduled syncs, Airbyte guarantees syncs will initiate with a schedule accuracy of +/- 30 minutes. -::: +| Cron string | Sync Timing| +| - | - | +| 0 0 * * * ? | Every hour, at 0 minutes past the hour | +| 0 0 15 * * ? | At 15:00 every day | +| 0 0 15 * * MON,TUE | At 15:00, only on Monday and Tuesday | +| 0 0 0,2,4,6 * * ? | At 12:00 AM, 02:00 AM, 04:00 AM and 06:00 AM every day | +| 0 0 */15 * * ? | At 0 minutes past the hour, every 15 hours | + +When setting up the cron expression, you will also be asked to choose a time zone the sync will run in. ## Manual Syncs When the connection is set to replicate with `Manual` frequency, the sync will not automatically run. -It can be triggered by clicking the "Sync Now" button at any time through the UI or be triggered through the UI. \ No newline at end of file +It can be triggered by clicking the "Sync Now" button at any time through the UI or be triggered through the API. \ No newline at end of file diff --git a/docs/using-airbyte/core-concepts/typing-deduping.md b/docs/using-airbyte/core-concepts/typing-deduping.md index b63bb2f73860..c0c6c57906bd 100644 --- a/docs/using-airbyte/core-concepts/typing-deduping.md +++ b/docs/using-airbyte/core-concepts/typing-deduping.md @@ -1,28 +1,41 @@ +--- +products: all +--- + # Typing and Deduping -This page refers to new functionality added by [Destinations V2](/release_notes/upgrading_to_destinations_v2/). Typing and deduping is the default method of transforming datasets within data warehouse and database destinations after they've been replicated. Please check each destination to learn if Typing and Deduping is supported. +This page refers to new functionality added by +[Destinations V2](/release_notes/upgrading_to_destinations_v2/). Typing and deduping is the default +method of transforming datasets within data warehouse and database destinations after they've been +replicated. Please check each destination to learn if Typing and Deduping is supported. ## What is Destinations V2? [Airbyte Destinations V2](/release_notes/upgrading_to_destinations_v2) provide: -- One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables. -- Improved per-row error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data. -- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your desired schema with raw data tables. -- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables when possible. No more waiting hours to see the first rows in your destination table. - -:::note - -Typing and Deduping may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is transformed and is not related to Airbyte credit usage. - -::: +- One-to-one table mapping: Data in one stream will always be mapped to one table in your data + warehouse. No more sub-tables. +- Improved per-row error handling with `_airbyte_meta`: Airbyte will now populate typing errors in + the `_airbyte_meta` column instead of failing your sync. You can query these results to audit + misformatted or unexpected data. +- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables + in the `airbyte_internal` schema. We no longer clutter your desired schema with raw data tables. +- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables + when possible. No more waiting hours to see the first rows in your destination table. ## `_airbyte_meta` Errors -"Per-row error handling" is a new paradigm for Airbyte which provides greater flexibility for our users. Airbyte now separates `data-moving problems` from `data-content problems`. Prior to Destinations V2, both types of errors were handled the same way: by failing the sync. Now, a failing sync means that Airbyte could not _move_ all of your data. You can query the `_airbyte_meta` column to see which rows failed for _content_ reasons, and why. This is a more flexible approach, as you can now decide how to handle rows with errors on a case-by-case basis. +"Per-row error handling" is a new paradigm for Airbyte which provides greater flexibility for our +users. Airbyte now separates `data-moving problems` from `data-content problems`. Prior to +Destinations V2, both types of errors were handled the same way: by failing the sync. Now, a failing +sync means that Airbyte could not _move_ all of your data. You can query the `_airbyte_meta` column +to see which rows failed for _content_ reasons, and why. This is a more flexible approach, as you +can now decide how to handle rows with errors on a case-by-case basis. :::tip -When using data downstream from Airbyte, we generally recommend you only include rows which do not have an error, e.g: + +When using data downstream from Airbyte, we generally recommend you only include rows which do not +have an error, e.g: ```sql -- postgres syntax @@ -33,10 +46,19 @@ SELECT COUNT(*) FROM _table_ WHERE json_array_length(_airbyte_meta ->> errors) = The types of errors which will be stored in `_airbyte_meta.errors` include: -- **Typing errors**: the source declared that the type of the column `id` should be an integer, but a string value was returned. -- **Size errors (coming soon)**: the source returned content which cannot be stored within this this row or column (e.g. [a Redshift Super column has a 16mb limit](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html)). Destinations V2 will allow us to trim records which cannot fit into destinations, but retain the primary key(s) and cursors and include "too big" error messages. +- **Typing errors**: the source declared that the type of the column `id` should be an integer, but + a string value was returned. +- **Size errors (coming soon)**: the source returned content which cannot be stored within this this + row or column (e.g. + [a Redshift Super column has a 16mb limit](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html)). + Destinations V2 will allow us to trim records which cannot fit into destinations, but retain the + primary key(s) and cursors and include "too big" error messages. -Depending on your use-case, it may still be valuable to consider rows with errors, especially for aggregations. For example, you may have a table `user_reviews`, and you would like to know the count of new reviews received today. You can choose to include reviews regardless of whether your data warehouse had difficulty storing the full contents of the `message` column. For this use case, `SELECT COUNT(*) from user_reviews WHERE DATE(created_at) = DATE(NOW())` is still valid. +Depending on your use-case, it may still be valuable to consider rows with errors, especially for +aggregations. For example, you may have a table `user_reviews`, and you would like to know the count +of new reviews received today. You can choose to include reviews regardless of whether your data +warehouse had difficulty storing the full contents of the `message` column. For this use case, +`SELECT COUNT(*) from user_reviews WHERE DATE(created_at) = DATE(NOW())` is still valid. ## Destinations V2 Example @@ -56,20 +78,23 @@ Consider the following [source schema](/integrations/sources/faker) for stream ` The data from one stream will now be mapped to one table in your schema as below: -#### Destination Table Name: _public.users_ +#### Final Destination Table Name: _public.users_ -| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_extracted_at | \_airbyte_meta | id | first_name | age | address | -| -------------------------------------------- | ---------------- | ---------------------- | ------------------------------------------------------------ | --- | ---------- | ---- | --------------------------------------- | +| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_extracted_at | \_airbyte_meta | id | first_name | age | address | +| -------------------------------------------- | ---------------- | ---------------------- | -------------------------------------------------------------- | --- | ---------- | ---- | ----------------------------------------- | | Successful typing and de-duping ⟶ | xxx-xxx-xxx | 2022-01-01 12:00:00 | `{}` | 1 | sarah | 39 | `{ city: “San Francisco”, zip: “94131” }` | | Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | 2022-01-01 12:00:00 | `{ errors: {[“fish” is not a valid integer for column “age”]}` | 2 | evan | NULL | `{ city: “Menlo Park”, zip: “94002” }` | -| Not-yet-typed ⟶ | | | | | | | | +| Not-yet-typed ⟶ | | | | | | | | -In legacy normalization, columns of [Airbyte type](/understanding-airbyte/supported-data-types/#the-types) `Object` in the Destination were "unnested" into separate tables. In this example, with Destinations V2, the previously unnested `public.users_address` table with columns `city` and `zip` will no longer be generated. +In legacy normalization, columns of +[Airbyte type](/understanding-airbyte/supported-data-types/#the-types) `Object` in the Destination +were "unnested" into separate tables. In this example, with Destinations V2, the previously unnested +`public.users_address` table with columns `city` and `zip` will no longer be generated. -#### Destination Table Name: _airbyte.raw_public_users_ (`airbyte.{namespace}_{stream}`) +#### Raw Destination Table Name: _airbyte_internal.raw_public\_\_users_ (`airbyte_internal.raw_{namespace}__{stream}`) -| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_data | \_airbyte_loaded_at | \_airbyte_extracted_at | -| -------------------------------------------- | ---------------- | ----------------------------------------------------------------------------------------- | -------------------- | ---------------------- | +| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_data | \_airbyte_loaded_at | \_airbyte_extracted_at | +| -------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------- | -------------------- | ---------------------- | | Successful typing and de-duping ⟶ | xxx-xxx-xxx | `{ id: 1, first_name: “sarah”, age: 39, address: { city: “San Francisco”, zip: “94131” } }` | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | | Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | `{ id: 2, first_name: “evan”, age: “fish”, address: { city: “Menlo Park”, zip: “94002” } }` | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | | Not-yet-typed ⟶ | zzz-zzz-zzz | `{ id: 3, first_name: “edward”, age: 35, address: { city: “Sunnyvale”, zip: “94003” } }` | NULL | 2022-01-01 13:00:00 | @@ -77,3 +102,50 @@ In legacy normalization, columns of [Airbyte type](/understanding-airbyte/suppor You also now see the following changes in Airbyte-provided columns: ![Airbyte Destinations V2 Column Changes](../../release_notes/assets/updated_table_columns.png) + +## On final table re-creation + +From time to time, Airbyte will drop and re-create the final table produced by a sync (sometimes +called a "soft reset"). This is done as transactionally as possible, and should be invisible to most +observers. This is done for a number of reasons, including: + +- **Schema Migrations** - Many destinations lack the ability to control column order, or cannot + alter one data type to another. Re-creating the table allows Airbyte to strictly control the + column order and data types of each column. +- **Transactional Full Refresh** - In order to keep your final tables consistently useful, when a + refresh or reset occurs, airbyte will erase the raw tables, and then build a new tmp final table + first. Airbyte attempts to do an atomic swap of old and tmp final tables, usually via a + rename at the last second. Otherwise, there would be a period of time where the final table is empty, which could + cause downstream issues. + +This means that additional permissions, constraints, views, or other rules you apply to the final +table outside of Airbyte could be lost during a sync. Many destinations provide ways to use roles or +wildcards to grant permissions to tables, which are better suited for this ELT process. We do not +recommend altering the final tables (e.g. adding constraints) as it may cause issues with the sync. + +### Manually triggering a final table re-creation + +In some cases, you need to manually run a soft reset - for example, if you accidentally delete some +records from the final table and want to repopulate them from the raw data. This can be done by: +1. Dropping the final table entirely (`DROP TABLE `) +1. Unsetting the raw table's `_airbyte_loaded_at` column + (`UPDATE airbyte_internal. SET _airbyte_loaded_at = NULL`) + 1. If you are using a nonstandard raw table schema, replace `airbyte_internal` with that schema. +1. And then running a sync. + +After the sync completes, your final table will be restored to its correct state. + +## Loading Data Incrementally to Final Tables + +:::note + +Typing and Deduping may cause an increase in your destination's compute cost. This cost will vary +depending on the amount of data that is transformed and is not related to Airbyte credit usage. +Enabling loading data incrementally to final tables may further increase this cost. + +::: + +V2 destinations may include the option "Enable Loading Data Incrementally to Final Tables". When +enabled your data will load into your final tables incrementally while your data is still being +synced. When Disabled (the default), your data loads into your final tables once at the end of a +sync. Note that this option only applies if you elect to create Final tables. diff --git a/docs/using-airbyte/getting-started/add-a-destination.md b/docs/using-airbyte/getting-started/add-a-destination.md index cc473d8384f3..4aa05d8970f2 100644 --- a/docs/using-airbyte/getting-started/add-a-destination.md +++ b/docs/using-airbyte/getting-started/add-a-destination.md @@ -1,10 +1,17 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + # Add a Destination Destinations are the data warehouses, data lakes, databases and analytics tools where you will load the data from your chosen source(s). The steps to setting up your first destination are very similar to those for [setting up a source](./add-a-source). -Once you've logged in to your Airbyte Open Source deployment, click on the **Destinations** tab in the navigation bar found on the left side of the dashboard. This will take you to the list of available destinations. +Once you've signed up for Airbyte Cloud or logged in to your Airbyte Open Source deployment, click on the **Destinations** tab in the navigation bar found on the left side of the dashboard. This will take you to the list of available destinations. -![Destination List](../../.gitbook/assets/add-a-destination/getting-started-destination-list.png) +![Destination List](./assets/getting-started-destination-list.png) You can use the provided search bar at the top of the page, or scroll down the list to find the destination you want to replicate data from. @@ -12,17 +19,34 @@ You can use the provided search bar at the top of the page, or scroll down the l You can filter the list of destinations by support level. Airbyte connectors are categorized in two support levels, Certified and Community. See our [Connector Support Levels](/integrations/connector-support-levels.md) page for more information on this topic. ::: -As an example, we'll be setting up a simple JSON file that will be saved on our local system as the destination. Select **Local JSON** from the list of destinations. This will take you to the destination setup page. + + + As an example, we'll be setting up a simple Google Sheets spreadsheet that will move data to a Google Sheet. Select **Google Sheets** from the list of destinations. This will take you to the destination setup page. + + ![Destination Page](./assets/getting-started-google-sheets-destination.png) + +:::info +Google Sheets imposes rate limits and hard limits on the amount of data it can receive. Only use Google Sheets as a destination for small, non-production use cases, as it is not designed for handling large-scale data operations. + +Read more about the [specific limitations](/integrations/destinations/google-sheets.md#limitations) in our Google Sheets documentation. +::: + + The left half of the page contains a set of fields that you will have to fill out. In the **Destination name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the destination (i.e., `Google Sheets`). -![Destination Page](../../.gitbook/assets/add-a-destination/getting-started-destination-page.png) + Authenticate into your Google account by clicking "Sign in with Google" and granting permissions to Airbyte. Because this is a simple Google Sheets destination, there is only one more required field, **Spreadsheet Link**. This is the path to your spreadsheet that can be copied directly from your browser. + + + As an example, we'll be setting up a simple JSON file that will be saved on our local system as the destination. Select **Local JSON** from the list of destinations. This will take you to the destination setup page. -The left half of the page contains a set of fields that you will have to fill out. In the **Destination name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the destination (i.e., `Local JSON`). + The left half of the page contains a set of fields that you will have to fill out. In the **Destination name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the destination (i.e., `Local JSON`). -Because this is a simple JSON file, there is only one more required field, **Destination Path**. This is the path in your local filesystem where the JSON file containing your data will be saved. In our example, if we set the path to `/my_first_destination`, the file will be saved in `/tmp/airbyte_local/my_first_destination`. + Because this is a simple JSON file, there is only one more required field, **Destination Path**. This is the path in your local filesystem where the JSON file containing your data will be saved. In our example, if we set the path to `/my_first_destination`, the file will be saved in `/tmp/airbyte_local/my_first_destination`. + + Each destination will have its own set of required fields to configure during setup. You can refer to your destination's provided setup guide on the right side of the page for specific details on the nature of each field. -:::info +:::tip Some destinations will also have an **Optional Fields** tab located beneath the required fields. You can open this tab to view and configure any additional optional parameters that exist for the source. These fields generally grant you more fine-grained control over your data replication, but you can safely ignore them. ::: diff --git a/docs/using-airbyte/getting-started/add-a-source.md b/docs/using-airbyte/getting-started/add-a-source.md index e5f59b2f7517..15f4ce57bd61 100644 --- a/docs/using-airbyte/getting-started/add-a-source.md +++ b/docs/using-airbyte/getting-started/add-a-source.md @@ -1,22 +1,27 @@ +--- +products: all +--- + # Add a Source Setting up a new source in Airbyte is a quick and simple process! When viewing the Airbyte UI, you'll see the main navigation bar on the left side of your screen. Click the **Sources** tab to bring up a list of all available sources. -![](../../.gitbook/assets/add-a-source/getting-started-source-list.png) + + -You can use the provided search bar, or simply scroll down the list to find the source you want to replicate data from. Let's use Google Sheets as an example. Clicking on the **Google Sheets** card will bring us to its setup page. +You can use the provided search bar, or simply scroll down the list to find the source you want to replicate data from. Let's use a demo source, Faker, as an example. Clicking on the **Sample Data (Faker)** card will bring us to its setup page. -![](../../.gitbook/assets/add-a-source/getting-started-source-page.png) +![](./assets/getting-started-faker-source.png) -The left half of the page contains a set of fields that you will have to fill out. In the **Source name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the source (ie, `Google Sheets`). +The left half of the page contains a set of fields that you will have to fill out. In the **Source name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the source (ie, `Sample Data (Faker)`). -Each connector in Airbyte will have its own set of authentication methods and configurable parameters. In the case of Google Sheets, you can select one of two authentication methods (OAuth2.0 or a Service Account Key), and must provide the link to the Google Sheet you want to replicate. You can always refer to your source's provided setup guide for specific instructions on filling out each field. +Each connector in Airbyte will have its own set of authentication methods and configurable parameters. In the case of Sample Data (Faker), you can adjust the number of records you want returned in your `Users` data, and optionally adjust additional configuration settings. You can always refer to your source's provided setup guide for specific instructions on filling out each field. :::info -Some sources will also have an **Optional Fields** tab. You can open this tab to view and configure any additional optional parameters that exist for the souce, but you do not have to do so to successfully set up the connector. +Some sources will have an **Optional Fields** tab. You can open this tab to view and configure any additional optional parameters that exist for the souce, but you do not have to do so to successfully set up the connector. ::: Once you've filled out all the required fields, click on the **Set up source** button and Airbyte will run a check to verify the connection. Happy replicating! -Can't find the connectors that you want? Try your hand at easily building one yourself using our [Connector Builder!](../../connector-development/connector-builder-ui/overview.md) +Can't find the connectors that you want? Try your hand at easily building one yourself using our [Connector Builder](../../connector-development/connector-builder-ui/overview.md)! diff --git a/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png b/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png new file mode 100644 index 000000000000..f034a559f026 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png b/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png new file mode 100644 index 000000000000..92921edd1dc4 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png differ diff --git a/docs/.gitbook/assets/add-a-destination/getting-started-destination-list.png b/docs/using-airbyte/getting-started/assets/getting-started-destination-list.png similarity index 100% rename from docs/.gitbook/assets/add-a-destination/getting-started-destination-list.png rename to docs/using-airbyte/getting-started/assets/getting-started-destination-list.png diff --git a/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png b/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png new file mode 100644 index 000000000000..ed8b9db12eff Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png b/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png new file mode 100644 index 000000000000..03c6a2aade97 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png differ diff --git a/docs/.gitbook/assets/add-a-source/getting-started-source-list.png b/docs/using-airbyte/getting-started/assets/getting-started-source-list.png similarity index 100% rename from docs/.gitbook/assets/add-a-source/getting-started-source-list.png rename to docs/using-airbyte/getting-started/assets/getting-started-source-list.png diff --git a/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png b/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png new file mode 100644 index 000000000000..fc7cc81d0ddc Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png differ diff --git a/docs/using-airbyte/getting-started/readme.md b/docs/using-airbyte/getting-started/readme.md index ab860999e2fb..7b43f108ed9a 100644 --- a/docs/using-airbyte/getting-started/readme.md +++ b/docs/using-airbyte/getting-started/readme.md @@ -1,8 +1,16 @@ +--- +products: all +--- + # Getting Started Getting started with Airbyte takes only a few steps! This page guides you through the initial steps to get started and you'll learn how to setup your first connection on the following pages. -You have two options to run Airbyte: Use **Airbyte Cloud** (recommended) or **self-host Airbyte** in your infrastructure. +You have two options to run Airbyte: Use **Airbyte Cloud** (recommended) or **self-manage Airbyte** in your infrastructure. + +:::tip +If you are have already deployed Airbyte or signed up for Airbyte Cloud, jump ahead to [set up a source](./add-a-source.md). +::: ## Sign Up for Airbyte Cloud @@ -12,21 +20,27 @@ Airbyte Cloud offers a 14-day free trial that begins after your first successful To start setting up a data pipeline, see how to [set up a source](./add-a-source.md). -:::info -Depending on your data residency, you may need to [allowlist IP addresses](/operating-airbyte/security.md#network-security-1) to enable access to Airbyte. -::: -## Deploy Airbyte (Open Source) +## Deploy Airbyte (Self-Managed) -To use Airbyte Open Source, you can use on the following options to deploy it on your infrastructure. +When self-managing Airbyte, your data never leaves your premises. Get started immediately by deploying locally using Docker. + +### Self-Managed Community (Open Source) + +With Airbyte Self-Managed Community (Open Source), you can use one of the following options in your infrastructure: - [Local Deployment](/deploying-airbyte/local-deployment.md) (recommended when trying out Airbyte) -- [On Aws](/deploying-airbyte/on-aws-ec2.md) +- [On AWS](/deploying-airbyte/on-aws-ec2.md) - [On Azure VM Cloud Shell](/deploying-airbyte/on-azure-vm-cloud-shell.md) - [On Digital Ocean Droplet](/deploying-airbyte/on-digitalocean-droplet.md) -- [On GCP.md](/deploying-airbyte/on-gcp-compute-engine.md) +- [On GCP](/deploying-airbyte/on-gcp-compute-engine.md) - [On Kubernetes](/deploying-airbyte/on-kubernetes-via-helm.md) - [On OCI VM](/deploying-airbyte/on-oci-vm.md) - [On Restack](/deploying-airbyte/on-restack.md) - [On Plural](/deploying-airbyte/on-plural.md) - [On AWS ECS](/deploying-airbyte/on-aws-ecs.md) (Spoiler alert: it doesn't work) + +### Self-Managed Enterprise +Airbyte Self-Managed Enterprise is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. + +To start with Self-Managed Enterprise, navigate to our [Enterprise setup guide](/enterprise-setup/README.md). diff --git a/docs/using-airbyte/getting-started/set-up-a-connection.md b/docs/using-airbyte/getting-started/set-up-a-connection.md index 7948eeeda06a..c27c77c16f97 100644 --- a/docs/using-airbyte/getting-started/set-up-a-connection.md +++ b/docs/using-airbyte/getting-started/set-up-a-connection.md @@ -1,14 +1,21 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + # Set up a Connection Now that you've learned how to set up your first [source](./add-a-source) and [destination](./add-a-destination), it's time to finish the job by creating your very first connection! -On the left side of your main Airbyte dashboard, select **Connections**. You will be prompted to choose which source and destination to use for this connection. As an example, we'll use the **Google Sheets** source and **Local JSON** destination. +On the left side of your main Airbyte dashboard, select **Connections**. You will be prompted to choose which source and destination to use for this connection. For this example, we'll use the **Google Sheets** source and the destination you previously set up, either **Local JSON** or **Google Sheets**. ## Configure the connection Once you've chosen your source and destination, you'll be able to configure the connection. You can refer to [this page](/cloud/managing-airbyte-cloud/configuring-connections.md) for more information on each available configuration. For this demo, we'll simply set the **Replication frequency** to a 24 hour interval and leave the other fields at their default values. -![Connection config](../../.gitbook/assets/set-up-a-connection/getting-started-connection-config.png) +![Connection config](./assets/getting-started-connection-configuration.png) :::note By default, data will sync to the default defined in the destination. To ensure your data is synced to the correct place, see our examples for [Destination Namespace](/using-airbyte/core-concepts/namespaces.md) @@ -16,9 +23,9 @@ By default, data will sync to the default defined in the destination. To ensure Next, you can toggle which streams you want to replicate, as well as setting up the desired sync mode for each stream. For more information on the nature of each sync mode supported by Airbyte, see [this page](/using-airbyte/core-concepts/sync-modes). -Our test data consists of a single stream cleverly named `Test Data`, which we've enabled and set to `Full Refresh - Overwrite` sync mode. +Our test data consists of three streams, which we've enabled and set to `Incremental - Append + Deduped` sync mode. -![Stream config](../../.gitbook/assets/set-up-a-connection/getting-started-connection-streams.png) +![Stream config](./assets/getting-started-stream-selection.png) Click **Set up connection** to complete your first connection. Your first sync is about to begin! @@ -26,34 +33,44 @@ Click **Set up connection** to complete your first connection. Your first sync i Once you've finished setting up the connection, you will be automatically redirected to a connection overview containing all the tools you need to keep track of your connection. -![Connection dashboard](../../.gitbook/assets/set-up-a-connection/getting-started-connection-success.png) +![Connection dashboard](./assets/getting-started-connection-complete.png) Here's a basic overview of the tabs and their use: 1. The **Status** tab shows you an overview of your connector's sync health. 2. The **Job History** tab allows you to check the logs for each sync. If you encounter any errors or unexpected behaviors during a sync, checking the logs is always a good first step to finding the cause and solution. 3. The **Replication** tab allows you to modify the configurations you chose during the connection setup. +4. The **Transformation** tab allows you to set up a custom post-sync transformations using dbt. 4. The **Settings** tab contains additional settings, and the option to delete the connection if you no longer wish to use it. ### Check the data from your first sync Once the first sync has completed, you can verify the sync has completed by checking the data in your destination. -If you followed along and created your own connection using a `Local JSON` destination, you can use this command to check the file's contents to make sure the replication worked as intended (be sure to replace YOUR_PATH with the path you chose in your destination setup, and YOUR_STREAM_NAME with the name of an actual stream you replicated): + + + If you followed along and created your own connection using a **Google Sheets** destination, you will now see three tabs created in your Google Sheet, `products`, `users`, and `purchases`. -```bash -cat /tmp/airbyte_local/YOUR_PATH/_airbyte_raw_YOUR_STREAM_NAME.jsonl -``` + + + If you followed along and created your own connection using a `Local JSON` destination, you can use this command to check the file's contents to make sure the replication worked as intended (be sure to replace YOUR_PATH with the path you chose in your destination setup, and YOUR_STREAM_NAME with the name of an actual stream you replicated): -You should see a list of JSON objects, each containing a unique `airbyte_ab_id`, an `emitted_at` timestamp, and `airbyte_data` containing the extracted record. + ```bash + cat /tmp/airbyte_local/YOUR_PATH/_airbyte_raw_YOUR_STREAM_NAME.jsonl + ``` + + You should see a list of JSON objects, each containing a unique `airbyte_ab_id`, an `emitted_at` timestamp, and `airbyte_data` containing the extracted record. :::tip If you are using Airbyte on Windows with WSL2 and Docker, refer to [this guide](/integrations/locating-files-local-destination.md) to locate the replicated folder and file. ::: + + + ## What's next? -Congratulations on successfully setting up your first connection using Airbyte Open Source! We hope that this will be just the first step on your journey with us. We support a large, ever-growing [catalog of sources and destinations](/integrations/), and you can even [contribute your own](/connector-development/). +Congratulations on successfully setting up your first connection using Airbyte! We hope that this will be just the first step on your journey with us. We support a large, ever-growing [catalog of sources and destinations](/integrations/), and you can even [contribute your own](/connector-development/). If you have any questions at all, please reach out to us on [Slack](https://slack.airbyte.io/). If you would like to see a missing feature or connector added, please create an issue on our [Github](https://github.com/airbytehq/airbyte). Our community's participation is invaluable in helping us grow and improve every day, and we always welcome your feedback. diff --git a/docs/using-airbyte/workspaces.md b/docs/using-airbyte/workspaces.md index 710242ca4728..7b211c0a0cb0 100644 --- a/docs/using-airbyte/workspaces.md +++ b/docs/using-airbyte/workspaces.md @@ -1,9 +1,11 @@ +--- +products: cloud, oss-enterprise +--- + # Manage your workspace A workspace in Airbyte allows you to collaborate with other users and manage connections together. On Airbyte Cloud it will allow you to share billing details for a workspace. - - :::info Airbyte [credits](https://airbyte.com/pricing) are assigned per workspace and cannot be transferred between workspaces. ::: @@ -14,7 +16,7 @@ To add a user to your workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **Access Management**. +2. Click **Workspace** > **Access Management**. 3. Click **+ New user**. @@ -32,7 +34,7 @@ To remove a user from your workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **Access Management**. +2. Click **Workspace** > **Access Management**. 3. Click **Remove** next to the user’s email. @@ -44,7 +46,7 @@ To rename a workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **General Settings**. +2. Click **Workspace** > **General**. 3. In the **Workspace name** field, enter the new name for your workspace. @@ -56,7 +58,7 @@ To delete a workspace: 1. Go to the **Settings** via the side navigation in Airbyte. -2. Click **General Settings**. +2. Click **Workspace** > **General**. 3. In the **Delete your workspace** section, click **Delete**. diff --git a/docusaurus/.nvmrc b/docusaurus/.nvmrc new file mode 100644 index 000000000000..55bffd620b9a --- /dev/null +++ b/docusaurus/.nvmrc @@ -0,0 +1 @@ +18.15.0 diff --git a/docusaurus/README.md b/docusaurus/README.md index f03e15d4c705..8f6b20d23122 100644 --- a/docusaurus/README.md +++ b/docusaurus/README.md @@ -1,43 +1,80 @@ # Documentation and Docusaurus -We use [docusaurus](https://docusaurus.io) for consistent process, in `Airbyte` **no website is generated**. -Functionally this is a very fancy **linter** - -Running the build process will **check for broken links**, please read the output and address -any broken links that you are able to do. +We use [Docusaurus](https://docusaurus.io) to build Airbyte's +[documentation site](https://docs.airbyte.io) from documentation source files in Markdown, and lint +the source files. We host the resulting docs site on Vercel. It deploys automatically when any +changes get merged to `master`. ## Installation -For consistency across other Airbyte projects we use yarn (A Javascript based software package manager) +For consistency across other Airbyte projects we use `pnpm` (A Javascript based software package +manager). ```bash -brew install yarn +brew install pnpm cd docusaurus -yarn install -yarn build +pnpm install +pnpm build ``` -At this point you will see any broken links that docusaurus was able to find. +`pnpm build` will build Docusaurus site in `docusaurus/build` directory. ## Developing Locally +If you want to make changes to the documentation, you can run docusaurus locally in a way that would +listen to any source docs changes and live-reload them: + ```bash -yarn start # any changes will automatically be reflected in your browser! +pnpm start # any changes will automatically be reflected in your browser! ``` -## Making Changes +All the content for docs.airbyte.com lives in the `/docs` directory in this repo. All files are +markdown. Make changes or add new files, and you should see them in your browser! + +## Changing Navigation Structure + +If you have created any new files, be sure to add them manually to the table of contents found here +in [`sidebars.js`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js) + +## Contributing + +We welcome documentation updates! If you'd like to contribute a change, please make sure to: -All the content for docs.airbyte.com lives in the `/docs` directory in this repo. All files are markdown. Make changes or add new files, and you should see them in your browser! +- Run `pnpm build` and check that all build steps are successful. +- Push your changes into a pull request, and follow the PR template instructions. -If you have created any new files, be sure to add them manually to the table of contents found here in this [file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js) +When you make a pull request, Vercel will automatically build a test instance of the full docs site +and link it in the pull request for review. -## Plugin Client Redirects +### Checking for broken links + +Airbyte's docs site checks links with Docusaurus at build time, and with an additional GitHub action +periodically: + +- Running the build process will **check for broken links**, please read the output and address any + broken links that you are able to do. +- [This GitHub Action](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/doc-link-check.yml) + checks all links on Airbyte production docs site, and tells us if any of them are broken. + +> [!NOTE] Docusaurus links checker only checks _relative_ links, and assumes that absolute links are +> fine. For that reason, if you're linking to another Airbyte documentation page, make it a relative +> link. I.e. `[link](/connector-development/overview.md)` instead of +> `[link](https://docs.airbyte.com/connector-development/)`. That way, if your link breaks in the +> future due to a navigation restructure, it will be caught with `pnpm build`. + +## Docusaurus Plugins We Use + +### Plugin Client Redirects A silly name, but a useful plugin that adds redirect functionality to docusaurus [Official documentation here](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-client-redirects) -You will need to edit [this docusaurus file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) +If you're proposing to move an existing documentation file or change its name, please setup a +redirect rule. + +You will need to edit +[this docusaurus file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) You will see a commented section the reads something like this @@ -48,13 +85,18 @@ You will see a commented section the reads something like this // }, ``` -Copy this section, replace the values, and [test it locally](locally_testing_docusaurus.md) by going to the -path you created a redirect for and checked to see that the address changes to your new one. +Copy this section, replace the values, and [test it locally](locally_testing_docusaurus.md) by going +to the path you created a redirect for and checked to see that the address changes to your new one. _Note:_ Your path \*_needs_ a leading slash `/` to work ## Deploying Docs -We use Github Pages for hosting this docs website, and [Docusaurus](https://docusaurus.io/) as the docs framework. Any change to the `/docs` directory you make is deployed when you merge to your PR to the master branch automagically! +Airbyte docs live on Vercel. Any change to the `/docs` directory you make is deployed when you merge +to your PR to the master branch automagically! -The source code for the docs lives in the [airbyte monorepo's `docs/` directory](https://github.com/airbytehq/airbyte/tree/master/docs). Any changes to the `/docs` directory will be tested automatically in your PR. Be sure that you wait for the tests to pass before merging! If there are CI problems publishing your docs, you can run `tools/bin/deploy_docusaurus` locally - this is the publish script that CI runs. +The source code for the docs lives in the +[airbyte monorepo's `docs/` directory](https://github.com/airbytehq/airbyte/tree/master/docs). Any +changes to the `/docs` directory will be tested automatically in your PR. Be sure that you wait for +the tests to pass before merging! If there are CI problems publishing your docs, you can run +`tools/bin/deploy_docusaurus` locally - this is the publish script that CI runs. diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index db39086531f6..06c4311a1d9e 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -9,9 +9,10 @@ const { themes } = require('prism-react-renderer'); const lightCodeTheme = themes.github; const darkCodeTheme = themes.dracula; - - const docsHeaderDecoration = require("./src/remark/docsHeaderDecoration"); +const productInformation = require("./src/remark/productInformation"); +const connectorList = require("./src/remark/connectorList"); +const specDecoration = require("./src/remark/specDecoration"); const redirects = yaml.load( fs.readFileSync(path.join(__dirname, "redirects.yml"), "utf-8") @@ -66,6 +67,10 @@ const config = { test: /\.ya?ml$/, use: "yaml-loader", }, + { + test: /\.html$/i, + loader: "html-loader", + }, ], }, }; @@ -73,7 +78,10 @@ const config = { }), ], - clientModules: [require.resolve("./src/scripts/cloudStatus.js")], + clientModules: [ + require.resolve("./src/scripts/fontAwesomeIcons.js"), + require.resolve("./src/scripts/cloudStatus.js"), + ], presets: [ [ @@ -87,7 +95,8 @@ const config = { editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs", path: "../docs", exclude: ["**/*.inapp.md"], - remarkPlugins: [docsHeaderDecoration], + beforeDefaultRemarkPlugins: [specDecoration, connectorList], // use before-default plugins so TOC rendering picks up inserted headings + remarkPlugins: [docsHeaderDecoration, productInformation], }, blog: false, theme: { diff --git a/docusaurus/package.json b/docusaurus/package.json index 760738093b86..c2476ce304c4 100644 --- a/docusaurus/package.json +++ b/docusaurus/package.json @@ -14,65 +14,65 @@ "write-heading-ids": "docusaurus write-heading-ids" }, "dependencies": { - "@babel/core": "7.18.6", - "@babel/helper-builder-binary-assignment-operator-visitor": "7.18.6", + "@babel/core": "7.23.6", + "@babel/helper-builder-binary-assignment-operator-visitor": "7.22.15", "@babel/helper-explode-assignable-expression": "7.18.6", - "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "7.18.6", - "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "7.18.6", - "@babel/plugin-proposal-async-generator-functions": "7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "7.23.3", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "7.23.3", + "@babel/plugin-proposal-async-generator-functions": "7.20.7", "@babel/plugin-proposal-class-properties": "7.18.6", - "@babel/plugin-proposal-class-static-block": "7.18.6", + "@babel/plugin-proposal-class-static-block": "7.21.0", "@babel/plugin-proposal-dynamic-import": "7.18.6", - "@babel/plugin-proposal-export-namespace-from": "7.18.6", + "@babel/plugin-proposal-export-namespace-from": "7.18.9", "@babel/plugin-proposal-json-strings": "7.18.6", - "@babel/plugin-proposal-logical-assignment-operators": "7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "7.20.7", "@babel/plugin-proposal-nullish-coalescing-operator": "7.18.6", "@babel/plugin-proposal-numeric-separator": "7.18.6", - "@babel/plugin-proposal-object-rest-spread": "7.18.6", + "@babel/plugin-proposal-object-rest-spread": "7.20.7", "@babel/plugin-proposal-optional-catch-binding": "7.18.6", "@babel/plugin-proposal-private-methods": "7.18.6", - "@babel/plugin-proposal-private-property-in-object": "7.18.6", - "@babel/plugin-syntax-import-assertions": "7.18.6", - "@babel/plugin-syntax-typescript": "7.18.6", - "@babel/plugin-transform-arrow-functions": "7.18.6", - "@babel/plugin-transform-async-to-generator": "7.18.6", - "@babel/plugin-transform-block-scoped-functions": "7.18.6", - "@babel/plugin-transform-block-scoping": "7.18.6", - "@babel/plugin-transform-classes": "7.18.6", - "@babel/plugin-transform-computed-properties": "7.18.6", - "@babel/plugin-transform-destructuring": "7.18.6", - "@babel/plugin-transform-duplicate-keys": "7.18.6", - "@babel/plugin-transform-exponentiation-operator": "7.18.6", - "@babel/plugin-transform-for-of": "7.18.6", - "@babel/plugin-transform-function-name": "7.18.6", - "@babel/plugin-transform-literals": "7.18.6", - "@babel/plugin-transform-member-expression-literals": "7.18.6", - "@babel/plugin-transform-modules-amd": "7.18.6", - "@babel/plugin-transform-modules-commonjs": "7.18.6", - "@babel/plugin-transform-modules-systemjs": "7.18.6", - "@babel/plugin-transform-modules-umd": "7.18.6", - "@babel/plugin-transform-named-capturing-groups-regex": "7.18.6", - "@babel/plugin-transform-new-target": "7.18.6", - "@babel/plugin-transform-object-super": "7.18.6", - "@babel/plugin-transform-property-literals": "7.18.6", - "@babel/plugin-transform-react-display-name": "7.18.6", - "@babel/plugin-transform-react-jsx-development": "7.18.6", - "@babel/plugin-transform-react-pure-annotations": "7.18.6", - "@babel/plugin-transform-regenerator": "7.18.6", - "@babel/plugin-transform-reserved-words": "7.18.6", - "@babel/plugin-transform-runtime": "7.18.6", - "@babel/plugin-transform-shorthand-properties": "7.18.6", - "@babel/plugin-transform-spread": "7.18.6", - "@babel/plugin-transform-sticky-regex": "7.18.6", - "@babel/plugin-transform-template-literals": "7.18.6", - "@babel/plugin-transform-typeof-symbol": "7.18.6", - "@babel/plugin-transform-typescript": "7.18.6", - "@babel/plugin-transform-unicode-escapes": "7.18.6", - "@babel/plugin-transform-unicode-regex": "7.18.6", - "@babel/preset-env": "7.18.6", - "@babel/preset-react": "7.18.6", - "@babel/preset-typescript": "7.18.6", - "@babel/runtime-corejs3": "7.18.6", + "@babel/plugin-proposal-private-property-in-object": "7.21.11", + "@babel/plugin-syntax-import-assertions": "7.23.3", + "@babel/plugin-syntax-typescript": "7.23.3", + "@babel/plugin-transform-arrow-functions": "7.23.3", + "@babel/plugin-transform-async-to-generator": "7.23.3", + "@babel/plugin-transform-block-scoped-functions": "7.23.3", + "@babel/plugin-transform-block-scoping": "7.23.4", + "@babel/plugin-transform-classes": "7.23.5", + "@babel/plugin-transform-computed-properties": "7.23.3", + "@babel/plugin-transform-destructuring": "7.23.3", + "@babel/plugin-transform-duplicate-keys": "7.23.3", + "@babel/plugin-transform-exponentiation-operator": "7.23.3", + "@babel/plugin-transform-for-of": "7.23.6", + "@babel/plugin-transform-function-name": "7.23.3", + "@babel/plugin-transform-literals": "7.23.3", + "@babel/plugin-transform-member-expression-literals": "7.23.3", + "@babel/plugin-transform-modules-amd": "7.23.3", + "@babel/plugin-transform-modules-commonjs": "7.23.3", + "@babel/plugin-transform-modules-systemjs": "7.23.3", + "@babel/plugin-transform-modules-umd": "7.23.3", + "@babel/plugin-transform-named-capturing-groups-regex": "7.22.5", + "@babel/plugin-transform-new-target": "7.23.3", + "@babel/plugin-transform-object-super": "7.23.3", + "@babel/plugin-transform-property-literals": "7.23.3", + "@babel/plugin-transform-react-display-name": "7.23.3", + "@babel/plugin-transform-react-jsx-development": "7.22.5", + "@babel/plugin-transform-react-pure-annotations": "7.23.3", + "@babel/plugin-transform-regenerator": "7.23.3", + "@babel/plugin-transform-reserved-words": "7.23.3", + "@babel/plugin-transform-runtime": "7.23.6", + "@babel/plugin-transform-shorthand-properties": "7.23.3", + "@babel/plugin-transform-spread": "7.23.3", + "@babel/plugin-transform-sticky-regex": "7.23.3", + "@babel/plugin-transform-template-literals": "7.23.3", + "@babel/plugin-transform-typeof-symbol": "7.23.3", + "@babel/plugin-transform-typescript": "7.23.6", + "@babel/plugin-transform-unicode-escapes": "7.23.3", + "@babel/plugin-transform-unicode-regex": "7.23.3", + "@babel/preset-env": "7.23.6", + "@babel/preset-react": "7.23.3", + "@babel/preset-typescript": "7.23.3", + "@babel/runtime-corejs3": "7.23.6", "@cmfcmf/docusaurus-search-local": "^1.1.0", "@docsearch/react": "3.1.0", "@docusaurus/core": "^3.0.1", @@ -86,42 +86,49 @@ "@docusaurus/theme-mermaid": "^3.0.1", "@docusaurus/theme-search-algolia": "^3.0.1", "@docusaurus/types": "^3.0.1", - "@fortawesome/fontawesome-svg-core": "^6.4.2", - "@fortawesome/free-regular-svg-icons": "^6.4.2", - "@fortawesome/free-solid-svg-icons": "^6.4.2", + "@fortawesome/fontawesome-svg-core": "^6.5.1", + "@fortawesome/free-regular-svg-icons": "^6.5.1", + "@fortawesome/free-solid-svg-icons": "^6.5.1", "@fortawesome/react-fontawesome": "^0.2.0", + "@headlessui/react": "^1.7.18", "@mdx-js/react": "^3.0.0", "async": "2.6.4", - "autoprefixer": "10.4.7", + "autoprefixer": "10.4.16", + "classnames": "^2.3.2", "clsx": "^1.1.1", "copy-webpack-plugin": "11.0.0", - "core-js": "3.23.3", + "core-js": "3.35.0", "css-declaration-sorter": "6.3.0", "css-minimizer-webpack-plugin": "4.0.0", - "cssnano": "5.1.12", - "cssnano-preset-advanced": "5.3.8", + "cssnano": "6.0.2", + "cssnano-preset-advanced": "6.0.2", "del": "6.1.1", "docusaurus-plugin-hubspot": "^1.0.0", "docusaurus-plugin-segment": "^1.0.3", + "html-loader": "^4.2.0", "js-yaml": "^4.1.0", + "json-schema-faker": "^0.5.4", "node-fetch": "^3.3.2", "nth-check": "2.0.1", - "postcss-convert-values": "5.1.2", - "postcss-discard-comments": "5.1.2", - "postcss-loader": "7.0.0", - "postcss-merge-longhand": "5.1.6", - "postcss-merge-rules": "5.1.2", - "postcss-minify-selectors": "5.2.1", - "postcss-normalize-positions": "5.1.1", - "postcss-normalize-repeat-style": "5.1.1", - "postcss-ordered-values": "5.1.3", - "prism-react-renderer": "^2.1.0", + "postcss-convert-values": "6.0.1", + "postcss-discard-comments": "6.0.1", + "postcss-loader": "7.3.4", + "postcss-merge-longhand": "6.0.1", + "postcss-merge-rules": "6.0.2", + "postcss-minify-selectors": "6.0.1", + "postcss-normalize-positions": "6.0.1", + "postcss-normalize-repeat-style": "6.0.1", + "postcss-ordered-values": "6.0.1", + "prism-react-renderer": "^2.3.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-markdown": "^8.0.7", "react-router": "5.3.3", + "sanitize-html": "^2.11.0", "sockjs": "0.3.24", "trim": "0.0.3", + "unist-builder": "^4.0.0", + "unist-util-select": "^5.1.0", "unist-util-visit": "^5.0.0", "webpack-dev-server": "4.9.2", "yaml-loader": "^0.8.0" diff --git a/docusaurus/pnpm-lock.yaml b/docusaurus/pnpm-lock.yaml new file mode 100644 index 000000000000..ec5d26397ead --- /dev/null +++ b/docusaurus/pnpm-lock.yaml @@ -0,0 +1,11550 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + '@babel/core': + specifier: 7.23.6 + version: 7.23.6 + '@babel/helper-builder-binary-assignment-operator-visitor': + specifier: 7.22.15 + version: 7.22.15 + '@babel/helper-explode-assignable-expression': + specifier: 7.18.6 + version: 7.18.6 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-proposal-async-generator-functions': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-class-properties': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-class-static-block': + specifier: 7.21.0 + version: 7.21.0(@babel/core@7.23.6) + '@babel/plugin-proposal-dynamic-import': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-export-namespace-from': + specifier: 7.18.9 + version: 7.18.9(@babel/core@7.23.6) + '@babel/plugin-proposal-json-strings': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-logical-assignment-operators': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-nullish-coalescing-operator': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-numeric-separator': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-object-rest-spread': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-optional-catch-binding': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-private-methods': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-private-property-in-object': + specifier: 7.21.11 + version: 7.21.11(@babel/core@7.23.6) + '@babel/plugin-syntax-import-assertions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-typescript': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-arrow-functions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-async-to-generator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoped-functions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoping': + specifier: 7.23.4 + version: 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-classes': + specifier: 7.23.5 + version: 7.23.5(@babel/core@7.23.6) + '@babel/plugin-transform-computed-properties': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-destructuring': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-duplicate-keys': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-exponentiation-operator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-for-of': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-function-name': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-member-expression-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-amd': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-systemjs': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-umd': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-named-capturing-groups-regex': + specifier: 7.22.5 + version: 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-new-target': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-object-super': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-property-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-display-name': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx-development': + specifier: 7.22.5 + version: 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-react-pure-annotations': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-regenerator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-reserved-words': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-runtime': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-shorthand-properties': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-spread': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-sticky-regex': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-template-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typeof-symbol': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typescript': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-escapes': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-regex': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/preset-env': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/runtime-corejs3': + specifier: 7.23.6 + version: 7.23.6 + '@cmfcmf/docusaurus-search-local': + specifier: ^1.1.0 + version: 1.1.0(@docusaurus/core@3.0.1)(search-insights@2.13.0) + '@docsearch/react': + specifier: 3.1.0 + version: 3.1.0(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/core': + specifier: ^3.0.1 + version: 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/cssnano-preset': + specifier: ^3.0.1 + version: 3.0.1 + '@docusaurus/module-type-aliases': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-client-redirects': + specifier: ^3.0.1 + version: 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-debug': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-sitemap': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/preset-classic': + specifier: ^3.0.1 + version: 3.0.1(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/theme-classic': + specifier: ^3.0.1 + version: 3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-mermaid': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-search-algolia': + specifier: ^3.0.1 + version: 3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/types': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@fortawesome/fontawesome-svg-core': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/free-regular-svg-icons': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/free-solid-svg-icons': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/react-fontawesome': + specifier: ^0.2.0 + version: 0.2.0(@fortawesome/fontawesome-svg-core@6.5.1)(react@18.2.0) + '@headlessui/react': + specifier: ^1.7.18 + version: 1.7.18(react-dom@18.2.0)(react@18.2.0) + '@mdx-js/react': + specifier: ^3.0.0 + version: 3.0.0(@types/react@18.2.46)(react@18.2.0) + async: + specifier: 2.6.4 + version: 2.6.4 + autoprefixer: + specifier: 10.4.16 + version: 10.4.16(postcss@8.4.32) + classnames: + specifier: ^2.3.2 + version: 2.3.2 + clsx: + specifier: ^1.1.1 + version: 1.1.1 + copy-webpack-plugin: + specifier: 11.0.0 + version: 11.0.0(webpack@5.89.0) + core-js: + specifier: 3.35.0 + version: 3.35.0 + css-declaration-sorter: + specifier: 6.3.0 + version: 6.3.0(postcss@8.4.32) + css-minimizer-webpack-plugin: + specifier: 4.0.0 + version: 4.0.0(webpack@5.89.0) + cssnano: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + cssnano-preset-advanced: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + del: + specifier: 6.1.1 + version: 6.1.1 + docusaurus-plugin-hubspot: + specifier: ^1.0.0 + version: 1.0.0 + docusaurus-plugin-segment: + specifier: ^1.0.3 + version: 1.0.3 + html-loader: + specifier: ^4.2.0 + version: 4.2.0(webpack@5.89.0) + js-yaml: + specifier: ^4.1.0 + version: 4.1.0 + json-schema-faker: + specifier: ^0.5.4 + version: 0.5.4 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + nth-check: + specifier: 2.0.1 + version: 2.0.1 + postcss-convert-values: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-discard-comments: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-loader: + specifier: 7.3.4 + version: 7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0) + postcss-merge-longhand: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-merge-rules: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + postcss-minify-selectors: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-normalize-positions: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-normalize-repeat-style: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-ordered-values: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + prism-react-renderer: + specifier: ^2.3.1 + version: 2.3.1(react@18.2.0) + react: + specifier: ^18.2.0 + version: 18.2.0 + react-dom: + specifier: ^18.2.0 + version: 18.2.0(react@18.2.0) + react-markdown: + specifier: ^8.0.7 + version: 8.0.7(@types/react@18.2.46)(react@18.2.0) + react-router: + specifier: 5.3.3 + version: 5.3.3(react@18.2.0) + sanitize-html: + specifier: ^2.11.0 + version: 2.11.0 + sockjs: + specifier: 0.3.24 + version: 0.3.24 + trim: + specifier: 0.0.3 + version: 0.0.3 + unist-builder: + specifier: ^4.0.0 + version: 4.0.0 + unist-util-select: + specifier: ^5.1.0 + version: 5.1.0 + unist-util-visit: + specifier: ^5.0.0 + version: 5.0.0 + webpack-dev-server: + specifier: 4.9.2 + version: 4.9.2(webpack@5.89.0) + yaml-loader: + specifier: ^0.8.0 + version: 0.8.0 + +packages: + + /@algolia/autocomplete-core@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-0v3mHfkvJBVx0aO1U290EHaLPp9pkUL8zkgbVY0JlitItrbXfYYHQHtNs1TxpA63mQAD0K0LyLzO2x+uWiBbGQ==} + dependencies: + '@algolia/autocomplete-plugin-algolia-insights': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + - search-insights + dev: false + + /@algolia/autocomplete-core@1.6.3: + resolution: {integrity: sha512-dqQqRt01fX3YuVFrkceHsoCnzX0bLhrrg8itJI1NM68KjrPYQPYsE+kY8EZTCM4y8VDnhqJErR73xe/ZsV+qAA==} + dependencies: + '@algolia/autocomplete-shared': 1.6.3 + dev: false + + /@algolia/autocomplete-core@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-009HdfugtGCdC4JdXUbVJClA0q0zh24yyePn+KUGk3rP7j8FEe/m5Yo/z65gn6nP/cM39PxpzqKrL7A6fP6PPw==} + dependencies: + '@algolia/autocomplete-plugin-algolia-insights': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + - search-insights + dev: false + + /@algolia/autocomplete-js@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-gw2jbkIzSH+xljX3yoOg+5nfJwMh7jqw5T/jy/WPwgmPhn5Mv6PmosCM0huGwH2E88nwxNlY2AhbkDrS4qceAw==} + peerDependencies: + '@algolia/client-search': '>= 4.5.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-core': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-preset-algolia': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + htm: 3.1.1 + preact: 10.19.3 + transitivePeerDependencies: + - search-insights + dev: false + + /@algolia/autocomplete-plugin-algolia-insights@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-Q0rRUZ72x7piqvJKi1//SBZvoImnYdJLRC7Yaa0rwKtkIVQFl6MmZw/p4AEDSWIu5HY3Ki3bzgYxeDyhm//P/w==} + peerDependencies: + search-insights: '>= 1 < 3' + dependencies: + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + dev: false + + /@algolia/autocomplete-plugin-algolia-insights@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-a/yTUkcO/Vyy+JffmAnTWbr4/90cLzw+CC3bRbhnULr/EM0fGNvM13oQQ14f2moLMcVDyAx/leczLlAOovhSZg==} + peerDependencies: + search-insights: '>= 1 < 3' + dependencies: + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + dev: false + + /@algolia/autocomplete-preset-algolia@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-IlanOCLT2EvfygX5cGFR5iKgfhQB0MqCv163ldctq8l0QCVdEOM1VLIQhl0tB3ViJc5XKUB8QZ7V+DcSVtZAuQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-preset-algolia@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-d4qlt6YmrLMYy95n5TB52wtNDr6EgAIPH81dvvvW8UmuWRgxEtY0NJiPwl/h95JtG2vmRM804M0DSwMCNZlzRA==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-shared@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-YB7JlPl1coHai3Xd4OdNIMavAMbgx8eHPH9nlEgcrCqCx57njh0qReruTMRxaThBaWIkkl47jZlUnKvb8MjGGQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-shared@1.6.3: + resolution: {integrity: sha512-UV46bnkTztyADFaETfzFC5ryIdGVb2zpAoYgu0tfcuYWjhg1KbLXveFffZIrGVoboqmAk1b+jMrl6iCja1i3lg==} + dev: false + + /@algolia/autocomplete-shared@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-Wnm9E4Ye6Rl6sTTqjoymD+l8DjSTHsHboVRYrKgEt8Q7UHm9nYbqhN/i0fhUYA3OAEH7WA8x3jfpnmJm3rKvaQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-theme-classic@1.13.0: + resolution: {integrity: sha512-YAyfcpi+VJ0h5PUTThDmc/V2OB47RNlvIBQgffzrjAw5vDkoBcAj5bsReJW8/QtLnRGB85XhrmWoYFtP4W3HgQ==} + dev: false + + /@algolia/cache-browser-local-storage@4.22.0: + resolution: {integrity: sha512-uZ1uZMLDZb4qODLfTSNHxSi4fH9RdrQf7DXEzW01dS8XK7QFtFh29N5NGKa9S+Yudf1vUMIF+/RiL4i/J0pWlQ==} + dependencies: + '@algolia/cache-common': 4.22.0 + dev: false + + /@algolia/cache-common@4.22.0: + resolution: {integrity: sha512-TPwUMlIGPN16eW67qamNQUmxNiGHg/WBqWcrOoCddhqNTqGDPVqmgfaM85LPbt24t3r1z0zEz/tdsmuq3Q6oaA==} + dev: false + + /@algolia/cache-in-memory@4.22.0: + resolution: {integrity: sha512-kf4Cio9NpPjzp1+uXQgL4jsMDeck7MP89BYThSvXSjf2A6qV/0KeqQf90TL2ECS02ovLOBXkk98P7qVarM+zGA==} + dependencies: + '@algolia/cache-common': 4.22.0 + dev: false + + /@algolia/client-account@4.22.0: + resolution: {integrity: sha512-Bjb5UXpWmJT+yGWiqAJL0prkENyEZTBzdC+N1vBuHjwIJcjLMjPB6j1hNBRbT12Lmwi55uzqeMIKS69w+0aPzA==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-analytics@4.22.0: + resolution: {integrity: sha512-os2K+kHUcwwRa4ArFl5p/3YbF9lN3TLOPkbXXXxOvDpqFh62n9IRZuzfxpHxMPKAQS3Et1s0BkKavnNP02E9Hg==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-common@4.22.0: + resolution: {integrity: sha512-BlbkF4qXVWuwTmYxVWvqtatCR3lzXwxx628p1wj1Q7QP2+LsTmGt1DiUYRuy9jG7iMsnlExby6kRMOOlbhv2Ag==} + dependencies: + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-personalization@4.22.0: + resolution: {integrity: sha512-pEOftCxeBdG5pL97WngOBi9w5Vxr5KCV2j2D+xMVZH8MuU/JX7CglDSDDb0ffQWYqcUN+40Ry+xtXEYaGXTGow==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-search@4.22.0: + resolution: {integrity: sha512-bn4qQiIdRPBGCwsNuuqB8rdHhGKKWIij9OqidM1UkQxnSG8yzxHdb7CujM30pvp5EnV7jTqDZRbxacbjYVW20Q==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/events@4.0.1: + resolution: {integrity: sha512-FQzvOCgoFXAbf5Y6mYozw2aj5KCJoA3m4heImceldzPSMbdyS4atVjJzXKMsfX3wnZTFYwkkt8/z8UesLHlSBQ==} + dev: false + + /@algolia/logger-common@4.22.0: + resolution: {integrity: sha512-HMUQTID0ucxNCXs5d1eBJ5q/HuKg8rFVE/vOiLaM4Abfeq1YnTtGV3+rFEhOPWhRQxNDd+YHa4q864IMc0zHpQ==} + dev: false + + /@algolia/logger-console@4.22.0: + resolution: {integrity: sha512-7JKb6hgcY64H7CRm3u6DRAiiEVXMvCJV5gRE672QFOUgDxo4aiDpfU61g6Uzy8NKjlEzHMmgG4e2fklELmPXhQ==} + dependencies: + '@algolia/logger-common': 4.22.0 + dev: false + + /@algolia/requester-browser-xhr@4.22.0: + resolution: {integrity: sha512-BHfv1h7P9/SyvcDJDaRuIwDu2yrDLlXlYmjvaLZTtPw6Ok/ZVhBR55JqW832XN/Fsl6k3LjdkYHHR7xnsa5Wvg==} + dependencies: + '@algolia/requester-common': 4.22.0 + dev: false + + /@algolia/requester-common@4.22.0: + resolution: {integrity: sha512-Y9cEH/cKjIIZgzvI1aI0ARdtR/xRrOR13g5psCxkdhpgRN0Vcorx+zePhmAa4jdQNqexpxtkUdcKYugBzMZJgQ==} + dev: false + + /@algolia/requester-node-http@4.22.0: + resolution: {integrity: sha512-8xHoGpxVhz3u2MYIieHIB6MsnX+vfd5PS4REgglejJ6lPigftRhTdBCToe6zbwq4p0anZXjjPDvNWMlgK2+xYA==} + dependencies: + '@algolia/requester-common': 4.22.0 + dev: false + + /@algolia/transporter@4.22.0: + resolution: {integrity: sha512-ieO1k8x2o77GNvOoC+vAkFKppydQSVfbjM3YrSjLmgywiBejPTvU1R1nEvG59JIIUvtSLrZsLGPkd6vL14zopA==} + dependencies: + '@algolia/cache-common': 4.22.0 + '@algolia/logger-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + dev: false + + /@ampproject/remapping@2.2.1: + resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@babel/code-frame@7.23.5: + resolution: {integrity: sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/highlight': 7.23.4 + chalk: 2.4.2 + dev: false + + /@babel/compat-data@7.23.5: + resolution: {integrity: sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/core@7.23.6: + resolution: {integrity: sha512-FxpRyGjrMJXh7X3wGLGhNDCRiwpWEF74sKjTLDJSG5Kyvow3QZaG0Adbqzi9ZrVjTWpsX+2cxWXD71NMg93kdw==} + engines: {node: '>=6.9.0'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helpers': 7.23.7 + '@babel/parser': 7.23.6 + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.7 + '@babel/types': 7.23.6 + convert-source-map: 2.0.0 + debug: 4.3.4 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/generator@7.23.6: + resolution: {integrity: sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + jsesc: 2.5.2 + dev: false + + /@babel/helper-annotate-as-pure@7.22.5: + resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-builder-binary-assignment-operator-visitor@7.22.15: + resolution: {integrity: sha512-QkBXwGgaoC2GtGZRoma6kv7Szfv06khvhFav67ZExau2RaXzy8MpHSMO2PNoP2XtmQphJQRHFfg77Bq731Yizw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-compilation-targets@7.23.6: + resolution: {integrity: sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/helper-validator-option': 7.23.5 + browserslist: 4.22.2 + lru-cache: 5.1.1 + semver: 6.3.1 + dev: false + + /@babel/helper-create-class-features-plugin@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-xCoqR/8+BoNnXOY7RVSgv6X+o7pmT5q1d+gGcRlXYkI+9B31glE4jeejhKVpA04O1AtzOt7OSQ6VYKP5FcRl9g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-member-expression-to-functions': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + semver: 6.3.1 + dev: false + + /@babel/helper-create-regexp-features-plugin@7.22.15(@babel/core@7.23.6): + resolution: {integrity: sha512-29FkPLFjn4TPEa3RE7GpW+qbE8tlsu3jntNYNfcGsc49LphF1PQIiD+vMZ1z1xVOKt+93khA9tc2JBs3kBjA7w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + regexpu-core: 5.3.2 + semver: 6.3.1 + dev: false + + /@babel/helper-define-polyfill-provider@0.4.4(@babel/core@7.23.6): + resolution: {integrity: sha512-QcJMILQCu2jm5TFPGA3lCpJJTeEP+mqeXooG/NZbg/h5FTFi6V0+99ahlRsW8/kRLyb24LZVCCiclDedhLKcBA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + debug: 4.3.4 + lodash.debounce: 4.0.8 + resolve: 1.22.8 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/helper-environment-visitor@7.22.20: + resolution: {integrity: sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-explode-assignable-expression@7.18.6: + resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-function-name@7.23.0: + resolution: {integrity: sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-hoist-variables@7.22.5: + resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-member-expression-to-functions@7.23.0: + resolution: {integrity: sha512-6gfrPwh7OuT6gZyJZvd6WbTfrqAo7vm4xCzAXOusKqq/vWdKXphTpj5klHKNmRUU6/QRGlBsyU9mAIPaWHlqJA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-module-imports@7.22.15: + resolution: {integrity: sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-module-transforms@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.20 + dev: false + + /@babel/helper-optimise-call-expression@7.22.5: + resolution: {integrity: sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-plugin-utils@7.22.5: + resolution: {integrity: sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-remap-async-to-generator@7.22.20(@babel/core@7.23.6): + resolution: {integrity: sha512-pBGyV4uBqOns+0UvhsTO8qgl8hO89PmiDYv+/COyp1aeMcmfrfruz+/nCMFiYyFF/Knn0yfrC85ZzNFjembFTw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-wrap-function': 7.22.20 + dev: false + + /@babel/helper-replace-supers@7.22.20(@babel/core@7.23.6): + resolution: {integrity: sha512-qsW0In3dbwQUbK8kejJ4R7IHVGwHJlV6lpG6UA7a9hSa2YEiAib+N1T2kr6PEeUT+Fl7najmSOS6SmAwCHK6Tw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-member-expression-to-functions': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + dev: false + + /@babel/helper-simple-access@7.22.5: + resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-skip-transparent-expression-wrappers@7.22.5: + resolution: {integrity: sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-split-export-declaration@7.22.6: + resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-string-parser@7.23.4: + resolution: {integrity: sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-validator-identifier@7.22.20: + resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-validator-option@7.23.5: + resolution: {integrity: sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-wrap-function@7.22.20: + resolution: {integrity: sha512-pms/UwkOpnQe/PDAEdV/d7dVCoBbB+R4FvYoHGZz+4VPcg7RtYy2KP7S2lbuWM6FCSgob5wshfGESbC/hzNXZw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-function-name': 7.23.0 + '@babel/template': 7.22.15 + '@babel/types': 7.23.6 + dev: false + + /@babel/helpers@7.23.7: + resolution: {integrity: sha512-6AMnjCoC8wjqBzDHkuqpa7jAKwvMo4dC+lr/TFBz+ucfulO1XMpDnwWPGBNwClOKZ8h6xn5N81W/R5OrcKtCbQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.7 + '@babel/types': 7.23.6 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/highlight@7.23.4: + resolution: {integrity: sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-validator-identifier': 7.22.20 + chalk: 2.4.2 + js-tokens: 4.0.0 + dev: false + + /@babel/parser@7.23.6: + resolution: {integrity: sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-iRkKcCqb7iGnq9+3G6rZ+Ciz5VywC4XNRHe57lKM+jOeYAoR0lVqdeeDRfh0tQcTfw/+vBhHn926FmQhLtlFLQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-WwlxbfMNdVEpQjZmK5mhm7oSwD3dS6eU+Iwsi4Knl9wAletWem7kaRsGOG+8UEbRyqxY4SS5zvtfXwX+jMxUwQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.13.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-transform-optional-chaining': 7.23.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-LlRT7HgaifEpQA1ZgLVOIJZZFVPWN5iReq/7/JixwBtwcoeVGDBD53ZV28rrsLYOZs1Y/EHhA8N/Z6aazHR8cw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-async-generator-functions@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-async-generator-functions instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-class-properties@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-properties instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-class-static-block@7.21.0(@babel/core@7.23.6): + resolution: {integrity: sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-static-block instead. + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-dynamic-import@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-dynamic-import instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-export-namespace-from@7.18.9(@babel/core@7.23.6): + resolution: {integrity: sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-export-namespace-from instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-json-strings@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-json-strings instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-logical-assignment-operators@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-y7C7cZgpMIjWlKE5T7eJwp+tnRYM89HmRvWM5EQuB5BoHEONjmQ8lSNmBUwOyy/GFRsohJED51YBF79hE1djug==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-logical-assignment-operators instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-nullish-coalescing-operator@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-nullish-coalescing-operator instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-numeric-separator@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-numeric-separator instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-object-rest-spread@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-object-rest-spread instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-optional-catch-binding@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-catch-binding instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-private-methods@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-methods instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-private-property-in-object@7.21.0-placeholder-for-preset-env.2(@babel/core@7.23.6): + resolution: {integrity: sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@babel/plugin-proposal-private-property-in-object@7.21.11(@babel/core@7.23.6): + resolution: {integrity: sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-property-in-object instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.23.6): + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.23.6): + resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-dynamic-import@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-assertions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-lPgDSU+SJLK3xmFDTV2ZRQAiM7UuUjGidwBywFavObCiZc1BeAAcMtHJKUya92hPHO+at63JJPLygilZard8jw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-attributes@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-pawnE0P9g10xgoP7yKr6CK63K2FMsTE+FZidZO/1PwRdzmAPVs+HS1mAURUsgaoxammTJvULUdIkEK0gOcU2tA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-jsx@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-typescript@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-9EiNjVJOMwCO+43TqoTrgQ8jMwcAd0sWyXi9RPfIsLTj4R2MADDDQXELhffaUx/uJv2AYcxBgPwH6j4TIA4ytQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-unicode-sets-regex@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-arrow-functions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-NzQcQrzaQPkaEwoTm4Mhyl8jI1huEL/WWIEvudjTCMJ9aBZNpsJbMASx7EQECtQQPS/DcnFpo0FIh3LvEO9cxQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-async-generator-functions@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-PdxEpL71bJp1byMG0va5gwQcXHxuEYC/BgI/e88mGTtohbZN28O5Yit0Plkkm/dBzCF/BxmbNcses1RH1T+urA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-async-to-generator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-A7LFsKi4U4fomjqXJlZg/u0ft/n8/7n7lpffUP/ZULx/DtV9SGlNKZolHH6PE8Xl1ngCc0M11OaeZptXVkfKSw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-block-scoped-functions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vI+0sIaPIO6CNuM9Kk5VmXcMVRiOpDh7w2zZt9GXzmE/9KD70CUEVhvPR/etAeNK/FAEkhxQtXOzVF3EuRL41A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-block-scoping@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-0QqbP6B6HOh7/8iNR4CQU2Th/bbRtBp4KS9vcaZd1fZ0wSh5Fyssg0UCIHwxh+ka+pNDREbVLQnHCMHKZfPwfw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-class-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-uM+AN8yCIjDPccsKGlw271xjJtGii+xQIF/uMPS8H15L12jZTsLfF4o5vNO7d/oUguOyfdikHGc/yi9ge4SGIg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-class-static-block@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-nsWu/1M+ggti1SOALj3hfx5FXzAY06fwPJsUZD4/A5e1bWi46VUIWtD+kOX6/IdhXGsXBWllLFDSnqSCdUNydQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-classes@7.23.5(@babel/core@7.23.6): + resolution: {integrity: sha512-jvOTR4nicqYC9yzOHIhXG5emiFEOpappSJAl73SDSEDcybD+Puuze8Tnpb9p9qEyYup24tq891gkaygIFvWDqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + '@babel/helper-split-export-declaration': 7.22.6 + globals: 11.12.0 + dev: false + + /@babel/plugin-transform-computed-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-dTj83UVTLw/+nbiHqQSFdwO9CbTtwq1DsDqm3CUEtDrZNET5rT5E6bIdTlOftDTDLMYxvxHNEYO4B9SLl8SLZw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/template': 7.22.15 + dev: false + + /@babel/plugin-transform-destructuring@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-n225npDqjDIr967cMScVKHXJs7rout1q+tt50inyBCPkyZ8KxeI6d+GIbSBTT/w/9WdlWDOej3V9HE5Lgk57gw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-dotall-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vgnFYDHAKzFaTVp+mneDsIEbnJ2Np/9ng9iviHw3P/KVcgONxpNULEW/51Z/BaFojG2GI2GwwXck5uV1+1NOYQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-duplicate-keys@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-RrqQ+BQmU3Oyav3J+7/myfvRCq7Tbz+kKLLshUmMwNlDHExbGL7ARhajvoBJEvc+fCguPPu887N+3RRXBVKZUA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-dynamic-import@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-V6jIbLhdJK86MaLh4Jpghi8ho5fGzt3imHOBu/x0jlBaPYqDoWz4RDXjmMOfnh+JWNaQleEAByZLV0QzBT4YQQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-exponentiation-operator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-5fhCsl1odX96u7ILKHBj4/Y8vipoqwsJMh4csSA8qFfxrZDEA4Ssku2DyNvMJSmZNOEBT750LfFPbtrnTP90BQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-builder-binary-assignment-operator-visitor': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-export-namespace-from@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-GzuSBcKkx62dGzZI1WVgTWvkkz84FZO5TC5T8dl/Tht/rAla6Dg/Mz9Yhypg+ezVACf/rgDuQt3kbWEv7LdUDQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-for-of@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-aYH4ytZ0qSuBbpfhuofbg/e96oQ7U2w1Aw/UQmKT+1l39uEhUPoFS3fHevDc1G0OvewyDudfMKY1OulczHzWIw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: false + + /@babel/plugin-transform-function-name@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-I1QXp1LxIvt8yLaib49dRW5Okt7Q4oaxao6tFVKS/anCdEOMtYwWVKoiOA1p34GOWIZjUK0E+zCp7+l1pfQyiw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-json-strings@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-81nTOqM1dMwZ/aRXQ59zVubN9wHGqk6UtqRK+/q+ciXmRy8fSolhGVvG09HHRGo4l6fr/c4ZhXUQH0uFW7PZbg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-wZ0PIXRxnwZvl9AYpqNUxpZ5BiTGrYt7kueGQ+N5FiQ7RCOD4cm8iShd6S6ggfVIWaJf2EMk8eRzAh52RfP4rQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-logical-assignment-operators@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-Mc/ALf1rmZTP4JKKEhUwiORU+vcfarFVLfcFiolKUo6sewoxSEgl36ak5t+4WamRsNr6nzjZXQjM35WsU+9vbg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-member-expression-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-sC3LdDBDi5x96LA+Ytekz2ZPk8i/Ck+DEuDbRAll5rknJ5XRTSaPKEYwomLcs1AA8wg9b3KjIQRsnApj+q51Ag==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-amd@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vJYQGxeKM4t8hYCKVBlZX/gtIY2I7mRGFNcm85sgXGMTBcoV3QdVtdpbcWEbzbfUIUZKwvgFT82mRvaQIebZzw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-commonjs@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-aVS0F65LKsdNOtcz6FRCpE4OgsP2OFnW46qNxNIX9h3wuzaNcSQsJysuMwqSibC98HPrf2vCgtxKNwS0DAlgcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-systemjs@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-ZxyKGTkF9xT9YJuKQRo19ewf3pXpopuYQd8cDXqNzc3mUNbOME0RKMoZxviQk74hwzfQsEe66dE92MaZbdHKNQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-identifier': 7.22.20 + dev: false + + /@babel/plugin-transform-modules-umd@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-zHsy9iXX2nIsCBFPud3jKn1IRPWg3Ing1qOZgeKV39m1ZgIdpJqvlWVeiHBZC6ITRG0MfskhYe9cLgntfSFPIg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-named-capturing-groups-regex@7.22.5(@babel/core@7.23.6): + resolution: {integrity: sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-new-target@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-YJ3xKqtJMAT5/TIZnpAR3I+K+WaDowYbN3xyxI8zxx/Gsypwf9B9h0VB+1Nh6ACAAPRS5NSRje0uVv5i79HYGQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-nullish-coalescing-operator@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-jHE9EVVqHKAQx+VePv5LLGHjmHSJR76vawFPTdlxR/LVJPfOEGxREQwQfjuZEOPTwG92X3LINSh3M40Rv4zpVA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-numeric-separator@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-mps6auzgwjRrwKEZA05cOwuDc9FAzoyFS4ZsG/8F43bTLf/TgkJg7QXOrPO1JO599iA3qgK9MXdMGOEC8O1h6Q==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-object-rest-spread@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9x9K1YyeQVw0iOXJlIzwm8ltobIIv7j2iLyP2jIhEbqPRQ7ScNgwQufU2I0Gq11VjyG4gI4yMXt2VFags+1N3g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-object-super@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-BwQ8q0x2JG+3lxCVFohg+KbQM7plfpBwThdW9A6TMtWwLsbDA01Ek2Zb/AgDN39BiZsExm4qrXxjk+P1/fzGrA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-optional-catch-binding@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-XIq8t0rJPHf6Wvmbn9nFxU6ao4c7WhghTR5WyV8SrJfUFzyxhCm4nhC+iAp3HFhbAKLfYpgzhJ6t4XCtVwqO5A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-optional-chaining@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-ZU8y5zWOfjM5vZ+asjgAPwDaBjJzgufjES89Rs4Lpq63O300R/kOz30WCLo6BxxX6QVEilwSlpClnG5cZaikTA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-parameters@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-09lMt6UsUb3/34BbECKVbVwrT9bO6lILWln237z7sLaWnMsTi7Yc9fhX5DLpkJzAGfaReXI22wP41SZmnAA3Vw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-private-methods@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-UzqRcRtWsDMTLrRWFvUBDwmw06tCQH9Rl1uAjfh6ijMSmGYQ+fpdB+cnqRC8EMh5tuuxSv0/TejGL+7vyj+50g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-private-property-in-object@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9G3K1YqTq3F4Vt88Djx1UZ79PDyj+yKRnUy7cZGSMe+a7jkwD259uKKuUzQlPkGam7R+8RJwh5z4xO27fA1o2A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-property-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-jR3Jn3y7cZp4oEWPFAlRsSWjxKe4PZILGBSd4nis1TsC5qeSpb+nrtihJuDhNI7QHiVbUaiXa0X2RZY3/TI6Nw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-constant-elements@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-zP0QKq/p6O42OL94udMgSfKXyse4RyJ0JqbQ34zDAONWjyrEsghYEyTSK5FIpmXmCpB55SHokL1cRRKHv8L2Qw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-display-name@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-GnvhtVfA2OAtzdX58FJxU19rhoGeQzyVndw3GgtdECQvQFXPEZIOVULHVZGAYmOgmqjXpVpfocAbSjh99V/Fqw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-jsx-development@7.22.5(@babel/core@7.23.6): + resolution: {integrity: sha512-bDhuzwWMuInwCYeDeMzyi7TaBgRQei6DqxhbyniL7/VG4RSS7HtSL2QbY4eESy1KJqlWt8g3xeEBGPuo+XqC8A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/plugin-transform-react-jsx': 7.23.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-react-jsx@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-5xOpoPguCZCRbo/JeHlloSkTA8Bld1J/E1/kLfD1nsuiW1m8tduTA1ERCgIZokDflX/IBzKcqR3l7VlRgiIfHA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-jsx': 7.23.3(@babel/core@7.23.6) + '@babel/types': 7.23.6 + dev: false + + /@babel/plugin-transform-react-pure-annotations@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-qMFdSS+TUhB7Q/3HVPnEdYJDQIk57jkntAwSuz9xfSE4n+3I+vHYCli3HoHawN1Z3RfCz/y1zXA/JXjG6cVImQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-regenerator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KP+75h0KghBMcVpuKisx3XTu9Ncut8Q8TuvGO4IhY+9D5DFEckQefOuIsB/gQ2tG71lCke4NMrtIPS8pOj18BQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + regenerator-transform: 0.15.2 + dev: false + + /@babel/plugin-transform-reserved-words@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-QnNTazY54YqgGxwIexMZva9gqbPa15t/x9VS+0fsEFWplwVpXYZivtgl43Z1vMpc1bdPP2PP8siFeVcnFvA3Cg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-runtime@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-kF1Zg62aPseQ11orDhFRw+aPG/eynNQtI+TyY+m33qJa2cJ5EEvza2P2BNTIA9E5MyqFABHEyY6CPHwgdy9aNg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + babel-plugin-polyfill-corejs2: 0.4.7(@babel/core@7.23.6) + babel-plugin-polyfill-corejs3: 0.8.7(@babel/core@7.23.6) + babel-plugin-polyfill-regenerator: 0.5.4(@babel/core@7.23.6) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/plugin-transform-shorthand-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-ED2fgqZLmexWiN+YNFX26fx4gh5qHDhn1O2gvEhreLW2iI63Sqm4llRLCXALKrCnbN4Jy0VcMQZl/SAzqug/jg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-spread@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-VvfVYlrlBVu+77xVTOAoxQ6mZbnIq5FM0aGBSFEcIh03qHf+zNqA4DC/3XMUozTg7bZV3e3mZQ0i13VB6v5yUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: false + + /@babel/plugin-transform-sticky-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-HZOyN9g+rtvnOU3Yh7kSxXrKbzgrm5X4GncPY1QOquu7epga5MxKHVpYu2hvQnry/H+JjckSYRb93iNfsioAGg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-template-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-Flok06AYNp7GV2oJPZZcP9vZdszev6vPBkHLwxwSpaIqx75wn6mUd3UFWsSsA0l8nXAKkyCmL/sR02m8RYGeHg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-typeof-symbol@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-4t15ViVnaFdrPC74be1gXBSMzXk3B4Us9lP7uLRQHTFpV5Dvt33pn+2MyyNxmN3VTTm3oTrZVMUmuw3oBnQ2oQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-typescript@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-6cBG5mBvUu4VUD04OHKnYzbuHNP8huDsD3EDqqpIpsswTDoqHCjLoHb6+QgsV1WsT2nipRqCPgxD3LXnEO7XfA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-unicode-escapes@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-OMCUx/bU6ChE3r4+ZdylEqAjaQgHAgipgW8nsCfu5pGqDcFytVd91AwRvUJSBZDz0exPGgnjoqhgRYLRjFZc9Q==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-property-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KcLIm+pDZkWZQAFJ9pdfmh89EwVfmNovFBcXko8szpBeF8z68kWIPeKlmSOkT9BXJxs2C0uk+5LxoxIv62MROA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-wMHpNA4x2cIA32b/ci3AfwNgheiva2W0WUKWTK7vBHBhDKfPsc5cFGNWm69WBqpwd86u1qwZ9PWevKqm1A3yAw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-sets-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-W7lliA/v9bNR83Qc3q1ip9CQMZ09CcHDbHfbLRDNuAhn1Mvkr1ZNF7hPmztMQvtTGVLJ9m8IZqWsTkXOml8dbw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/preset-env@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-2XPn/BqKkZCpzYhUUNZ1ssXw7DcXfKQEjv/uXZUXgaebCMYmkEsfZ2yY+vv+xtXv50WmL5SGhyB6/xsWxIvvOQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly': 7.23.7(@babel/core@7.23.6) + '@babel/plugin-proposal-private-property-in-object': 7.21.0-placeholder-for-preset-env.2(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.23.6) + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-assertions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-attributes': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-unicode-sets-regex': 7.18.6(@babel/core@7.23.6) + '@babel/plugin-transform-arrow-functions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-async-generator-functions': 7.23.7(@babel/core@7.23.6) + '@babel/plugin-transform-async-to-generator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoped-functions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoping': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-class-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-class-static-block': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-classes': 7.23.5(@babel/core@7.23.6) + '@babel/plugin-transform-computed-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-destructuring': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-dotall-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-duplicate-keys': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-dynamic-import': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-exponentiation-operator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-export-namespace-from': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-for-of': 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-function-name': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-json-strings': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-logical-assignment-operators': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-member-expression-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-amd': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-systemjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-umd': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-named-capturing-groups-regex': 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-new-target': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-nullish-coalescing-operator': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-numeric-separator': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-object-rest-spread': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-object-super': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-optional-catch-binding': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-optional-chaining': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-private-methods': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-private-property-in-object': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-property-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-regenerator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-reserved-words': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-shorthand-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-spread': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-sticky-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-template-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typeof-symbol': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-escapes': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-property-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-sets-regex': 7.23.3(@babel/core@7.23.6) + '@babel/preset-modules': 0.1.6-no-external-plugins(@babel/core@7.23.6) + babel-plugin-polyfill-corejs2: 0.4.7(@babel/core@7.23.6) + babel-plugin-polyfill-corejs3: 0.8.7(@babel/core@7.23.6) + babel-plugin-polyfill-regenerator: 0.5.4(@babel/core@7.23.6) + core-js-compat: 3.35.0 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.23.6): + resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==} + peerDependencies: + '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/types': 7.23.6 + esutils: 2.0.3 + dev: false + + /@babel/preset-react@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-tbkHOS9axH6Ysf2OUEqoSZ6T3Fa2SrNH6WTWSPBboxKzdxNc9qOICeLXkNG0ZEwbQ1HY8liwOce4aN/Ceyuq6w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-transform-react-display-name': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx-development': 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-react-pure-annotations': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/preset-typescript@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-17oIGVlqz6CchO9RFYn5U6ZpWRZIngayYCtrPRSgANSwC2V1Jb+iP74nVxzzXJte8b8BYxrL1yY96xfhTBrNNQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-syntax-jsx': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typescript': 7.23.6(@babel/core@7.23.6) + dev: false + + /@babel/regjsgen@0.8.0: + resolution: {integrity: sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==} + dev: false + + /@babel/runtime-corejs3@7.23.6: + resolution: {integrity: sha512-Djs/ZTAnpyj0nyg7p1J6oiE/tZ9G2stqAFlLGZynrW+F3k2w2jGK2mLOBxzYIOcZYA89+c3d3wXKpYLcpwcU6w==} + engines: {node: '>=6.9.0'} + dependencies: + core-js-pure: 3.35.0 + regenerator-runtime: 0.14.1 + dev: false + + /@babel/runtime@7.23.7: + resolution: {integrity: sha512-w06OXVOFso7LcbzMiDGt+3X7Rh7Ho8MmgPoWU3rarH+8upf+wSU/grlGbWzQyr3DkdN6ZeuMFjpdwW0Q+HxobA==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.1 + dev: false + + /@babel/template@7.22.15: + resolution: {integrity: sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/parser': 7.23.6 + '@babel/types': 7.23.6 + dev: false + + /@babel/traverse@7.23.7: + resolution: {integrity: sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.23.6 + '@babel/types': 7.23.6 + debug: 4.3.4 + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/types@7.23.6: + resolution: {integrity: sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.23.4 + '@babel/helper-validator-identifier': 7.22.20 + to-fast-properties: 2.0.0 + dev: false + + /@braintree/sanitize-url@6.0.4: + resolution: {integrity: sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==} + dev: false + + /@cmfcmf/docusaurus-search-local@1.1.0(@docusaurus/core@3.0.1)(search-insights@2.13.0): + resolution: {integrity: sha512-0IVb/aA0IK8ZlktuxmgXmluXfcSpo6Vdd2nG21y1aOH9nVYnPP231Dn0H8Ng9Qf9ronQQCDWHnuWpYOr9rUrEQ==} + peerDependencies: + '@docusaurus/core': ^2.0.0 + nodejieba: ^2.5.0 + peerDependenciesMeta: + nodejieba: + optional: true + dependencies: + '@algolia/autocomplete-js': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-theme-classic': 1.13.0 + '@algolia/client-search': 4.22.0 + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + algoliasearch: 4.22.0 + cheerio: 1.0.0-rc.12 + clsx: 1.1.1 + lunr-languages: 1.14.0 + mark.js: 8.11.1 + transitivePeerDependencies: + - search-insights + dev: false + + /@colors/colors@1.5.0: + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + requiresBuild: true + dev: false + optional: true + + /@discoveryjs/json-ext@0.5.7: + resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} + engines: {node: '>=10.0.0'} + dev: false + + /@docsearch/css@3.1.0: + resolution: {integrity: sha512-bh5IskwkkodbvC0FzSg1AxMykfDl95hebEKwxNoq4e5QaGzOXSBgW8+jnMFZ7JU4sTBiB04vZWoUSzNrPboLZA==} + dev: false + + /@docsearch/css@3.5.2: + resolution: {integrity: sha512-SPiDHaWKQZpwR2siD0KQUwlStvIAnEyK6tAE2h2Wuoq8ue9skzhlyVQ1ddzOxX6khULnAALDiR/isSF3bnuciA==} + dev: false + + /@docsearch/react@3.1.0(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-bjB6ExnZzf++5B7Tfoi6UXgNwoUnNOfZ1NyvnvPhWgCMy5V/biAtLL4o7owmZSYdAKeFSvZ5Lxm0is4su/dBWg==} + peerDependencies: + '@types/react': '>= 16.8.0 < 19.0.0' + react: '>= 16.8.0 < 19.0.0' + react-dom: '>= 16.8.0 < 19.0.0' + dependencies: + '@algolia/autocomplete-core': 1.6.3 + '@docsearch/css': 3.1.0 + '@types/react': 18.2.46 + algoliasearch: 4.22.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: false + + /@docsearch/react@3.5.2(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0): + resolution: {integrity: sha512-9Ahcrs5z2jq/DcAvYtvlqEBHImbm4YJI8M9y0x6Tqg598P40HTEkX7hsMcIuThI+hTFxRGZ9hll0Wygm2yEjng==} + peerDependencies: + '@types/react': '>= 16.8.0 < 19.0.0' + react: '>= 16.8.0 < 19.0.0' + react-dom: '>= 16.8.0 < 19.0.0' + search-insights: '>= 1 < 3' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + react-dom: + optional: true + search-insights: + optional: true + dependencies: + '@algolia/autocomplete-core': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-preset-algolia': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@docsearch/css': 3.5.2 + '@types/react': 18.2.46 + algoliasearch: 4.22.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + dev: false + + /@docusaurus/core@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-CXrLpOnW+dJdSv8M5FAJ3JBwXtL6mhUWxFA8aS0ozK6jBG/wgxERk5uvH28fCeFxOGbAT9v1e9dOMo1X2IEVhQ==} + engines: {node: '>=18.0'} + hasBin: true + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/generator': 7.23.6 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-runtime': 7.23.6(@babel/core@7.23.6) + '@babel/preset-env': 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': 7.23.3(@babel/core@7.23.6) + '@babel/runtime': 7.23.7 + '@babel/runtime-corejs3': 7.23.6 + '@babel/traverse': 7.23.7 + '@docusaurus/cssnano-preset': 3.0.1 + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/react-loadable': 5.5.2(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@slorber/static-site-generator-webpack-plugin': 4.0.7 + '@svgr/webpack': 6.5.1 + autoprefixer: 10.4.16(postcss@8.4.32) + babel-loader: 9.1.3(@babel/core@7.23.6)(webpack@5.89.0) + babel-plugin-dynamic-import-node: 2.3.3 + boxen: 6.2.1 + chalk: 4.1.2 + chokidar: 3.5.3 + clean-css: 5.3.3 + cli-table3: 0.6.3 + combine-promises: 1.2.0 + commander: 5.1.0 + copy-webpack-plugin: 11.0.0(webpack@5.89.0) + core-js: 3.35.0 + css-loader: 6.8.1(webpack@5.89.0) + css-minimizer-webpack-plugin: 4.2.2(clean-css@5.3.3)(webpack@5.89.0) + cssnano: 5.1.15(postcss@8.4.32) + del: 6.1.1 + detect-port: 1.5.1 + escape-html: 1.0.3 + eta: 2.2.0 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + html-minifier-terser: 7.2.0 + html-tags: 3.3.1 + html-webpack-plugin: 5.6.0(webpack@5.89.0) + leven: 3.1.0 + lodash: 4.17.21 + mini-css-extract-plugin: 2.7.6(webpack@5.89.0) + postcss: 8.4.32 + postcss-loader: 7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0) + prompts: 2.4.2 + react: 18.2.0 + react-dev-utils: 12.0.1(typescript@5.3.3)(webpack@5.89.0) + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 1.3.0(react-dom@18.2.0)(react@18.2.0) + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + react-loadable-ssr-addon-v5-slorber: 1.0.1(@docusaurus/react-loadable@5.5.2)(webpack@5.89.0) + react-router: 5.3.4(react@18.2.0) + react-router-config: 5.1.1(react-router@5.3.4)(react@18.2.0) + react-router-dom: 5.3.4(react@18.2.0) + rtl-detect: 1.1.2 + semver: 7.5.4 + serve-handler: 6.1.5 + shelljs: 0.8.5 + terser-webpack-plugin: 5.3.10(webpack@5.89.0) + tslib: 2.6.2 + update-notifier: 6.0.2 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + webpack: 5.89.0 + webpack-bundle-analyzer: 4.10.1 + webpack-dev-server: 4.15.1(webpack@5.89.0) + webpack-merge: 5.10.0 + webpackbar: 5.0.2(webpack@5.89.0) + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/cssnano-preset@3.0.1: + resolution: {integrity: sha512-wjuXzkHMW+ig4BD6Ya1Yevx9UJadO4smNZCEljqBoQfIQrQskTswBs7lZ8InHP7mCt273a/y/rm36EZhqJhknQ==} + engines: {node: '>=18.0'} + dependencies: + cssnano-preset-advanced: 5.3.10(postcss@8.4.32) + postcss: 8.4.32 + postcss-sort-media-queries: 4.4.1(postcss@8.4.32) + tslib: 2.6.2 + dev: false + + /@docusaurus/logger@3.0.1: + resolution: {integrity: sha512-I5L6Nk8OJzkVA91O2uftmo71LBSxe1vmOn9AMR6JRCzYeEBrqneWMH02AqMvjJ2NpMiviO+t0CyPjyYV7nxCWQ==} + engines: {node: '>=18.0'} + dependencies: + chalk: 4.1.2 + tslib: 2.6.2 + dev: false + + /@docusaurus/mdx-loader@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-ldnTmvnvlrONUq45oKESrpy+lXtbnTcTsFkOTIDswe5xx5iWJjt6eSa0f99ZaWlnm24mlojcIGoUWNCS53qVlQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@babel/parser': 7.23.6 + '@babel/traverse': 7.23.7 + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@mdx-js/mdx': 3.0.0 + '@slorber/remark-comment': 1.0.0 + escape-html: 1.0.3 + estree-util-value-to-estree: 3.0.1 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + image-size: 1.1.1 + mdast-util-mdx: 3.0.0 + mdast-util-to-string: 4.0.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + rehype-raw: 7.0.0 + remark-directive: 3.0.0 + remark-emoji: 4.0.1 + remark-frontmatter: 5.0.0 + remark-gfm: 4.0.0 + stringify-object: 3.3.0 + tslib: 2.6.2 + unified: 11.0.4 + unist-util-visit: 5.0.0 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + vfile: 6.0.1 + webpack: 5.89.0 + transitivePeerDependencies: + - '@docusaurus/types' + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/module-type-aliases@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-DEHpeqUDsLynl3AhQQiO7AbC7/z/lBra34jTcdYuvp9eGm01pfH1wTVq8YqWZq6Jyx0BgcVl/VJqtE9StRd9Ag==} + peerDependencies: + react: '*' + react-dom: '*' + dependencies: + '@docusaurus/react-loadable': 5.5.2(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router-config': 5.0.11 + '@types/react-router-dom': 5.3.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 2.0.4(react-dom@18.2.0)(react@18.2.0) + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/plugin-client-redirects@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-CoZapnHbV3j5jsHCa/zmKaa8+H+oagHBgg91dN5I8/3kFit/xtZPfRaznvDX49cHg2nSoV74B3VMAT+bvCmzFQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + eta: 2.2.0 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-blog@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-cLOvtvAyaMQFLI8vm4j26svg3ktxMPSXpuUJ7EERKoGbfpJSsgtowNHcRsaBVmfuCsRSk1HZ/yHBsUkTmHFEsg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + cheerio: 1.0.0-rc.12 + feed: 4.2.2 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + reading-time: 1.5.0 + srcset: 4.0.0 + tslib: 2.6.2 + unist-util-visit: 5.0.0 + utility-types: 3.10.0 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-docs@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-dRfAOA5Ivo+sdzzJGXEu33yAtvGg8dlZkvt/NEJ7nwi1F2j4LEdsxtfX2GKeETB2fP6XoGNSQnFXqa2NYGrHFg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@types/react-router-config': 5.0.11 + combine-promises: 1.2.0 + fs-extra: 11.2.0 + js-yaml: 4.1.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-pages@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-oP7PoYizKAXyEttcvVzfX3OoBIXEmXTMzCdfmC4oSwjG4SPcJsRge3mmI6O8jcZBgUPjIzXD21bVGWEE1iu8gg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-debug@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-09dxZMdATky4qdsZGzhzlUvvC+ilQ2hKbYF+wez+cM2mGo4qHbv8+qKXqxq0CQZyimwlAOWQLoSozIXU0g0i7g==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-json-view-lite: 1.2.1(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-analytics@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-jwseSz1E+g9rXQwDdr0ZdYNjn8leZBnKPjjQhMBEiwDoenL3JYFcNW0+p0sWoVF/f2z5t7HkKA+cYObrUh18gg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-gtag@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-UFTDvXniAWrajsulKUJ1DB6qplui1BlKLQZjX4F7qS/qfJ+qkKqSkhJ/F4VuGQ2JYeZstYb+KaUzUzvaPK1aRQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@types/gtag.js': 0.0.12 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-tag-manager@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-IPFvuz83aFuheZcWpTlAdiiX1RqWIHM+OH8wS66JgwAKOiQMR3+nLywGjkLV4bp52x7nCnwhNk1rE85Cpy/CIw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-sitemap@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-xARiWnjtVvoEniZudlCq5T9ifnhCu/GAZ5nA7XgyLfPcNpHQa241HZdsTlLtVcecEVVdllevBKOp7qknBBaMGw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + sitemap: 7.1.1 + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/preset-classic@3.0.1(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3): + resolution: {integrity: sha512-il9m9xZKKjoXn6h0cRcdnt6wce0Pv1y5t4xk2Wx7zBGhKG1idu4IFHtikHlD0QPuZ9fizpXspXcTzjL5FXc1Gw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-debug': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-analytics': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-gtag': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-tag-manager': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-sitemap': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-classic': 3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-search-algolia': 3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@algolia/client-search' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - search-insights + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/react-loadable@5.5.2(react@18.2.0): + resolution: {integrity: sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==} + peerDependencies: + react: '*' + dependencies: + '@types/react': 18.2.46 + prop-types: 15.8.1 + react: 18.2.0 + dev: false + + /@docusaurus/theme-classic@3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-XD1FRXaJiDlmYaiHHdm27PNhhPboUah9rqIH0lMpBt5kYtsGjJzhqa27KuZvHLzOP2OEpqd2+GZ5b6YPq7Q05Q==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-translations': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@mdx-js/react': 3.0.0(@types/react@18.2.46)(react@18.2.0) + clsx: 2.1.0 + copy-text-to-clipboard: 3.2.0 + infima: 0.2.0-alpha.43 + lodash: 4.17.21 + nprogress: 0.2.0 + postcss: 8.4.32 + prism-react-renderer: 2.3.1(react@18.2.0) + prismjs: 1.29.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-router-dom: 5.3.4(react@18.2.0) + rtlcss: 4.1.1 + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-common@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-cr9TOWXuIOL0PUfuXv6L5lPlTgaphKP+22NdVBOYah5jSq5XAAulJTjfe+IfLsEG4L7lJttLbhW7LXDFSAI7Ag==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router-config': 5.0.11 + clsx: 2.1.0 + parse-numeric-range: 1.3.0 + prism-react-renderer: 2.3.1(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-mermaid@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-jquSDnZfazABnC5i+02GzRIvufXKruKgvbYkQjKbI7/LWo0XvBs0uKAcCDGgHhth0t/ON5+Sn27joRfpeSk3Lw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + mermaid: 10.6.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-search-algolia@3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3): + resolution: {integrity: sha512-DDiPc0/xmKSEdwFkXNf1/vH1SzJPzuJBar8kMcBbDAZk/SAmo/4lf6GU2drou4Ae60lN2waix+jYWTWcJRahSA==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docsearch/react': 3.5.2(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0) + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-translations': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + algoliasearch: 4.22.0 + algoliasearch-helper: 3.16.1(algoliasearch@4.22.0) + clsx: 2.1.0 + eta: 2.2.0 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@algolia/client-search' + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - search-insights + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-translations@3.0.1: + resolution: {integrity: sha512-6UrbpzCTN6NIJnAtZ6Ne9492vmPVX+7Fsz4kmp+yor3KQwA1+MCzQP7ItDNkP38UmVLnvB/cYk/IvehCUqS3dg==} + engines: {node: '>=18.0'} + dependencies: + fs-extra: 11.2.0 + tslib: 2.6.2 + dev: false + + /@docusaurus/types@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-plyX2iU1tcUsF46uQ01pAd4JhexR7n0iiQ5MSnBFX6M6NSJgDYdru/i1/YNPKOnQHBoXGLHv0dNT6OAlDWNjrg==} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + commander: 5.1.0 + joi: 17.11.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 1.3.0(react-dom@18.2.0)(react@18.2.0) + utility-types: 3.10.0 + webpack: 5.89.0 + webpack-merge: 5.10.0 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/utils-common@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-W0AxD6w6T8g6bNro8nBRWf7PeZ/nn7geEWM335qHU2DDDjHuV4UZjgUGP1AQsdcSikPrlIqTJJbKzer1lRSlIg==} + engines: {node: '>=18.0'} + peerDependencies: + '@docusaurus/types': '*' + peerDependenciesMeta: + '@docusaurus/types': + optional: true + dependencies: + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + tslib: 2.6.2 + dev: false + + /@docusaurus/utils-validation@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-ujTnqSfyGQ7/4iZdB4RRuHKY/Nwm58IIb+41s5tCXOv/MBU2wGAjOHq3U+AEyJ8aKQcHbxvTKJaRchNHYUVUQg==} + engines: {node: '>=18.0'} + dependencies: + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + joi: 17.11.0 + js-yaml: 4.1.0 + tslib: 2.6.2 + transitivePeerDependencies: + - '@docusaurus/types' + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/utils@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-TwZ33Am0q4IIbvjhUOs+zpjtD/mXNmLmEgeTGuRq01QzulLHuPhaBTTAC/DHu6kFx3wDgmgpAlaRuCHfTcXv8g==} + engines: {node: '>=18.0'} + peerDependencies: + '@docusaurus/types': '*' + peerDependenciesMeta: + '@docusaurus/types': + optional: true + dependencies: + '@docusaurus/logger': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@svgr/webpack': 6.5.1 + escape-string-regexp: 4.0.0 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + github-slugger: 1.5.0 + globby: 11.1.0 + gray-matter: 4.0.3 + jiti: 1.21.0 + js-yaml: 4.1.0 + lodash: 4.17.21 + micromatch: 4.0.5 + resolve-pathname: 3.0.0 + shelljs: 0.8.5 + tslib: 2.6.2 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + webpack: 5.89.0 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@fortawesome/fontawesome-common-types@6.5.1: + resolution: {integrity: sha512-GkWzv+L6d2bI5f/Vk6ikJ9xtl7dfXtoRu3YGE6nq0p/FFqA1ebMOAWg3XgRyb0I6LYyYkiAo+3/KrwuBp8xG7A==} + engines: {node: '>=6'} + requiresBuild: true + dev: false + + /@fortawesome/fontawesome-svg-core@6.5.1: + resolution: {integrity: sha512-MfRCYlQPXoLlpem+egxjfkEuP9UQswTrlCOsknus/NcMoblTH2g0jPrapbcIb04KGA7E2GZxbAccGZfWoYgsrQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/free-regular-svg-icons@6.5.1: + resolution: {integrity: sha512-m6ShXn+wvqEU69wSP84coxLbNl7sGVZb+Ca+XZq6k30SzuP3X4TfPqtycgUh9ASwlNh5OfQCd8pDIWxl+O+LlQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/free-solid-svg-icons@6.5.1: + resolution: {integrity: sha512-S1PPfU3mIJa59biTtXJz1oI0+KAXW6bkAb31XKhxdxtuXDiUIFsih4JR1v5BbxY7hVHsD1RKq+jRkVRaf773NQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/react-fontawesome@0.2.0(@fortawesome/fontawesome-svg-core@6.5.1)(react@18.2.0): + resolution: {integrity: sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==} + peerDependencies: + '@fortawesome/fontawesome-svg-core': ~1 || ~6 + react: '>=16.3' + dependencies: + '@fortawesome/fontawesome-svg-core': 6.5.1 + prop-types: 15.8.1 + react: 18.2.0 + dev: false + + /@hapi/hoek@9.3.0: + resolution: {integrity: sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==} + dev: false + + /@hapi/topo@5.1.0: + resolution: {integrity: sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==} + dependencies: + '@hapi/hoek': 9.3.0 + dev: false + + /@headlessui/react@1.7.18(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-4i5DOrzwN4qSgNsL4Si61VMkUcWbcSKueUV7sFhpHzQcSShdlHENE5+QBntMSRvHt8NyoFO2AGG8si9lq+w4zQ==} + engines: {node: '>=10'} + peerDependencies: + react: ^16 || ^17 || ^18 + react-dom: ^16 || ^17 || ^18 + dependencies: + '@tanstack/react-virtual': 3.0.1(react-dom@18.2.0)(react@18.2.0) + client-only: 0.0.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: false + + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: false + + /@jest/types@29.6.3: + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 20.10.6 + '@types/yargs': 17.0.32 + chalk: 4.1.2 + dev: false + + /@jridgewell/gen-mapping@0.3.3: + resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/set-array': 1.1.2 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@jridgewell/resolve-uri@3.1.1: + resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} + engines: {node: '>=6.0.0'} + dev: false + + /@jridgewell/set-array@1.1.2: + resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} + engines: {node: '>=6.0.0'} + dev: false + + /@jridgewell/source-map@0.3.5: + resolution: {integrity: sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@jridgewell/sourcemap-codec@1.4.15: + resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + dev: false + + /@jridgewell/trace-mapping@0.3.20: + resolution: {integrity: sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==} + dependencies: + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 + dev: false + + /@leichtgewicht/ip-codec@2.0.4: + resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} + dev: false + + /@mdx-js/mdx@3.0.0: + resolution: {integrity: sha512-Icm0TBKBLYqroYbNW3BPnzMGn+7mwpQOK310aZ7+fkCtiU3aqv2cdcX+nd0Ydo3wI5Rx8bX2Z2QmGb/XcAClCw==} + dependencies: + '@types/estree': 1.0.5 + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdx': 2.0.10 + collapse-white-space: 2.1.0 + devlop: 1.1.0 + estree-util-build-jsx: 3.0.1 + estree-util-is-identifier-name: 3.0.0 + estree-util-to-js: 2.0.0 + estree-walker: 3.0.3 + hast-util-to-estree: 3.1.0 + hast-util-to-jsx-runtime: 2.3.0 + markdown-extensions: 2.0.0 + periscopic: 3.1.0 + remark-mdx: 3.0.0 + remark-parse: 11.0.0 + remark-rehype: 11.0.0 + source-map: 0.7.4 + unified: 11.0.4 + unist-util-position-from-estree: 2.0.0 + unist-util-stringify-position: 4.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@mdx-js/react@3.0.0(@types/react@18.2.46)(react@18.2.0): + resolution: {integrity: sha512-nDctevR9KyYFyV+m+/+S4cpzCWHqj+iHDHq3QrsWezcC+B17uZdIWgCguESUkwFhM3n/56KxWVE3V6EokrmONQ==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + dependencies: + '@types/mdx': 2.0.10 + '@types/react': 18.2.46 + react: 18.2.0 + dev: false + + /@ndhoule/each@2.0.1: + resolution: {integrity: sha512-wHuJw6x+rF6Q9Skgra++KccjBozCr9ymtna0FhxmV/8xT/hZ2ExGYR8SV8prg8x4AH/7mzDYErNGIVHuzHeybw==} + dependencies: + '@ndhoule/keys': 2.0.0 + dev: false + + /@ndhoule/keys@2.0.0: + resolution: {integrity: sha512-vtCqKBC1Av6dsBA8xpAO+cgk051nfaI+PnmTZep2Px0vYrDvpUmLxv7z40COlWH5yCpu3gzNhepk+02yiQiZNw==} + dev: false + + /@ndhoule/map@2.0.1: + resolution: {integrity: sha512-WOEf2An9mL4DVY6NHgaRmFC82pZGrmzW4I0hpPPdczDP4Gp5+Q1Nny77x3w0qzENA8+cbgd9+Lx2ClSTLvkB0g==} + dependencies: + '@ndhoule/each': 2.0.1 + dev: false + + /@nodelib/fs.scandir@2.1.5: + resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} + engines: {node: '>= 8'} + dependencies: + '@nodelib/fs.stat': 2.0.5 + run-parallel: 1.2.0 + dev: false + + /@nodelib/fs.stat@2.0.5: + resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} + engines: {node: '>= 8'} + dev: false + + /@nodelib/fs.walk@1.2.8: + resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} + engines: {node: '>= 8'} + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.16.0 + dev: false + + /@pnpm/config.env-replace@1.1.0: + resolution: {integrity: sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==} + engines: {node: '>=12.22.0'} + dev: false + + /@pnpm/network.ca-file@1.0.2: + resolution: {integrity: sha512-YcPQ8a0jwYU9bTdJDpXjMi7Brhkr1mXsXrUJvjqM2mQDgkRiz8jFaQGOdaLxgjtUfQgZhKy/O3cG/YwmgKaxLA==} + engines: {node: '>=12.22.0'} + dependencies: + graceful-fs: 4.2.10 + dev: false + + /@pnpm/npm-conf@2.2.2: + resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} + engines: {node: '>=12'} + dependencies: + '@pnpm/config.env-replace': 1.1.0 + '@pnpm/network.ca-file': 1.0.2 + config-chain: 1.1.13 + dev: false + + /@polka/url@1.0.0-next.24: + resolution: {integrity: sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==} + dev: false + + /@segment/snippet@4.16.2: + resolution: {integrity: sha512-2fgsrt4U+vKv14ohOAsViCEzeZotaawF2Il7YUbmYVrhPn8Hq7xuGznHKRdZeoxScQ87X36xDX2Fzh5bAYRN7g==} + dependencies: + '@ndhoule/map': 2.0.1 + dev: false + + /@sideway/address@4.1.4: + resolution: {integrity: sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==} + dependencies: + '@hapi/hoek': 9.3.0 + dev: false + + /@sideway/formula@3.0.1: + resolution: {integrity: sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==} + dev: false + + /@sideway/pinpoint@2.0.0: + resolution: {integrity: sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==} + dev: false + + /@sinclair/typebox@0.27.8: + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + dev: false + + /@sindresorhus/is@4.6.0: + resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} + engines: {node: '>=10'} + dev: false + + /@sindresorhus/is@5.6.0: + resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} + engines: {node: '>=14.16'} + dev: false + + /@slorber/remark-comment@1.0.0: + resolution: {integrity: sha512-RCE24n7jsOj1M0UPvIQCHTe7fI0sFL4S2nwKVWwHyVr/wI/H8GosgsJGyhnsZoGFnD/P2hLf1mSbrrgSLN93NA==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + dev: false + + /@slorber/static-site-generator-webpack-plugin@4.0.7: + resolution: {integrity: sha512-Ug7x6z5lwrz0WqdnNFOMYrDQNTPAprvHLSh6+/fmml3qUiz6l5eq+2MzLKWtn/q5K5NpSiFsZTP/fck/3vjSxA==} + engines: {node: '>=14'} + dependencies: + eval: 0.1.8 + p-map: 4.0.0 + webpack-sources: 3.2.3 + dev: false + + /@svgr/babel-plugin-add-jsx-attribute@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-9PYGcXrAxitycIjRmZB+Q0JaN07GZIWaTBIGQzfaZv+qr1n8X1XUEJ5rZ/vx6OVD9RRYlrNnXWExQXcmZeD/BQ==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-remove-jsx-attribute@8.0.0(@babel/core@7.23.6): + resolution: {integrity: sha512-BcCkm/STipKvbCl6b7QFrMh/vx00vIP63k2eM66MfHJzPr6O2U0jYEViXkHJWqXqQYjdeA9cuCl5KWmlwjDvbA==} + engines: {node: '>=14'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-remove-jsx-empty-expression@8.0.0(@babel/core@7.23.6): + resolution: {integrity: sha512-5BcGCBfBxB5+XSDSWnhTThfI9jcO5f0Ai2V24gZpG+wXF14BzwxxdDb4g6trdOux0rhibGs385BeFMSmxtS3uA==} + engines: {node: '>=14'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-replace-jsx-attribute-value@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-8DPaVVE3fd5JKuIC29dqyMB54sA6mfgki2H2+swh+zNJoynC8pMPzOkidqHOSc6Wj032fhl8Z0TVn1GiPpAiJg==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-svg-dynamic-title@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-FwOEi0Il72iAzlkaHrlemVurgSQRDFbk0OC8dSvD5fSBPHltNh7JtLsxmZUhjYBZo2PpcU/RJvvi6Q0l7O7ogw==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-svg-em-dimensions@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-gWGsiwjb4tw+ITOJ86ndY/DZZ6cuXMNE/SjcDRg+HLuCmwpcjOktwRF9WgAiycTqJD/QXqL2f8IzE2Rzh7aVXA==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-transform-react-native-svg@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-2jT3nTayyYP7kI6aGutkyfJ7UMGtuguD72OjeGLwVNyfPRBD8zQthlvL+fAbAKk5n9ZNcvFkp/b1lZ7VsYqVJg==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-transform-svg-component@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-a1p6LF5Jt33O3rZoVRBqdxL350oge54iZWHNI6LJB5tQ7EelvD/Mb1mfBiZNAan0dt4i3VArkFRjA4iObuNykQ==} + engines: {node: '>=12'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-preset@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-6127fvO/FF2oi5EzSQOAjo1LE3OtNVh11R+/8FXa+mHx1ptAaS4cknIjnUA7e6j6fwGGJ17NzaTJFUwOV2zwCw==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-plugin-add-jsx-attribute': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-remove-jsx-attribute': 8.0.0(@babel/core@7.23.6) + '@svgr/babel-plugin-remove-jsx-empty-expression': 8.0.0(@babel/core@7.23.6) + '@svgr/babel-plugin-replace-jsx-attribute-value': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-svg-dynamic-title': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-svg-em-dimensions': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-transform-react-native-svg': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-transform-svg-component': 6.5.1(@babel/core@7.23.6) + dev: false + + /@svgr/core@6.5.1: + resolution: {integrity: sha512-/xdLSWxK5QkqG524ONSjvg3V/FkNyCv538OIBdQqPNaAta3AsXj/Bd2FbvR87yMbXO2hFSWiAe/Q6IkVPDw+mw==} + engines: {node: '>=10'} + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-preset': 6.5.1(@babel/core@7.23.6) + '@svgr/plugin-jsx': 6.5.1(@svgr/core@6.5.1) + camelcase: 6.3.0 + cosmiconfig: 7.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /@svgr/hast-util-to-babel-ast@6.5.1: + resolution: {integrity: sha512-1hnUxxjd83EAxbL4a0JDJoD3Dao3hmjvyvyEV8PzWmLK3B9m9NPlW7GKjFyoWE8nM7HnXzPcmmSyOW8yOddSXw==} + engines: {node: '>=10'} + dependencies: + '@babel/types': 7.23.6 + entities: 4.5.0 + dev: false + + /@svgr/plugin-jsx@6.5.1(@svgr/core@6.5.1): + resolution: {integrity: sha512-+UdQxI3jgtSjCykNSlEMuy1jSRQlGC7pqBCPvkG/2dATdWo082zHTTK3uhnAju2/6XpE6B5mZ3z4Z8Ns01S8Gw==} + engines: {node: '>=10'} + peerDependencies: + '@svgr/core': ^6.0.0 + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-preset': 6.5.1(@babel/core@7.23.6) + '@svgr/core': 6.5.1 + '@svgr/hast-util-to-babel-ast': 6.5.1 + svg-parser: 2.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /@svgr/plugin-svgo@6.5.1(@svgr/core@6.5.1): + resolution: {integrity: sha512-omvZKf8ixP9z6GWgwbtmP9qQMPX4ODXi+wzbVZgomNFsUIlHA1sf4fThdwTWSsZGgvGAG6yE+b/F5gWUkcZ/iQ==} + engines: {node: '>=10'} + peerDependencies: + '@svgr/core': '*' + dependencies: + '@svgr/core': 6.5.1 + cosmiconfig: 7.1.0 + deepmerge: 4.3.1 + svgo: 2.8.0 + dev: false + + /@svgr/webpack@6.5.1: + resolution: {integrity: sha512-cQ/AsnBkXPkEK8cLbv4Dm7JGXq2XrumKnL1dRpJD9rIO2fTIlJI9a1uCciYG1F2aUsox/hJQyNGbt3soDxSRkA==} + engines: {node: '>=10'} + dependencies: + '@babel/core': 7.23.6 + '@babel/plugin-transform-react-constant-elements': 7.23.3(@babel/core@7.23.6) + '@babel/preset-env': 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': 7.23.3(@babel/core@7.23.6) + '@svgr/core': 6.5.1 + '@svgr/plugin-jsx': 6.5.1(@svgr/core@6.5.1) + '@svgr/plugin-svgo': 6.5.1(@svgr/core@6.5.1) + transitivePeerDependencies: + - supports-color + dev: false + + /@szmarczak/http-timer@5.0.1: + resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} + engines: {node: '>=14.16'} + dependencies: + defer-to-connect: 2.0.1 + dev: false + + /@tanstack/react-virtual@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-IFOFuRUTaiM/yibty9qQ9BfycQnYXIDHGP2+cU+0LrFFGNhVxCXSQnaY6wkX8uJVteFEBjUondX0Hmpp7TNcag==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@tanstack/virtual-core': 3.0.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: false + + /@tanstack/virtual-core@3.0.0: + resolution: {integrity: sha512-SYXOBTjJb05rXa2vl55TTwO40A6wKu0R5i1qQwhJYNDIqaIGF7D0HsLw+pJAyi2OvntlEIVusx3xtbbgSUi6zg==} + dev: false + + /@trysound/sax@0.2.0: + resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} + engines: {node: '>=10.13.0'} + dev: false + + /@types/acorn@4.0.6: + resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /@types/body-parser@1.19.5: + resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + dependencies: + '@types/connect': 3.4.38 + '@types/node': 20.10.6 + dev: false + + /@types/bonjour@3.5.13: + resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/connect-history-api-fallback@1.5.4: + resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} + dependencies: + '@types/express-serve-static-core': 4.17.41 + '@types/node': 20.10.6 + dev: false + + /@types/connect@3.4.38: + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/d3-scale-chromatic@3.0.3: + resolution: {integrity: sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==} + dev: false + + /@types/d3-scale@4.0.8: + resolution: {integrity: sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==} + dependencies: + '@types/d3-time': 3.0.3 + dev: false + + /@types/d3-time@3.0.3: + resolution: {integrity: sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==} + dev: false + + /@types/debug@4.1.12: + resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + dependencies: + '@types/ms': 0.7.34 + dev: false + + /@types/eslint-scope@3.7.7: + resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} + dependencies: + '@types/eslint': 8.56.1 + '@types/estree': 1.0.5 + dev: false + + /@types/eslint@8.56.1: + resolution: {integrity: sha512-18PLWRzhy9glDQp3+wOgfLYRWlhgX0azxgJ63rdpoUHyrC9z0f5CkFburjQx4uD7ZCruw85ZtMt6K+L+R8fLJQ==} + dependencies: + '@types/estree': 1.0.5 + '@types/json-schema': 7.0.15 + dev: false + + /@types/estree-jsx@1.0.3: + resolution: {integrity: sha512-pvQ+TKeRHeiUGRhvYwRrQ/ISnohKkSJR14fT2yqyZ4e9K5vqc7hrtY2Y1Dw0ZwAzQ6DQsxsaCUuSIIi8v0Cq6w==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /@types/estree@1.0.5: + resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} + dev: false + + /@types/express-serve-static-core@4.17.41: + resolution: {integrity: sha512-OaJ7XLaelTgrvlZD8/aa0vvvxZdUmlCn6MtWeB7TkiKW70BQLc9XEPpDLPdbo52ZhXUCrznlWdCHWxJWtdyajA==} + dependencies: + '@types/node': 20.10.6 + '@types/qs': 6.9.11 + '@types/range-parser': 1.2.7 + '@types/send': 0.17.4 + dev: false + + /@types/express@4.17.21: + resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + dependencies: + '@types/body-parser': 1.19.5 + '@types/express-serve-static-core': 4.17.41 + '@types/qs': 6.9.11 + '@types/serve-static': 1.15.5 + dev: false + + /@types/gtag.js@0.0.12: + resolution: {integrity: sha512-YQV9bUsemkzG81Ea295/nF/5GijnD2Af7QhEofh7xu+kvCN6RdodgNwwGWXB5GMI3NoyvQo0odNctoH/qLMIpg==} + dev: false + + /@types/hast@2.3.9: + resolution: {integrity: sha512-pTHyNlaMD/oKJmS+ZZUyFUcsZeBZpC0lmGquw98CqRVNgAdJZJeD7GoeLiT6Xbx5rU9VCjSt0RwEvDgzh4obFw==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /@types/hast@3.0.3: + resolution: {integrity: sha512-2fYGlaDy/qyLlhidX42wAH0KBi2TCjKMH8CHmBXgRlJ3Y+OXTiqsPQ6IWarZKwF1JoUcAJdPogv1d4b0COTpmQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /@types/history@4.7.11: + resolution: {integrity: sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA==} + dev: false + + /@types/html-minifier-terser@6.1.0: + resolution: {integrity: sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==} + dev: false + + /@types/http-cache-semantics@4.0.4: + resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} + dev: false + + /@types/http-errors@2.0.4: + resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} + dev: false + + /@types/http-proxy@1.17.14: + resolution: {integrity: sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/istanbul-lib-coverage@2.0.6: + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + dev: false + + /@types/istanbul-lib-report@3.0.3: + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + dev: false + + /@types/istanbul-reports@3.0.4: + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + dependencies: + '@types/istanbul-lib-report': 3.0.3 + dev: false + + /@types/json-schema@7.0.15: + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + dev: false + + /@types/mdast@3.0.15: + resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /@types/mdast@4.0.3: + resolution: {integrity: sha512-LsjtqsyF+d2/yFOYaN22dHZI1Cpwkrj+g06G8+qtUKlhovPW89YhqSnfKtMbkgmEtYpH2gydRNULd6y8mciAFg==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /@types/mdx@2.0.10: + resolution: {integrity: sha512-Rllzc5KHk0Al5/WANwgSPl1/CwjqCy+AZrGd78zuK+jO9aDM6ffblZ+zIjgPNAaEBmlO0RYDvLNh7wD0zKVgEg==} + dev: false + + /@types/mime@1.3.5: + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + dev: false + + /@types/mime@3.0.4: + resolution: {integrity: sha512-iJt33IQnVRkqeqC7PzBHPTC6fDlRNRW8vjrgqtScAhrmMwe8c4Eo7+fUGTa+XdWrpEgpyKWMYmi2dIwMAYRzPw==} + dev: false + + /@types/ms@0.7.34: + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + dev: false + + /@types/node-forge@1.3.11: + resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/node@17.0.45: + resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} + dev: false + + /@types/node@20.10.6: + resolution: {integrity: sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==} + dependencies: + undici-types: 5.26.5 + dev: false + + /@types/parse-json@4.0.2: + resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} + dev: false + + /@types/prismjs@1.26.3: + resolution: {integrity: sha512-A0D0aTXvjlqJ5ZILMz3rNfDBOx9hHxLZYv2by47Sm/pqW35zzjusrZTryatjN/Rf8Us2gZrJD+KeHbUSTux1Cw==} + dev: false + + /@types/prop-types@15.7.11: + resolution: {integrity: sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==} + dev: false + + /@types/qs@6.9.11: + resolution: {integrity: sha512-oGk0gmhnEJK4Yyk+oI7EfXsLayXatCWPHary1MtcmbAifkobT9cM9yutG/hZKIseOU0MqbIwQ/u2nn/Gb+ltuQ==} + dev: false + + /@types/range-parser@1.2.7: + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + dev: false + + /@types/react-router-config@5.0.11: + resolution: {integrity: sha512-WmSAg7WgqW7m4x8Mt4N6ZyKz0BubSj/2tVUMsAHp+Yd2AMwcSbeFq9WympT19p5heCFmF97R9eD5uUR/t4HEqw==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router': 5.1.20 + dev: false + + /@types/react-router-dom@5.3.3: + resolution: {integrity: sha512-kpqnYK4wcdm5UaWI3fLcELopqLrHgLqNsdpHauzlQktfkHL3npOSwtj1Uz9oKBAzs7lFtVkV8j83voAz2D8fhw==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router': 5.1.20 + dev: false + + /@types/react-router@5.1.20: + resolution: {integrity: sha512-jGjmu/ZqS7FjSH6owMcD5qpq19+1RS9DeVRqfl1FeBMxTDQAGwlMWOcs52NDoXaNKyG3d1cYQFMs9rCrb88o9Q==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + dev: false + + /@types/react@18.2.46: + resolution: {integrity: sha512-nNCvVBcZlvX4NU1nRRNV/mFl1nNRuTuslAJglQsq+8ldXe5Xv0Wd2f7WTE3jOxhLH2BFfiZGC6GCp+kHQbgG+w==} + dependencies: + '@types/prop-types': 15.7.11 + '@types/scheduler': 0.16.8 + csstype: 3.1.3 + dev: false + + /@types/retry@0.12.0: + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + dev: false + + /@types/sax@1.2.7: + resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} + dependencies: + '@types/node': 17.0.45 + dev: false + + /@types/scheduler@0.16.8: + resolution: {integrity: sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==} + dev: false + + /@types/send@0.17.4: + resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} + dependencies: + '@types/mime': 1.3.5 + '@types/node': 20.10.6 + dev: false + + /@types/serve-index@1.9.4: + resolution: {integrity: sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==} + dependencies: + '@types/express': 4.17.21 + dev: false + + /@types/serve-static@1.15.5: + resolution: {integrity: sha512-PDRk21MnK70hja/YF8AHfC7yIsiQHn1rcXx7ijCFBX/k+XQJhQT/gw3xekXKJvx+5SXaMMS8oqQy09Mzvz2TuQ==} + dependencies: + '@types/http-errors': 2.0.4 + '@types/mime': 3.0.4 + '@types/node': 20.10.6 + dev: false + + /@types/sockjs@0.3.36: + resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/unist@2.0.10: + resolution: {integrity: sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==} + dev: false + + /@types/unist@3.0.2: + resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==} + dev: false + + /@types/ws@8.5.10: + resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/yargs-parser@21.0.3: + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + dev: false + + /@types/yargs@17.0.32: + resolution: {integrity: sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==} + dependencies: + '@types/yargs-parser': 21.0.3 + dev: false + + /@ungap/structured-clone@1.2.0: + resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + dev: false + + /@webassemblyjs/ast@1.11.6: + resolution: {integrity: sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==} + dependencies: + '@webassemblyjs/helper-numbers': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + dev: false + + /@webassemblyjs/floating-point-hex-parser@1.11.6: + resolution: {integrity: sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==} + dev: false + + /@webassemblyjs/helper-api-error@1.11.6: + resolution: {integrity: sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==} + dev: false + + /@webassemblyjs/helper-buffer@1.11.6: + resolution: {integrity: sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA==} + dev: false + + /@webassemblyjs/helper-numbers@1.11.6: + resolution: {integrity: sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==} + dependencies: + '@webassemblyjs/floating-point-hex-parser': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/helper-wasm-bytecode@1.11.6: + resolution: {integrity: sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==} + dev: false + + /@webassemblyjs/helper-wasm-section@1.11.6: + resolution: {integrity: sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + dev: false + + /@webassemblyjs/ieee754@1.11.6: + resolution: {integrity: sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==} + dependencies: + '@xtuc/ieee754': 1.2.0 + dev: false + + /@webassemblyjs/leb128@1.11.6: + resolution: {integrity: sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==} + dependencies: + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/utf8@1.11.6: + resolution: {integrity: sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==} + dev: false + + /@webassemblyjs/wasm-edit@1.11.6: + resolution: {integrity: sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/helper-wasm-section': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + '@webassemblyjs/wasm-opt': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + '@webassemblyjs/wast-printer': 1.11.6 + dev: false + + /@webassemblyjs/wasm-gen@1.11.6: + resolution: {integrity: sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/ieee754': 1.11.6 + '@webassemblyjs/leb128': 1.11.6 + '@webassemblyjs/utf8': 1.11.6 + dev: false + + /@webassemblyjs/wasm-opt@1.11.6: + resolution: {integrity: sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + dev: false + + /@webassemblyjs/wasm-parser@1.11.6: + resolution: {integrity: sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/ieee754': 1.11.6 + '@webassemblyjs/leb128': 1.11.6 + '@webassemblyjs/utf8': 1.11.6 + dev: false + + /@webassemblyjs/wast-printer@1.11.6: + resolution: {integrity: sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@xtuc/long': 4.2.2 + dev: false + + /@xtuc/ieee754@1.2.0: + resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} + dev: false + + /@xtuc/long@4.2.2: + resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + dev: false + + /accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + dev: false + + /acorn-import-assertions@1.9.0(acorn@8.11.3): + resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==} + peerDependencies: + acorn: ^8 + dependencies: + acorn: 8.11.3 + dev: false + + /acorn-jsx@5.3.2(acorn@8.11.3): + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + dependencies: + acorn: 8.11.3 + dev: false + + /acorn-walk@8.3.1: + resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} + engines: {node: '>=0.4.0'} + dev: false + + /acorn@8.11.3: + resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: false + + /address@1.2.2: + resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==} + engines: {node: '>= 10.0.0'} + dev: false + + /aggregate-error@3.1.0: + resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} + engines: {node: '>=8'} + dependencies: + clean-stack: 2.2.0 + indent-string: 4.0.0 + dev: false + + /ajv-formats@2.1.1(ajv@8.12.0): + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + dependencies: + ajv: 8.12.0 + dev: false + + /ajv-keywords@3.5.2(ajv@6.12.6): + resolution: {integrity: sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==} + peerDependencies: + ajv: ^6.9.1 + dependencies: + ajv: 6.12.6 + dev: false + + /ajv-keywords@5.1.0(ajv@8.12.0): + resolution: {integrity: sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==} + peerDependencies: + ajv: ^8.8.2 + dependencies: + ajv: 8.12.0 + fast-deep-equal: 3.1.3 + dev: false + + /ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + dev: false + + /ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + dev: false + + /algoliasearch-helper@3.16.1(algoliasearch@4.22.0): + resolution: {integrity: sha512-qxAHVjjmT7USVvrM8q6gZGaJlCK1fl4APfdAA7o8O6iXEc68G0xMNrzRkxoB/HmhhvyHnoteS/iMTiHiTcQQcg==} + peerDependencies: + algoliasearch: '>= 3.1 < 6' + dependencies: + '@algolia/events': 4.0.1 + algoliasearch: 4.22.0 + dev: false + + /algoliasearch@4.22.0: + resolution: {integrity: sha512-gfceltjkwh7PxXwtkS8KVvdfK+TSNQAWUeNSxf4dA29qW5tf2EGwa8jkJujlT9jLm17cixMVoGNc+GJFO1Mxhg==} + dependencies: + '@algolia/cache-browser-local-storage': 4.22.0 + '@algolia/cache-common': 4.22.0 + '@algolia/cache-in-memory': 4.22.0 + '@algolia/client-account': 4.22.0 + '@algolia/client-analytics': 4.22.0 + '@algolia/client-common': 4.22.0 + '@algolia/client-personalization': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/logger-common': 4.22.0 + '@algolia/logger-console': 4.22.0 + '@algolia/requester-browser-xhr': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/requester-node-http': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /ansi-align@3.0.1: + resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + dependencies: + string-width: 4.2.3 + dev: false + + /ansi-html-community@0.0.8: + resolution: {integrity: sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==} + engines: {'0': node >= 0.8.0} + hasBin: true + dev: false + + /ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + dev: false + + /ansi-regex@6.0.1: + resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} + engines: {node: '>=12'} + dev: false + + /ansi-styles@3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + dependencies: + color-convert: 1.9.3 + dev: false + + /ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + dependencies: + color-convert: 2.0.1 + dev: false + + /ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + dev: false + + /anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.1 + dev: false + + /arg@5.0.2: + resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + dev: false + + /argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + dependencies: + sprintf-js: 1.0.3 + dev: false + + /argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + + /array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + dev: false + + /array-union@2.1.0: + resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} + engines: {node: '>=8'} + dev: false + + /astring@1.8.6: + resolution: {integrity: sha512-ISvCdHdlTDlH5IpxQJIex7BWBywFWgjJSVdwst+/iQCoEYnyOaQ95+X1JGshuBjGp6nxKUy1jMgE3zPqN7fQdg==} + hasBin: true + dev: false + + /async@2.6.4: + resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==} + dependencies: + lodash: 4.17.21 + dev: false + + /at-least-node@1.0.0: + resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} + engines: {node: '>= 4.0.0'} + dev: false + + /autoprefixer@10.4.16(postcss@8.4.32): + resolution: {integrity: sha512-7vd3UC6xKp0HLfua5IjZlcXvGAGy7cBAXTg2lyQ/8WpNhd6SiZ8Be+xm3FyBSYJx5GKcpRCzBh7RH4/0dnY+uQ==} + engines: {node: ^10 || ^12 || >=14} + hasBin: true + peerDependencies: + postcss: ^8.1.0 + dependencies: + browserslist: 4.22.2 + caniuse-lite: 1.0.30001572 + fraction.js: 4.3.7 + normalize-range: 0.1.2 + picocolors: 1.0.0 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /babel-loader@9.1.3(@babel/core@7.23.6)(webpack@5.89.0): + resolution: {integrity: sha512-xG3ST4DglodGf8qSwv0MdeWLhrDsw/32QMdTO5T1ZIp9gQur0HkCyFs7Awskr10JKXFXwpAhiCuYX5oGXnRGbw==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@babel/core': ^7.12.0 + webpack: '>=5' + dependencies: + '@babel/core': 7.23.6 + find-cache-dir: 4.0.0 + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /babel-plugin-dynamic-import-node@2.3.3: + resolution: {integrity: sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==} + dependencies: + object.assign: 4.1.5 + dev: false + + /babel-plugin-polyfill-corejs2@0.4.7(@babel/core@7.23.6): + resolution: {integrity: sha512-LidDk/tEGDfuHW2DWh/Hgo4rmnw3cduK6ZkOI1NPFceSK3n/yAGeOsNT7FLnSGHkXj3RHGSEVkN3FsCTY6w2CQ==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /babel-plugin-polyfill-corejs3@0.8.7(@babel/core@7.23.6): + resolution: {integrity: sha512-KyDvZYxAzkC0Aj2dAPyDzi2Ym15e5JKZSK+maI7NAwSqofvuFglbSsxE7wUOvTg9oFVnHMzVzBKcqEb4PJgtOA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + core-js-compat: 3.35.0 + transitivePeerDependencies: + - supports-color + dev: false + + /babel-plugin-polyfill-regenerator@0.5.4(@babel/core@7.23.6): + resolution: {integrity: sha512-S/x2iOCvDaCASLYsOOgWOq4bCfKYVqvO/uxjkaYyZ3rVsVE3CeAI/c84NpyuBBymEgNvHgjEot3a9/Z/kXvqsg==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + transitivePeerDependencies: + - supports-color + dev: false + + /bail@2.0.2: + resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} + dev: false + + /balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + dev: false + + /batch@0.6.1: + resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} + dev: false + + /big.js@5.2.2: + resolution: {integrity: sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==} + dev: false + + /binary-extensions@2.2.0: + resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} + engines: {node: '>=8'} + dev: false + + /body-parser@1.20.1: + resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.11.0 + raw-body: 2.5.1 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /bonjour-service@1.2.0: + resolution: {integrity: sha512-xdzMA6JGckxyJzZByjEWRcfKmDxXaGXZWVftah3FkCqdlePNS9DjHSUN5zkP4oEfz/t0EXXlro88EIhzwMB4zA==} + dependencies: + fast-deep-equal: 3.1.3 + multicast-dns: 7.2.5 + dev: false + + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + + /boxen@6.2.1: + resolution: {integrity: sha512-H4PEsJXfFI/Pt8sjDWbHlQPx4zL/bvSQjcilJmaulGt5mLDorHOHpmdXAJcBcmru7PhYSp/cDMWRko4ZUMFkSw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + ansi-align: 3.0.1 + camelcase: 6.3.0 + chalk: 4.1.2 + cli-boxes: 3.0.0 + string-width: 5.1.2 + type-fest: 2.19.0 + widest-line: 4.0.1 + wrap-ansi: 8.1.0 + dev: false + + /boxen@7.1.1: + resolution: {integrity: sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==} + engines: {node: '>=14.16'} + dependencies: + ansi-align: 3.0.1 + camelcase: 7.0.1 + chalk: 5.3.0 + cli-boxes: 3.0.0 + string-width: 5.1.2 + type-fest: 2.19.0 + widest-line: 4.0.1 + wrap-ansi: 8.1.0 + dev: false + + /brace-expansion@1.1.11: + resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + dev: false + + /braces@3.0.2: + resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} + engines: {node: '>=8'} + dependencies: + fill-range: 7.0.1 + dev: false + + /browserslist@4.22.2: + resolution: {integrity: sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + dependencies: + caniuse-lite: 1.0.30001572 + electron-to-chromium: 1.4.618 + node-releases: 2.0.14 + update-browserslist-db: 1.0.13(browserslist@4.22.2) + dev: false + + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + + /bytes@3.0.0: + resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} + engines: {node: '>= 0.8'} + dev: false + + /bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + dev: false + + /cacheable-lookup@7.0.0: + resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} + engines: {node: '>=14.16'} + dev: false + + /cacheable-request@10.2.14: + resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} + engines: {node: '>=14.16'} + dependencies: + '@types/http-cache-semantics': 4.0.4 + get-stream: 6.0.1 + http-cache-semantics: 4.1.1 + keyv: 4.5.4 + mimic-response: 4.0.0 + normalize-url: 8.0.0 + responselike: 3.0.0 + dev: false + + /call-bind@1.0.5: + resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==} + dependencies: + function-bind: 1.1.2 + get-intrinsic: 1.2.2 + set-function-length: 1.1.1 + dev: false + + /call-me-maybe@1.0.2: + resolution: {integrity: sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==} + dev: false + + /callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + dev: false + + /camel-case@4.1.2: + resolution: {integrity: sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==} + dependencies: + pascal-case: 3.1.2 + tslib: 2.6.2 + dev: false + + /camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + dev: false + + /camelcase@7.0.1: + resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==} + engines: {node: '>=14.16'} + dev: false + + /caniuse-api@3.0.0: + resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} + dependencies: + browserslist: 4.22.2 + caniuse-lite: 1.0.30001572 + lodash.memoize: 4.1.2 + lodash.uniq: 4.5.0 + dev: false + + /caniuse-lite@1.0.30001572: + resolution: {integrity: sha512-1Pbh5FLmn5y4+QhNyJE9j3/7dK44dGB83/ZMjv/qJk86TvDbjk0LosiZo0i0WB0Vx607qMX9jYrn1VLHCkN4rw==} + dev: false + + /ccount@2.0.1: + resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + dev: false + + /chalk@2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + dev: false + + /chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + dev: false + + /chalk@5.3.0: + resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + dev: false + + /char-regex@1.0.2: + resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} + engines: {node: '>=10'} + dev: false + + /character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + dev: false + + /character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + dev: false + + /character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + dev: false + + /character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + dev: false + + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + + /chokidar@3.5.3: + resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} + engines: {node: '>= 8.10.0'} + dependencies: + anymatch: 3.1.3 + braces: 3.0.2 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.6.0 + optionalDependencies: + fsevents: 2.3.3 + dev: false + + /chrome-trace-event@1.0.3: + resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} + engines: {node: '>=6.0'} + dev: false + + /ci-info@3.9.0: + resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} + engines: {node: '>=8'} + dev: false + + /classnames@2.3.2: + resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} + dev: false + + /clean-css@5.3.3: + resolution: {integrity: sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==} + engines: {node: '>= 10.0'} + dependencies: + source-map: 0.6.1 + dev: false + + /clean-stack@2.2.0: + resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} + engines: {node: '>=6'} + dev: false + + /cli-boxes@3.0.0: + resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} + engines: {node: '>=10'} + dev: false + + /cli-table3@0.6.3: + resolution: {integrity: sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==} + engines: {node: 10.* || >= 12.*} + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + dev: false + + /client-only@0.0.1: + resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} + dev: false + + /clone-deep@4.0.1: + resolution: {integrity: sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==} + engines: {node: '>=6'} + dependencies: + is-plain-object: 2.0.4 + kind-of: 6.0.3 + shallow-clone: 3.0.1 + dev: false + + /clsx@1.1.1: + resolution: {integrity: sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==} + engines: {node: '>=6'} + dev: false + + /clsx@2.1.0: + resolution: {integrity: sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==} + engines: {node: '>=6'} + dev: false + + /collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + dev: false + + /color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + dependencies: + color-name: 1.1.3 + dev: false + + /color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + dependencies: + color-name: 1.1.4 + dev: false + + /color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + dev: false + + /color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: false + + /colord@2.9.3: + resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} + dev: false + + /colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + dev: false + + /combine-promises@1.2.0: + resolution: {integrity: sha512-VcQB1ziGD0NXrhKxiwyNbCDmRzs/OShMs2GqW2DlU2A/Sd0nQxE1oWDAE5O0ygSx5mgQOn9eIFh7yKPgFRVkPQ==} + engines: {node: '>=10'} + dev: false + + /comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + dev: false + + /commander@10.0.1: + resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} + engines: {node: '>=14'} + dev: false + + /commander@2.20.3: + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + dev: false + + /commander@5.1.0: + resolution: {integrity: sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==} + engines: {node: '>= 6'} + dev: false + + /commander@7.2.0: + resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} + engines: {node: '>= 10'} + dev: false + + /commander@8.3.0: + resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} + engines: {node: '>= 12'} + dev: false + + /common-path-prefix@3.0.0: + resolution: {integrity: sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==} + dev: false + + /compressible@2.0.18: + resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false + + /compression@1.7.4: + resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==} + engines: {node: '>= 0.8.0'} + dependencies: + accepts: 1.3.8 + bytes: 3.0.0 + compressible: 2.0.18 + debug: 2.6.9 + on-headers: 1.0.2 + safe-buffer: 5.1.2 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + dev: false + + /config-chain@1.1.13: + resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} + dependencies: + ini: 1.3.8 + proto-list: 1.2.4 + dev: false + + /configstore@6.0.0: + resolution: {integrity: sha512-cD31W1v3GqUlQvbBCGcXmd2Nj9SvLDOP1oQ0YFuLETufzSPaKp11rYBsSOm7rCsW3OnIRAFM3OxRhceaXNYHkA==} + engines: {node: '>=12'} + dependencies: + dot-prop: 6.0.1 + graceful-fs: 4.2.11 + unique-string: 3.0.0 + write-file-atomic: 3.0.3 + xdg-basedir: 5.1.0 + dev: false + + /connect-history-api-fallback@1.6.0: + resolution: {integrity: sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==} + engines: {node: '>=0.8'} + dev: false + + /connect-history-api-fallback@2.0.0: + resolution: {integrity: sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==} + engines: {node: '>=0.8'} + dev: false + + /consola@2.15.3: + resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} + dev: false + + /content-disposition@0.5.2: + resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} + engines: {node: '>= 0.6'} + dev: false + + /content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + dev: false + + /convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + dev: false + + /cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + dev: false + + /cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + dev: false + + /copy-text-to-clipboard@3.2.0: + resolution: {integrity: sha512-RnJFp1XR/LOBDckxTib5Qjr/PMfkatD0MUCQgdpqS8MdKiNUzBjAQBEN6oUy+jW7LI93BBG3DtMB2KOOKpGs2Q==} + engines: {node: '>=12'} + dev: false + + /copy-webpack-plugin@11.0.0(webpack@5.89.0): + resolution: {integrity: sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==} + engines: {node: '>= 14.15.0'} + peerDependencies: + webpack: ^5.1.0 + dependencies: + fast-glob: 3.3.2 + glob-parent: 6.0.2 + globby: 13.2.2 + normalize-path: 3.0.0 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + webpack: 5.89.0 + dev: false + + /core-js-compat@3.35.0: + resolution: {integrity: sha512-5blwFAddknKeNgsjBzilkdQ0+YK8L1PfqPYq40NOYMYFSS38qj+hpTcLLWwpIwA2A5bje/x5jmVn2tzUMg9IVw==} + dependencies: + browserslist: 4.22.2 + dev: false + + /core-js-pure@3.35.0: + resolution: {integrity: sha512-f+eRYmkou59uh7BPcyJ8MC76DiGhspj1KMxVIcF24tzP8NA9HVa1uC7BTW2tgx7E1QVCzDzsgp7kArrzhlz8Ew==} + requiresBuild: true + dev: false + + /core-js@3.35.0: + resolution: {integrity: sha512-ntakECeqg81KqMueeGJ79Q5ZgQNR+6eaE8sxGCx62zMbAIj65q+uYvatToew3m6eAGdU4gNZwpZ34NMe4GYswg==} + requiresBuild: true + dev: false + + /core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + dev: false + + /cose-base@1.0.3: + resolution: {integrity: sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==} + dependencies: + layout-base: 1.0.2 + dev: false + + /cose-base@2.2.0: + resolution: {integrity: sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==} + dependencies: + layout-base: 2.0.1 + dev: false + + /cosmiconfig@6.0.0: + resolution: {integrity: sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==} + engines: {node: '>=8'} + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.0 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 1.10.2 + dev: false + + /cosmiconfig@7.1.0: + resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==} + engines: {node: '>=10'} + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.0 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 1.10.2 + dev: false + + /cosmiconfig@8.3.6(typescript@5.3.3): + resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=4.9.5' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + import-fresh: 3.3.0 + js-yaml: 4.1.0 + parse-json: 5.2.0 + path-type: 4.0.0 + typescript: 5.3.3 + dev: false + + /cross-spawn@7.0.3: + resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} + engines: {node: '>= 8'} + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + dev: false + + /crypto-random-string@4.0.0: + resolution: {integrity: sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==} + engines: {node: '>=12'} + dependencies: + type-fest: 1.4.0 + dev: false + + /css-declaration-sorter@6.3.0(postcss@8.4.32): + resolution: {integrity: sha512-OGT677UGHJTAVMRhPO+HJ4oKln3wkBTwtDFH0ojbqm+MJm6xuDMHp2nkhh/ThaBqq20IbraBQSWKfSLNHQO9Og==} + engines: {node: ^10 || ^12 || >=14} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-declaration-sorter@6.4.1(postcss@8.4.32): + resolution: {integrity: sha512-rtdthzxKuyq6IzqX6jEcIzQF/YqccluefyCYheovBOLhFT/drQA9zj/UbRAa9J7C0o6EG6u3E6g+vKkay7/k3g==} + engines: {node: ^10 || ^12 || >=14} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-declaration-sorter@7.1.1(postcss@8.4.32): + resolution: {integrity: sha512-dZ3bVTEEc1vxr3Bek9vGwfB5Z6ESPULhcRvO472mfjVnj8jRcTnKO8/JTczlvxM10Myb+wBM++1MtdO76eWcaQ==} + engines: {node: ^14 || ^16 || >=18} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-loader@6.8.1(webpack@5.89.0): + resolution: {integrity: sha512-xDAXtEVGlD0gJ07iclwWVkLoZOpEvAWaSyf6W18S2pOC//K8+qUDIx8IIT3D+HjnmkJPQeesOPv5aiUaJsCM2g==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^5.0.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-modules-extract-imports: 3.0.0(postcss@8.4.32) + postcss-modules-local-by-default: 4.0.3(postcss@8.4.32) + postcss-modules-scope: 3.1.0(postcss@8.4.32) + postcss-modules-values: 4.0.0(postcss@8.4.32) + postcss-value-parser: 4.2.0 + semver: 7.5.4 + webpack: 5.89.0 + dev: false + + /css-minimizer-webpack-plugin@4.0.0(webpack@5.89.0): + resolution: {integrity: sha512-7ZXXRzRHvofv3Uac5Y+RkWRNo0ZMlcg8e9/OtrqUYmwDWJo+qs67GvdeFrXLsFb7czKNwjQhPkM0avlIYl+1nA==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@parcel/css': '*' + clean-css: '*' + csso: '*' + esbuild: '*' + webpack: ^5.0.0 + peerDependenciesMeta: + '@parcel/css': + optional: true + clean-css: + optional: true + csso: + optional: true + esbuild: + optional: true + dependencies: + cssnano: 5.1.15(postcss@8.4.32) + jest-worker: 27.5.1 + postcss: 8.4.32 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + source-map: 0.6.1 + webpack: 5.89.0 + dev: false + + /css-minimizer-webpack-plugin@4.2.2(clean-css@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-s3Of/4jKfw1Hj9CxEO1E5oXhQAxlayuHO2y/ML+C6I9sQ7FdzfEV6QgMLN3vI+qFsjJGIAFLKtQK7t8BOXAIyA==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@parcel/css': '*' + '@swc/css': '*' + clean-css: '*' + csso: '*' + esbuild: '*' + lightningcss: '*' + webpack: ^5.0.0 + peerDependenciesMeta: + '@parcel/css': + optional: true + '@swc/css': + optional: true + clean-css: + optional: true + csso: + optional: true + esbuild: + optional: true + lightningcss: + optional: true + dependencies: + clean-css: 5.3.3 + cssnano: 5.1.15(postcss@8.4.32) + jest-worker: 29.7.0 + postcss: 8.4.32 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + source-map: 0.6.1 + webpack: 5.89.0 + dev: false + + /css-select@4.3.0: + resolution: {integrity: sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 4.3.1 + domutils: 2.8.0 + nth-check: 2.0.1 + dev: false + + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.0.1 + dev: false + + /css-selector-parser@3.0.4: + resolution: {integrity: sha512-pnmS1dbKsz6KA4EW4BznyPL2xxkNDRg62hcD0v8g6DEw2W7hxOln5M953jsp9hmw5Dg57S6o/A8GOn37mbAgcQ==} + dev: false + + /css-tree@1.1.3: + resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==} + engines: {node: '>=8.0.0'} + dependencies: + mdn-data: 2.0.14 + source-map: 0.6.1 + dev: false + + /css-tree@2.2.1: + resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + dependencies: + mdn-data: 2.0.28 + source-map-js: 1.0.2 + dev: false + + /css-tree@2.3.1: + resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} + dependencies: + mdn-data: 2.0.30 + source-map-js: 1.0.2 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + + /cssesc@3.0.0: + resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /cssnano-preset-advanced@5.3.10(postcss@8.4.32): + resolution: {integrity: sha512-fnYJyCS9jgMU+cmHO1rPSPf9axbQyD7iUhLO5Df6O4G+fKIOMps+ZbU0PdGFejFBBZ3Pftf18fn1eG7MAPUSWQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + autoprefixer: 10.4.16(postcss@8.4.32) + cssnano-preset-default: 5.2.14(postcss@8.4.32) + postcss: 8.4.32 + postcss-discard-unused: 5.1.0(postcss@8.4.32) + postcss-merge-idents: 5.1.1(postcss@8.4.32) + postcss-reduce-idents: 5.2.0(postcss@8.4.32) + postcss-zindex: 5.1.0(postcss@8.4.32) + dev: false + + /cssnano-preset-advanced@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-1ziCYBklE4iQDuYy6RRumEhJDKv442d7ezzyDb1p3yYSmdz5GMan5y4xJc9YLgbiFJ9gufir9axrDUDjtT07pQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + autoprefixer: 10.4.16(postcss@8.4.32) + cssnano-preset-default: 6.0.3(postcss@8.4.32) + postcss: 8.4.32 + postcss-discard-unused: 6.0.2(postcss@8.4.32) + postcss-merge-idents: 6.0.1(postcss@8.4.32) + postcss-reduce-idents: 6.0.2(postcss@8.4.32) + postcss-zindex: 6.0.1(postcss@8.4.32) + dev: false + + /cssnano-preset-default@5.2.14(postcss@8.4.32): + resolution: {integrity: sha512-t0SFesj/ZV2OTylqQVOrFgEh5uanxbO6ZAdeCrNsUQ6fVuXwYTxJPNAGvGTxHbD68ldIJNec7PyYZDBrfDQ+6A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + css-declaration-sorter: 6.4.1(postcss@8.4.32) + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-calc: 8.2.4(postcss@8.4.32) + postcss-colormin: 5.3.1(postcss@8.4.32) + postcss-convert-values: 5.1.3(postcss@8.4.32) + postcss-discard-comments: 5.1.2(postcss@8.4.32) + postcss-discard-duplicates: 5.1.0(postcss@8.4.32) + postcss-discard-empty: 5.1.1(postcss@8.4.32) + postcss-discard-overridden: 5.1.0(postcss@8.4.32) + postcss-merge-longhand: 5.1.7(postcss@8.4.32) + postcss-merge-rules: 5.1.4(postcss@8.4.32) + postcss-minify-font-values: 5.1.0(postcss@8.4.32) + postcss-minify-gradients: 5.1.1(postcss@8.4.32) + postcss-minify-params: 5.1.4(postcss@8.4.32) + postcss-minify-selectors: 5.2.1(postcss@8.4.32) + postcss-normalize-charset: 5.1.0(postcss@8.4.32) + postcss-normalize-display-values: 5.1.0(postcss@8.4.32) + postcss-normalize-positions: 5.1.1(postcss@8.4.32) + postcss-normalize-repeat-style: 5.1.1(postcss@8.4.32) + postcss-normalize-string: 5.1.0(postcss@8.4.32) + postcss-normalize-timing-functions: 5.1.0(postcss@8.4.32) + postcss-normalize-unicode: 5.1.1(postcss@8.4.32) + postcss-normalize-url: 5.1.0(postcss@8.4.32) + postcss-normalize-whitespace: 5.1.1(postcss@8.4.32) + postcss-ordered-values: 5.1.3(postcss@8.4.32) + postcss-reduce-initial: 5.1.2(postcss@8.4.32) + postcss-reduce-transforms: 5.1.0(postcss@8.4.32) + postcss-svgo: 5.1.0(postcss@8.4.32) + postcss-unique-selectors: 5.1.1(postcss@8.4.32) + dev: false + + /cssnano-preset-default@6.0.3(postcss@8.4.32): + resolution: {integrity: sha512-4y3H370aZCkT9Ev8P4SO4bZbt+AExeKhh8wTbms/X7OLDo5E7AYUUy6YPxa/uF5Grf+AJwNcCnxKhZynJ6luBA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + css-declaration-sorter: 7.1.1(postcss@8.4.32) + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-calc: 9.0.1(postcss@8.4.32) + postcss-colormin: 6.0.2(postcss@8.4.32) + postcss-convert-values: 6.0.2(postcss@8.4.32) + postcss-discard-comments: 6.0.1(postcss@8.4.32) + postcss-discard-duplicates: 6.0.1(postcss@8.4.32) + postcss-discard-empty: 6.0.1(postcss@8.4.32) + postcss-discard-overridden: 6.0.1(postcss@8.4.32) + postcss-merge-longhand: 6.0.2(postcss@8.4.32) + postcss-merge-rules: 6.0.3(postcss@8.4.32) + postcss-minify-font-values: 6.0.1(postcss@8.4.32) + postcss-minify-gradients: 6.0.1(postcss@8.4.32) + postcss-minify-params: 6.0.2(postcss@8.4.32) + postcss-minify-selectors: 6.0.2(postcss@8.4.32) + postcss-normalize-charset: 6.0.1(postcss@8.4.32) + postcss-normalize-display-values: 6.0.1(postcss@8.4.32) + postcss-normalize-positions: 6.0.1(postcss@8.4.32) + postcss-normalize-repeat-style: 6.0.1(postcss@8.4.32) + postcss-normalize-string: 6.0.1(postcss@8.4.32) + postcss-normalize-timing-functions: 6.0.1(postcss@8.4.32) + postcss-normalize-unicode: 6.0.2(postcss@8.4.32) + postcss-normalize-url: 6.0.1(postcss@8.4.32) + postcss-normalize-whitespace: 6.0.1(postcss@8.4.32) + postcss-ordered-values: 6.0.1(postcss@8.4.32) + postcss-reduce-initial: 6.0.2(postcss@8.4.32) + postcss-reduce-transforms: 6.0.1(postcss@8.4.32) + postcss-svgo: 6.0.2(postcss@8.4.32) + postcss-unique-selectors: 6.0.2(postcss@8.4.32) + dev: false + + /cssnano-utils@3.1.0(postcss@8.4.32): + resolution: {integrity: sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /cssnano-utils@4.0.1(postcss@8.4.32): + resolution: {integrity: sha512-6qQuYDqsGoiXssZ3zct6dcMxiqfT6epy7x4R0TQJadd4LWO3sPR6JH6ZByOvVLoZ6EdwPGgd7+DR1EmX3tiXQQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /cssnano@5.1.15(postcss@8.4.32): + resolution: {integrity: sha512-j+BKgDcLDQA+eDifLx0EO4XSA56b7uut3BQFH+wbSaSTuGLuiyTa/wbRYthUXX8LC9mLg+WWKe8h+qJuwTAbHw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-preset-default: 5.2.14(postcss@8.4.32) + lilconfig: 2.1.0 + postcss: 8.4.32 + yaml: 1.10.2 + dev: false + + /cssnano@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-Tu9wv8UdN6CoiQnIVkCNvi+0rw/BwFWOJBlg2bVfEyKaadSuE3Gq/DD8tniVvggTJGwK88UjqZp7zL5sv6t1aA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-preset-default: 6.0.3(postcss@8.4.32) + lilconfig: 3.0.0 + postcss: 8.4.32 + dev: false + + /csso@4.2.0: + resolution: {integrity: sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==} + engines: {node: '>=8.0.0'} + dependencies: + css-tree: 1.1.3 + dev: false + + /csso@5.0.5: + resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + dependencies: + css-tree: 2.2.1 + dev: false + + /csstype@3.1.3: + resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + dev: false + + /cytoscape-cose-bilkent@4.1.0(cytoscape@3.28.1): + resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} + peerDependencies: + cytoscape: ^3.2.0 + dependencies: + cose-base: 1.0.3 + cytoscape: 3.28.1 + dev: false + + /cytoscape-fcose@2.2.0(cytoscape@3.28.1): + resolution: {integrity: sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==} + peerDependencies: + cytoscape: ^3.2.0 + dependencies: + cose-base: 2.2.0 + cytoscape: 3.28.1 + dev: false + + /cytoscape@3.28.1: + resolution: {integrity: sha512-xyItz4O/4zp9/239wCcH8ZcFuuZooEeF8KHRmzjDfGdXsj3OG9MFSMA0pJE0uX3uCN/ygof6hHf4L7lst+JaDg==} + engines: {node: '>=0.10'} + dependencies: + heap: 0.2.7 + lodash: 4.17.21 + dev: false + + /d3-array@2.12.1: + resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==} + dependencies: + internmap: 1.0.1 + dev: false + + /d3-array@3.2.4: + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} + engines: {node: '>=12'} + dependencies: + internmap: 2.0.3 + dev: false + + /d3-axis@3.0.0: + resolution: {integrity: sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==} + engines: {node: '>=12'} + dev: false + + /d3-brush@3.0.0: + resolution: {integrity: sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dev: false + + /d3-chord@3.0.1: + resolution: {integrity: sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==} + engines: {node: '>=12'} + dependencies: + d3-path: 3.1.0 + dev: false + + /d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + dev: false + + /d3-contour@4.0.2: + resolution: {integrity: sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-delaunay@6.0.4: + resolution: {integrity: sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==} + engines: {node: '>=12'} + dependencies: + delaunator: 5.0.0 + dev: false + + /d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + dev: false + + /d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + dev: false + + /d3-dsv@3.0.1: + resolution: {integrity: sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==} + engines: {node: '>=12'} + hasBin: true + dependencies: + commander: 7.2.0 + iconv-lite: 0.6.3 + rw: 1.3.3 + dev: false + + /d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + dev: false + + /d3-fetch@3.0.1: + resolution: {integrity: sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==} + engines: {node: '>=12'} + dependencies: + d3-dsv: 3.0.1 + dev: false + + /d3-force@3.0.0: + resolution: {integrity: sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-quadtree: 3.0.1 + d3-timer: 3.0.1 + dev: false + + /d3-format@3.1.0: + resolution: {integrity: sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==} + engines: {node: '>=12'} + dev: false + + /d3-geo@3.1.0: + resolution: {integrity: sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-hierarchy@3.1.2: + resolution: {integrity: sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==} + engines: {node: '>=12'} + dev: false + + /d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + dependencies: + d3-color: 3.1.0 + dev: false + + /d3-path@1.0.9: + resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} + dev: false + + /d3-path@3.1.0: + resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} + engines: {node: '>=12'} + dev: false + + /d3-polygon@3.0.1: + resolution: {integrity: sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==} + engines: {node: '>=12'} + dev: false + + /d3-quadtree@3.0.1: + resolution: {integrity: sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==} + engines: {node: '>=12'} + dev: false + + /d3-random@3.0.1: + resolution: {integrity: sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==} + engines: {node: '>=12'} + dev: false + + /d3-sankey@0.12.3: + resolution: {integrity: sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==} + dependencies: + d3-array: 2.12.1 + d3-shape: 1.3.7 + dev: false + + /d3-scale-chromatic@3.0.0: + resolution: {integrity: sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g==} + engines: {node: '>=12'} + dependencies: + d3-color: 3.1.0 + d3-interpolate: 3.0.1 + dev: false + + /d3-scale@4.0.2: + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + d3-format: 3.1.0 + d3-interpolate: 3.0.1 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + dev: false + + /d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + dev: false + + /d3-shape@1.3.7: + resolution: {integrity: sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==} + dependencies: + d3-path: 1.0.9 + dev: false + + /d3-shape@3.2.0: + resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} + engines: {node: '>=12'} + dependencies: + d3-path: 3.1.0 + dev: false + + /d3-time-format@4.1.0: + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} + engines: {node: '>=12'} + dependencies: + d3-time: 3.1.0 + dev: false + + /d3-time@3.1.0: + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + dev: false + + /d3-transition@3.0.1(d3-selection@3.0.0): + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + dev: false + + /d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dev: false + + /d3@7.8.5: + resolution: {integrity: sha512-JgoahDG51ncUfJu6wX/1vWQEqOflgXyl4MaHqlcSruTez7yhaRKR9i8VjjcQGeS2en/jnFivXuaIMnseMMt0XA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + d3-axis: 3.0.0 + d3-brush: 3.0.0 + d3-chord: 3.0.1 + d3-color: 3.1.0 + d3-contour: 4.0.2 + d3-delaunay: 6.0.4 + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-dsv: 3.0.1 + d3-ease: 3.0.1 + d3-fetch: 3.0.1 + d3-force: 3.0.0 + d3-format: 3.1.0 + d3-geo: 3.1.0 + d3-hierarchy: 3.1.2 + d3-interpolate: 3.0.1 + d3-path: 3.1.0 + d3-polygon: 3.0.1 + d3-quadtree: 3.0.1 + d3-random: 3.0.1 + d3-scale: 4.0.2 + d3-scale-chromatic: 3.0.0 + d3-selection: 3.0.0 + d3-shape: 3.2.0 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + d3-timer: 3.0.1 + d3-transition: 3.0.1(d3-selection@3.0.0) + d3-zoom: 3.0.0 + dev: false + + /dagre-d3-es@7.0.10: + resolution: {integrity: sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==} + dependencies: + d3: 7.8.5 + lodash-es: 4.17.21 + dev: false + + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + + /dayjs@1.11.10: + resolution: {integrity: sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==} + dev: false + + /debounce@1.2.1: + resolution: {integrity: sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==} + dev: false + + /debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.0.0 + dev: false + + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: false + + /decode-named-character-reference@1.0.2: + resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} + dependencies: + character-entities: 2.0.2 + dev: false + + /decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dependencies: + mimic-response: 3.1.0 + dev: false + + /deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + dev: false + + /deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + dev: false + + /default-gateway@6.0.3: + resolution: {integrity: sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==} + engines: {node: '>= 10'} + dependencies: + execa: 5.1.1 + dev: false + + /defer-to-connect@2.0.1: + resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} + engines: {node: '>=10'} + dev: false + + /define-data-property@1.1.1: + resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==} + engines: {node: '>= 0.4'} + dependencies: + get-intrinsic: 1.2.2 + gopd: 1.0.1 + has-property-descriptors: 1.0.1 + dev: false + + /define-lazy-prop@2.0.0: + resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} + engines: {node: '>=8'} + dev: false + + /define-properties@1.2.1: + resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.1 + has-property-descriptors: 1.0.1 + object-keys: 1.1.1 + dev: false + + /del@6.1.1: + resolution: {integrity: sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==} + engines: {node: '>=10'} + dependencies: + globby: 11.1.0 + graceful-fs: 4.2.11 + is-glob: 4.0.3 + is-path-cwd: 2.2.0 + is-path-inside: 3.0.3 + p-map: 4.0.0 + rimraf: 3.0.2 + slash: 3.0.0 + dev: false + + /delaunator@5.0.0: + resolution: {integrity: sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw==} + dependencies: + robust-predicates: 3.0.2 + dev: false + + /depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + dev: false + + /depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dev: false + + /dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + dev: false + + /destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dev: false + + /detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + dev: false + + /detect-port-alt@1.1.6: + resolution: {integrity: sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q==} + engines: {node: '>= 4.2.1'} + hasBin: true + dependencies: + address: 1.2.2 + debug: 2.6.9 + transitivePeerDependencies: + - supports-color + dev: false + + /detect-port@1.5.1: + resolution: {integrity: sha512-aBzdj76lueB6uUst5iAs7+0H/oOjqI5D16XUWxlWMIMROhcM0rfsNVk93zTngq1dDNpoXRr++Sus7ETAExppAQ==} + hasBin: true + dependencies: + address: 1.2.2 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + + /devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dependencies: + dequal: 2.0.3 + dev: false + + /diff@5.1.0: + resolution: {integrity: sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==} + engines: {node: '>=0.3.1'} + dev: false + + /dir-glob@3.0.1: + resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} + engines: {node: '>=8'} + dependencies: + path-type: 4.0.0 + dev: false + + /dns-packet@5.6.1: + resolution: {integrity: sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==} + engines: {node: '>=6'} + dependencies: + '@leichtgewicht/ip-codec': 2.0.4 + dev: false + + /docusaurus-plugin-hubspot@1.0.0: + resolution: {integrity: sha512-qeNRlI336M6mcANGrEDxMt7B+GWvWd+yXOpT6uquQLX0b2cDHncDj0+rbuUaJUNQnKQz1st7sKgJQ0P55736Ug==} + deprecated: docusaurus-plugin-hubspot is now available at @stackql/docusaurus-plugin-hubspot + dev: false + + /docusaurus-plugin-segment@1.0.3: + resolution: {integrity: sha512-9DqebTx9TqjujCnB22qEeCm8NGJUAH7VAKLAa20/CyfSSrs+khTQI0FmzEALtiCqKNO1D3GWm3VvE4gqbuGqnw==} + dependencies: + '@segment/snippet': 4.16.2 + dev: false + + /dom-converter@0.2.0: + resolution: {integrity: sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==} + dependencies: + utila: 0.4.0 + dev: false + + /dom-serializer@1.4.1: + resolution: {integrity: sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==} + dependencies: + domelementtype: 2.3.0 + domhandler: 4.3.1 + entities: 2.2.0 + dev: false + + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@4.3.1: + resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /dompurify@3.0.6: + resolution: {integrity: sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==} + dev: false + + /domutils@2.8.0: + resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} + dependencies: + dom-serializer: 1.4.1 + domelementtype: 2.3.0 + domhandler: 4.3.1 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + + /dot-case@3.0.4: + resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} + dependencies: + no-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /dot-prop@6.0.1: + resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} + engines: {node: '>=10'} + dependencies: + is-obj: 2.0.0 + dev: false + + /duplexer@0.1.2: + resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} + dev: false + + /eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + dev: false + + /ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + dev: false + + /electron-to-chromium@1.4.618: + resolution: {integrity: sha512-mTM2HieHLxs1RbD/R/ZoQLMsGI8lWIkP17G7cx32mJRBJt9wlNPkXwE3sYg/OnNb5GBkus98lXatSthoL8Y5Ag==} + dev: false + + /elkjs@0.8.2: + resolution: {integrity: sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ==} + dev: false + + /emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + dev: false + + /emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + dev: false + + /emojilib@2.4.0: + resolution: {integrity: sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw==} + dev: false + + /emojis-list@3.0.0: + resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==} + engines: {node: '>= 4'} + dev: false + + /emoticon@4.0.1: + resolution: {integrity: sha512-dqx7eA9YaqyvYtUhJwT4rC1HIp82j5ybS1/vQ42ur+jBe17dJMwZE4+gvL1XadSFfxaPFFGt3Xsw+Y8akThDlw==} + dev: false + + /encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + dev: false + + /enhanced-resolve@5.15.0: + resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==} + engines: {node: '>=10.13.0'} + dependencies: + graceful-fs: 4.2.11 + tapable: 2.2.1 + dev: false + + /entities@2.2.0: + resolution: {integrity: sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==} + dev: false + + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + + /error-ex@1.3.2: + resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + dependencies: + is-arrayish: 0.2.1 + dev: false + + /es-module-lexer@1.4.1: + resolution: {integrity: sha512-cXLGjP0c4T3flZJKQSuziYoq7MlT+rnvfZjfp7h+I7K9BNX54kP9nyWvdbwjQ4u1iWbOL4u96fgeZLToQlZC7w==} + dev: false + + /escalade@3.1.1: + resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} + engines: {node: '>=6'} + dev: false + + /escape-goat@4.0.0: + resolution: {integrity: sha512-2Sd4ShcWxbx6OY1IHyla/CVNwvg7XwZVoXZHcSu9w9SReNP1EzzD5T8NWKIR38fIqEns9kDWKUQTXXAmlDrdPg==} + engines: {node: '>=12'} + dev: false + + /escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + dev: false + + /escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + dev: false + + /escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + dev: false + + /escape-string-regexp@5.0.0: + resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} + engines: {node: '>=12'} + dev: false + + /eslint-scope@5.1.1: + resolution: {integrity: sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==} + engines: {node: '>=8.0.0'} + dependencies: + esrecurse: 4.3.0 + estraverse: 4.3.0 + dev: false + + /esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + dependencies: + estraverse: 5.3.0 + dev: false + + /estraverse@4.3.0: + resolution: {integrity: sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==} + engines: {node: '>=4.0'} + dev: false + + /estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + dev: false + + /estree-util-attach-comments@3.0.0: + resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /estree-util-build-jsx@3.0.1: + resolution: {integrity: sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==} + dependencies: + '@types/estree-jsx': 1.0.3 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-walker: 3.0.3 + dev: false + + /estree-util-is-identifier-name@3.0.0: + resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + dev: false + + /estree-util-to-js@2.0.0: + resolution: {integrity: sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==} + dependencies: + '@types/estree-jsx': 1.0.3 + astring: 1.8.6 + source-map: 0.7.4 + dev: false + + /estree-util-value-to-estree@3.0.1: + resolution: {integrity: sha512-b2tdzTurEIbwRh+mKrEcaWfu1wgb8J1hVsgREg7FFiecWwK/PhO8X0kyc+0bIcKNtD4sqxIdNoRy6/p/TvECEA==} + engines: {node: '>=16.0.0'} + dependencies: + '@types/estree': 1.0.5 + is-plain-obj: 4.1.0 + dev: false + + /estree-util-visit@2.0.0: + resolution: {integrity: sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/unist': 3.0.2 + dev: false + + /estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + dev: false + + /eta@2.2.0: + resolution: {integrity: sha512-UVQ72Rqjy/ZKQalzV5dCCJP80GrmPrMxh6NlNf+erV6ObL0ZFkhCstWRawS85z3smdr3d2wXPsZEY7rDPfGd2g==} + engines: {node: '>=6.0.0'} + dev: false + + /etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + dev: false + + /eval@0.1.8: + resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==} + engines: {node: '>= 0.8'} + dependencies: + '@types/node': 20.10.6 + require-like: 0.1.2 + dev: false + + /eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + dev: false + + /events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + dev: false + + /execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + dependencies: + cross-spawn: 7.0.3 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + dev: false + + /express@4.18.2: + resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} + engines: {node: '>= 0.10.0'} + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.1 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.5.0 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.2.0 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.1 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.7 + proxy-addr: 2.0.7 + qs: 6.11.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.18.0 + serve-static: 1.15.0 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /extend-shallow@2.0.1: + resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} + engines: {node: '>=0.10.0'} + dependencies: + is-extendable: 0.1.1 + dev: false + + /extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + dev: false + + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: false + + /fast-glob@3.3.2: + resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} + engines: {node: '>=8.6.0'} + dependencies: + '@nodelib/fs.stat': 2.0.5 + '@nodelib/fs.walk': 1.2.8 + glob-parent: 5.1.2 + merge2: 1.4.1 + micromatch: 4.0.5 + dev: false + + /fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + dev: false + + /fast-url-parser@1.1.3: + resolution: {integrity: sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ==} + dependencies: + punycode: 1.4.1 + dev: false + + /fastq@1.16.0: + resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} + dependencies: + reusify: 1.0.4 + dev: false + + /fault@2.0.1: + resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==} + dependencies: + format: 0.2.2 + dev: false + + /faye-websocket@0.11.4: + resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} + engines: {node: '>=0.8.0'} + dependencies: + websocket-driver: 0.7.4 + dev: false + + /feed@4.2.2: + resolution: {integrity: sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==} + engines: {node: '>=0.4.0'} + dependencies: + xml-js: 1.6.11 + dev: false + + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + + /file-loader@6.2.0(webpack@5.89.0): + resolution: {integrity: sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw==} + engines: {node: '>= 10.13.0'} + peerDependencies: + webpack: ^4.0.0 || ^5.0.0 + dependencies: + loader-utils: 2.0.4 + schema-utils: 3.3.0 + webpack: 5.89.0 + dev: false + + /filesize@8.0.7: + resolution: {integrity: sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==} + engines: {node: '>= 0.4.0'} + dev: false + + /fill-range@7.0.1: + resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} + engines: {node: '>=8'} + dependencies: + to-regex-range: 5.0.1 + dev: false + + /finalhandler@1.2.0: + resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} + engines: {node: '>= 0.8'} + dependencies: + debug: 2.6.9 + encodeurl: 1.0.2 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /find-cache-dir@4.0.0: + resolution: {integrity: sha512-9ZonPT4ZAK4a+1pUPVPZJapbi7O5qbbJPdYw/NOQWZZbVLdDTYM3A4R9z/DpAM08IDaFGsvPgiGZ82WEwUDWjg==} + engines: {node: '>=14.16'} + dependencies: + common-path-prefix: 3.0.0 + pkg-dir: 7.0.0 + dev: false + + /find-up@3.0.0: + resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==} + engines: {node: '>=6'} + dependencies: + locate-path: 3.0.0 + dev: false + + /find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + dev: false + + /find-up@6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + locate-path: 7.2.0 + path-exists: 5.0.0 + dev: false + + /flat@5.0.2: + resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} + hasBin: true + dev: false + + /follow-redirects@1.15.4: + resolution: {integrity: sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + + /fork-ts-checker-webpack-plugin@6.5.3(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-SbH/l9ikmMWycd5puHJKTkZJKddF4iRLyW3DeZ08HTI7NGyLS38MXd/KGgeWumQO7YNQbW2u/NtPT2YowbPaGQ==} + engines: {node: '>=10', yarn: '>=1.0.0'} + peerDependencies: + eslint: '>= 6' + typescript: '>= 2.7' + vue-template-compiler: '*' + webpack: '>= 4' + peerDependenciesMeta: + eslint: + optional: true + vue-template-compiler: + optional: true + dependencies: + '@babel/code-frame': 7.23.5 + '@types/json-schema': 7.0.15 + chalk: 4.1.2 + chokidar: 3.5.3 + cosmiconfig: 6.0.0 + deepmerge: 4.3.1 + fs-extra: 9.1.0 + glob: 7.2.3 + memfs: 3.5.3 + minimatch: 3.1.2 + schema-utils: 2.7.0 + semver: 7.5.4 + tapable: 1.1.3 + typescript: 5.3.3 + webpack: 5.89.0 + dev: false + + /form-data-encoder@2.1.4: + resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} + engines: {node: '>= 14.17'} + dev: false + + /format-util@1.0.5: + resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} + dev: false + + /format@0.2.2: + resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} + engines: {node: '>=0.4.x'} + dev: false + + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + + /forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + dev: false + + /fraction.js@4.3.7: + resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + dev: false + + /fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + dev: false + + /fs-extra@11.2.0: + resolution: {integrity: sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + + /fs-extra@9.1.0: + resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} + engines: {node: '>=10'} + dependencies: + at-least-node: 1.0.0 + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + + /fs-monkey@1.0.5: + resolution: {integrity: sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew==} + dev: false + + /fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + dev: false + + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + dev: false + + /gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + dev: false + + /get-intrinsic@1.2.2: + resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + dependencies: + function-bind: 1.1.2 + has-proto: 1.0.1 + has-symbols: 1.0.3 + hasown: 2.0.0 + dev: false + + /get-own-enumerable-property-symbols@3.0.2: + resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} + dev: false + + /get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + dev: false + + /github-slugger@1.5.0: + resolution: {integrity: sha512-wIh+gKBI9Nshz2o46B0B3f5k/W+WI9ZAv6y5Dn5WJ5SK1t0TnDimB4WE5rmTD05ZAIn8HALCZVmCsvj0w0v0lw==} + dev: false + + /glob-parent@5.1.2: + resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} + engines: {node: '>= 6'} + dependencies: + is-glob: 4.0.3 + dev: false + + /glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + dependencies: + is-glob: 4.0.3 + dev: false + + /glob-to-regexp@0.4.1: + resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} + dev: false + + /glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + dev: false + + /global-dirs@3.0.1: + resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} + engines: {node: '>=10'} + dependencies: + ini: 2.0.0 + dev: false + + /global-modules@2.0.0: + resolution: {integrity: sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==} + engines: {node: '>=6'} + dependencies: + global-prefix: 3.0.0 + dev: false + + /global-prefix@3.0.0: + resolution: {integrity: sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==} + engines: {node: '>=6'} + dependencies: + ini: 1.3.8 + kind-of: 6.0.3 + which: 1.3.1 + dev: false + + /globals@11.12.0: + resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==} + engines: {node: '>=4'} + dev: false + + /globby@11.1.0: + resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} + engines: {node: '>=10'} + dependencies: + array-union: 2.1.0 + dir-glob: 3.0.1 + fast-glob: 3.3.2 + ignore: 5.3.0 + merge2: 1.4.1 + slash: 3.0.0 + dev: false + + /globby@13.2.2: + resolution: {integrity: sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + dir-glob: 3.0.1 + fast-glob: 3.3.2 + ignore: 5.3.0 + merge2: 1.4.1 + slash: 4.0.0 + dev: false + + /gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + dependencies: + get-intrinsic: 1.2.2 + dev: false + + /got@12.6.1: + resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} + engines: {node: '>=14.16'} + dependencies: + '@sindresorhus/is': 5.6.0 + '@szmarczak/http-timer': 5.0.1 + cacheable-lookup: 7.0.0 + cacheable-request: 10.2.14 + decompress-response: 6.0.0 + form-data-encoder: 2.1.4 + get-stream: 6.0.1 + http2-wrapper: 2.2.1 + lowercase-keys: 3.0.0 + p-cancelable: 3.0.0 + responselike: 3.0.0 + dev: false + + /graceful-fs@4.2.10: + resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: false + + /graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + dev: false + + /gray-matter@4.0.3: + resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} + engines: {node: '>=6.0'} + dependencies: + js-yaml: 3.14.1 + kind-of: 6.0.3 + section-matter: 1.0.0 + strip-bom-string: 1.0.0 + dev: false + + /gzip-size@6.0.0: + resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} + engines: {node: '>=10'} + dependencies: + duplexer: 0.1.2 + dev: false + + /handle-thing@2.0.1: + resolution: {integrity: sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==} + dev: false + + /has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + dev: false + + /has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + dev: false + + /has-property-descriptors@1.0.1: + resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} + dependencies: + get-intrinsic: 1.2.2 + dev: false + + /has-proto@1.0.1: + resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} + engines: {node: '>= 0.4'} + dev: false + + /has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + dev: false + + /has-yarn@3.0.0: + resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /hasown@2.0.0: + resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} + engines: {node: '>= 0.4'} + dependencies: + function-bind: 1.1.2 + dev: false + + /hast-util-from-parse5@8.0.1: + resolution: {integrity: sha512-Er/Iixbc7IEa7r/XLtuG52zoqn/b3Xng/w6aZQ0xGVxzhw5xUFxcRqdPzP6yFi/4HBYRaifaI5fQ1RH8n0ZeOQ==} + dependencies: + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + devlop: 1.1.0 + hastscript: 8.0.0 + property-information: 6.4.0 + vfile: 6.0.1 + vfile-location: 5.0.2 + web-namespaces: 2.0.1 + dev: false + + /hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + dependencies: + '@types/hast': 3.0.3 + dev: false + + /hast-util-raw@9.0.1: + resolution: {integrity: sha512-5m1gmba658Q+lO5uqL5YNGQWeh1MYWZbZmWrM5lncdcuiXuo5E2HT/CIOp0rLF8ksfSwiCVJ3twlgVRyTGThGA==} + dependencies: + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + '@ungap/structured-clone': 1.2.0 + hast-util-from-parse5: 8.0.1 + hast-util-to-parse5: 8.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.0.2 + parse5: 7.1.2 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + dev: false + + /hast-util-to-estree@3.1.0: + resolution: {integrity: sha512-lfX5g6hqVh9kjS/B9E2gSkvHH4SZNiQFiqWS0x9fENzEl+8W12RqdRxX6d/Cwxi30tPQs3bIO+aolQJNp1bIyw==} + dependencies: + '@types/estree': 1.0.5 + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-attach-comments: 3.0.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + style-to-object: 0.4.4 + unist-util-position: 5.0.0 + zwitch: 2.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /hast-util-to-jsx-runtime@2.3.0: + resolution: {integrity: sha512-H/y0+IWPdsLLS738P8tDnrQ8Z+dj12zQQ6WC11TIM21C8WFVoIxcqWXf2H3hiTVZjF1AWqoimGwrTWecWrnmRQ==} + dependencies: + '@types/estree': 1.0.5 + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + style-to-object: 1.0.5 + unist-util-position: 5.0.0 + vfile-message: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: false + + /hast-util-to-parse5@8.0.0: + resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==} + dependencies: + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + dev: false + + /hast-util-whitespace@2.0.1: + resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} + dev: false + + /hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + dependencies: + '@types/hast': 3.0.3 + dev: false + + /hastscript@8.0.0: + resolution: {integrity: sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==} + dependencies: + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + dev: false + + /he@1.2.0: + resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} + hasBin: true + dev: false + + /heap@0.2.7: + resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} + dev: false + + /history@4.10.1: + resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==} + dependencies: + '@babel/runtime': 7.23.7 + loose-envify: 1.4.0 + resolve-pathname: 3.0.0 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + value-equal: 1.0.1 + dev: false + + /hoist-non-react-statics@3.3.2: + resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + dependencies: + react-is: 16.13.1 + dev: false + + /hpack.js@2.1.6: + resolution: {integrity: sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==} + dependencies: + inherits: 2.0.4 + obuf: 1.1.2 + readable-stream: 2.3.8 + wbuf: 1.7.3 + dev: false + + /htm@3.1.1: + resolution: {integrity: sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==} + dev: false + + /html-entities@2.4.0: + resolution: {integrity: sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==} + dev: false + + /html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + dev: false + + /html-loader@4.2.0(webpack@5.89.0): + resolution: {integrity: sha512-OxCHD3yt+qwqng2vvcaPApCEvbx+nXWu+v69TYHx1FO8bffHn/JjHtE3TTQZmHjwvnJe4xxzuecetDVBrQR1Zg==} + engines: {node: '>= 14.15.0'} + peerDependencies: + webpack: ^5.0.0 + dependencies: + html-minifier-terser: 7.2.0 + parse5: 7.1.2 + webpack: 5.89.0 + dev: false + + /html-minifier-terser@6.1.0: + resolution: {integrity: sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==} + engines: {node: '>=12'} + hasBin: true + dependencies: + camel-case: 4.1.2 + clean-css: 5.3.3 + commander: 8.3.0 + he: 1.2.0 + param-case: 3.0.4 + relateurl: 0.2.7 + terser: 5.26.0 + dev: false + + /html-minifier-terser@7.2.0: + resolution: {integrity: sha512-tXgn3QfqPIpGl9o+K5tpcj3/MN4SfLtsx2GWwBC3SSd0tXQGyF3gsSqad8loJgKZGM3ZxbYDd5yhiBIdWpmvLA==} + engines: {node: ^14.13.1 || >=16.0.0} + hasBin: true + dependencies: + camel-case: 4.1.2 + clean-css: 5.3.3 + commander: 10.0.1 + entities: 4.5.0 + param-case: 3.0.4 + relateurl: 0.2.7 + terser: 5.26.0 + dev: false + + /html-tags@3.3.1: + resolution: {integrity: sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==} + engines: {node: '>=8'} + dev: false + + /html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + dev: false + + /html-webpack-plugin@5.6.0(webpack@5.89.0): + resolution: {integrity: sha512-iwaY4wzbe48AfKLZ/Cc8k0L+FKG6oSNRaZ8x5A/T/IVDGyXcbHncM9TdDa93wn0FsSm82FhTKW7f3vS61thXAw==} + engines: {node: '>=10.13.0'} + peerDependencies: + '@rspack/core': 0.x || 1.x + webpack: ^5.20.0 + peerDependenciesMeta: + '@rspack/core': + optional: true + webpack: + optional: true + dependencies: + '@types/html-minifier-terser': 6.1.0 + html-minifier-terser: 6.1.0 + lodash: 4.17.21 + pretty-error: 4.0.0 + tapable: 2.2.1 + webpack: 5.89.0 + dev: false + + /htmlparser2@6.1.0: + resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==} + dependencies: + domelementtype: 2.3.0 + domhandler: 4.3.1 + domutils: 2.8.0 + entities: 2.2.0 + dev: false + + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + + /http-cache-semantics@4.1.1: + resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} + dev: false + + /http-deceiver@1.2.7: + resolution: {integrity: sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==} + dev: false + + /http-errors@1.6.3: + resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} + engines: {node: '>= 0.6'} + dependencies: + depd: 1.1.2 + inherits: 2.0.3 + setprototypeof: 1.1.0 + statuses: 1.5.0 + dev: false + + /http-errors@2.0.0: + resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} + engines: {node: '>= 0.8'} + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.1 + toidentifier: 1.0.1 + dev: false + + /http-parser-js@0.5.8: + resolution: {integrity: sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==} + dev: false + + /http-proxy-middleware@2.0.6(@types/express@4.17.21): + resolution: {integrity: sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==} + engines: {node: '>=12.0.0'} + peerDependencies: + '@types/express': ^4.17.13 + peerDependenciesMeta: + '@types/express': + optional: true + dependencies: + '@types/express': 4.17.21 + '@types/http-proxy': 1.17.14 + http-proxy: 1.18.1 + is-glob: 4.0.3 + is-plain-obj: 3.0.0 + micromatch: 4.0.5 + transitivePeerDependencies: + - debug + dev: false + + /http-proxy@1.18.1: + resolution: {integrity: sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==} + engines: {node: '>=8.0.0'} + dependencies: + eventemitter3: 4.0.7 + follow-redirects: 1.15.4 + requires-port: 1.0.0 + transitivePeerDependencies: + - debug + dev: false + + /http2-wrapper@2.2.1: + resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} + engines: {node: '>=10.19.0'} + dependencies: + quick-lru: 5.1.1 + resolve-alpn: 1.2.1 + dev: false + + /human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + dev: false + + /iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /icss-utils@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + dev: false + + /ignore@5.3.0: + resolution: {integrity: sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==} + engines: {node: '>= 4'} + dev: false + + /image-size@1.1.1: + resolution: {integrity: sha512-541xKlUw6jr/6gGuk92F+mYM5zaFAc5ahphvkqvNe2bQ6gVBkd6bfrmVJ2t4KDAfikAYZyIqTnktX3i6/aQDrQ==} + engines: {node: '>=16.x'} + hasBin: true + dependencies: + queue: 6.0.2 + dev: false + + /immer@9.0.21: + resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} + dev: false + + /import-fresh@3.3.0: + resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} + engines: {node: '>=6'} + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + dev: false + + /import-lazy@4.0.0: + resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} + engines: {node: '>=8'} + dev: false + + /imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + dev: false + + /indent-string@4.0.0: + resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} + engines: {node: '>=8'} + dev: false + + /infima@0.2.0-alpha.43: + resolution: {integrity: sha512-2uw57LvUqW0rK/SWYnd/2rRfxNA5DDNOh33jxF7fy46VWoNhGxiUQyVZHbBMjQ33mQem0cjdDVwgWVAmlRfgyQ==} + engines: {node: '>=12'} + dev: false + + /inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + dev: false + + /inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false + + /ini@2.0.0: + resolution: {integrity: sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==} + engines: {node: '>=10'} + dev: false + + /inline-style-parser@0.1.1: + resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} + dev: false + + /inline-style-parser@0.2.2: + resolution: {integrity: sha512-EcKzdTHVe8wFVOGEYXiW9WmJXPjqi1T+234YpJr98RiFYKHV3cdy1+3mkTE+KHTHxFFLH51SfaGOoUdW+v7ViQ==} + dev: false + + /internmap@1.0.1: + resolution: {integrity: sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==} + dev: false + + /internmap@2.0.3: + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} + engines: {node: '>=12'} + dev: false + + /interpret@1.4.0: + resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} + engines: {node: '>= 0.10'} + dev: false + + /invariant@2.2.4: + resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + dependencies: + loose-envify: 1.4.0 + dev: false + + /ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + dev: false + + /ipaddr.js@2.1.0: + resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==} + engines: {node: '>= 10'} + dev: false + + /is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + dev: false + + /is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + dev: false + + /is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + dev: false + + /is-binary-path@2.1.0: + resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} + engines: {node: '>=8'} + dependencies: + binary-extensions: 2.2.0 + dev: false + + /is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + dev: false + + /is-ci@3.0.1: + resolution: {integrity: sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==} + hasBin: true + dependencies: + ci-info: 3.9.0 + dev: false + + /is-core-module@2.13.1: + resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} + dependencies: + hasown: 2.0.0 + dev: false + + /is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + dev: false + + /is-docker@2.2.1: + resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} + engines: {node: '>=8'} + hasBin: true + dev: false + + /is-extendable@0.1.1: + resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} + engines: {node: '>=0.10.0'} + dev: false + + /is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + dev: false + + /is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + dev: false + + /is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + dependencies: + is-extglob: 2.1.1 + dev: false + + /is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + dev: false + + /is-installed-globally@0.4.0: + resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} + engines: {node: '>=10'} + dependencies: + global-dirs: 3.0.1 + is-path-inside: 3.0.3 + dev: false + + /is-npm@6.0.0: + resolution: {integrity: sha512-JEjxbSmtPSt1c8XTkVrlujcXdKV1/tvuQ7GwKcAlyiVLeYFQ2VHat8xfrDJsIkhCdF/tZ7CiIR3sy141c6+gPQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + dev: false + + /is-obj@1.0.1: + resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} + engines: {node: '>=0.10.0'} + dev: false + + /is-obj@2.0.0: + resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} + engines: {node: '>=8'} + dev: false + + /is-path-cwd@2.2.0: + resolution: {integrity: sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==} + engines: {node: '>=6'} + dev: false + + /is-path-inside@3.0.3: + resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} + engines: {node: '>=8'} + dev: false + + /is-plain-obj@3.0.0: + resolution: {integrity: sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==} + engines: {node: '>=10'} + dev: false + + /is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + dev: false + + /is-plain-object@2.0.4: + resolution: {integrity: sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==} + engines: {node: '>=0.10.0'} + dependencies: + isobject: 3.0.1 + dev: false + + /is-plain-object@5.0.0: + resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} + engines: {node: '>=0.10.0'} + dev: false + + /is-reference@3.0.2: + resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /is-regexp@1.0.0: + resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} + engines: {node: '>=0.10.0'} + dev: false + + /is-root@2.1.0: + resolution: {integrity: sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg==} + engines: {node: '>=6'} + dev: false + + /is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + dev: false + + /is-typedarray@1.0.0: + resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} + dev: false + + /is-wsl@2.2.0: + resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} + engines: {node: '>=8'} + dependencies: + is-docker: 2.2.1 + dev: false + + /is-yarn-global@0.4.1: + resolution: {integrity: sha512-/kppl+R+LO5VmhYSEWARUFjodS25D68gvj8W7z0I7OWhUla5xWu8KL6CtB2V0R6yqhnRgbcaREMr4EEM6htLPQ==} + engines: {node: '>=12'} + dev: false + + /isarray@0.0.1: + resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} + dev: false + + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + + /isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + dev: false + + /isobject@3.0.1: + resolution: {integrity: sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==} + engines: {node: '>=0.10.0'} + dev: false + + /javascript-stringify@2.1.0: + resolution: {integrity: sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg==} + dev: false + + /jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/node': 20.10.6 + chalk: 4.1.2 + ci-info: 3.9.0 + graceful-fs: 4.2.11 + picomatch: 2.3.1 + dev: false + + /jest-worker@27.5.1: + resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} + engines: {node: '>= 10.13.0'} + dependencies: + '@types/node': 20.10.6 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: false + + /jest-worker@29.7.0: + resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@types/node': 20.10.6 + jest-util: 29.7.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: false + + /jiti@1.21.0: + resolution: {integrity: sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==} + hasBin: true + dev: false + + /joi@17.11.0: + resolution: {integrity: sha512-NgB+lZLNoqISVy1rZocE9PZI36bL/77ie924Ri43yEvi9GUUMPeyVIr8KdFTMUlby1p0PBYMk9spIxEUQYqrJQ==} + dependencies: + '@hapi/hoek': 9.3.0 + '@hapi/topo': 5.1.0 + '@sideway/address': 4.1.4 + '@sideway/formula': 3.0.1 + '@sideway/pinpoint': 2.0.0 + dev: false + + /js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + dev: false + + /js-yaml@3.14.1: + resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} + hasBin: true + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + dev: false + + /js-yaml@4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: false + + /jsesc@0.5.0: + resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==} + hasBin: true + dev: false + + /jsesc@2.5.2: + resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + dev: false + + /json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + dev: false + + /json-schema-faker@0.5.4: + resolution: {integrity: sha512-DdRRnRNSxkQVXEsUUXzAtvBpsROZHvM59/LQcV6+3gQVMvaeMsqfNKN3ivRwaiahTW7pvxa+LJfOaPP+nhFo4g==} + hasBin: true + dependencies: + json-schema-ref-parser: 6.1.0 + jsonpath-plus: 7.2.0 + dev: false + + /json-schema-ref-parser@6.1.0: + resolution: {integrity: sha512-pXe9H1m6IgIpXmE5JSb8epilNTGsmTb2iPohAXpOdhqGFbQjNeHHsZxU+C8w6T81GZxSPFLeUoqDJmzxx5IGuw==} + deprecated: Please switch to @apidevtools/json-schema-ref-parser + dependencies: + call-me-maybe: 1.0.2 + js-yaml: 3.14.1 + ono: 4.0.11 + dev: false + + /json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + dev: false + + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + dev: false + + /json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + dev: false + + /jsonfile@6.1.0: + resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} + dependencies: + universalify: 2.0.1 + optionalDependencies: + graceful-fs: 4.2.11 + dev: false + + /jsonpath-plus@7.2.0: + resolution: {integrity: sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==} + engines: {node: '>=12.0.0'} + dev: false + + /keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + dependencies: + json-buffer: 3.0.1 + dev: false + + /khroma@2.1.0: + resolution: {integrity: sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==} + dev: false + + /kind-of@6.0.3: + resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} + engines: {node: '>=0.10.0'} + dev: false + + /kleur@3.0.3: + resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} + engines: {node: '>=6'} + dev: false + + /kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + dev: false + + /latest-version@7.0.0: + resolution: {integrity: sha512-KvNT4XqAMzdcL6ka6Tl3i2lYeFDgXNCuIX+xNx6ZMVR1dFq+idXd9FLKNMOIx0t9mJ9/HudyX4oZWXZQ0UJHeg==} + engines: {node: '>=14.16'} + dependencies: + package-json: 8.1.1 + dev: false + + /launch-editor@2.6.1: + resolution: {integrity: sha512-eB/uXmFVpY4zezmGp5XtU21kwo7GBbKB+EQ+UZeWtGb9yAM5xt/Evk+lYH3eRNAtId+ej4u7TYPFZ07w4s7rRw==} + dependencies: + picocolors: 1.0.0 + shell-quote: 1.8.1 + dev: false + + /layout-base@1.0.2: + resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} + dev: false + + /layout-base@2.0.1: + resolution: {integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==} + dev: false + + /leven@3.1.0: + resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} + engines: {node: '>=6'} + dev: false + + /lilconfig@2.1.0: + resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} + engines: {node: '>=10'} + dev: false + + /lilconfig@3.0.0: + resolution: {integrity: sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==} + engines: {node: '>=14'} + dev: false + + /lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + dev: false + + /loader-runner@4.3.0: + resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==} + engines: {node: '>=6.11.5'} + dev: false + + /loader-utils@2.0.4: + resolution: {integrity: sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==} + engines: {node: '>=8.9.0'} + dependencies: + big.js: 5.2.2 + emojis-list: 3.0.0 + json5: 2.2.3 + dev: false + + /loader-utils@3.2.1: + resolution: {integrity: sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==} + engines: {node: '>= 12.13.0'} + dev: false + + /locate-path@3.0.0: + resolution: {integrity: sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==} + engines: {node: '>=6'} + dependencies: + p-locate: 3.0.0 + path-exists: 3.0.0 + dev: false + + /locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + dependencies: + p-locate: 5.0.0 + dev: false + + /locate-path@7.2.0: + resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-locate: 6.0.0 + dev: false + + /lodash-es@4.17.21: + resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} + dev: false + + /lodash.debounce@4.0.8: + resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + dev: false + + /lodash.memoize@4.1.2: + resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} + dev: false + + /lodash.uniq@4.5.0: + resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} + dev: false + + /lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + + /longest-streak@3.1.0: + resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} + dev: false + + /loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + dependencies: + js-tokens: 4.0.0 + dev: false + + /lower-case@2.0.2: + resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} + dependencies: + tslib: 2.6.2 + dev: false + + /lowercase-keys@3.0.0: + resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + dependencies: + yallist: 3.1.1 + dev: false + + /lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + dependencies: + yallist: 4.0.0 + dev: false + + /lunr-languages@1.14.0: + resolution: {integrity: sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==} + dev: false + + /mark.js@8.11.1: + resolution: {integrity: sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==} + dev: false + + /markdown-extensions@2.0.0: + resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} + engines: {node: '>=16'} + dev: false + + /markdown-table@3.0.3: + resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==} + dev: false + + /mdast-util-definitions@5.1.2: + resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==} + dependencies: + '@types/mdast': 3.0.15 + '@types/unist': 2.0.10 + unist-util-visit: 4.1.2 + dev: false + + /mdast-util-directive@3.0.0: + resolution: {integrity: sha512-JUpYOqKI4mM3sZcNxmF/ox04XYFFkNwr0CFlrQIkCwbvH0xzMCqkMqAde9wRd80VAhaUrwFwKm2nxretdT1h7Q==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + parse-entities: 4.0.1 + stringify-entities: 4.0.3 + unist-util-visit-parents: 6.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-find-and-replace@3.0.1: + resolution: {integrity: sha512-SG21kZHGC3XRTSUhtofZkBzZTJNM5ecCi0SK2IMKmSXR8vO3peL+kb1O0z7Zl83jKtutG4k5Wv/W7V3/YHvzPA==} + dependencies: + '@types/mdast': 4.0.3 + escape-string-regexp: 5.0.0 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 + dev: false + + /mdast-util-from-markdown@1.3.1: + resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} + dependencies: + '@types/mdast': 3.0.15 + '@types/unist': 2.0.10 + decode-named-character-reference: 1.0.2 + mdast-util-to-string: 3.2.0 + micromark: 3.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-decode-string: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + unist-util-stringify-position: 3.0.3 + uvu: 0.5.6 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-from-markdown@2.0.0: + resolution: {integrity: sha512-n7MTOr/z+8NAX/wmhhDji8O3bRvPTV/U0oTCaZJkjhPSKTPhS3xufVhKGF8s1pJ7Ox4QgoIU7KHseh09S+9rTA==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + mdast-util-to-string: 4.0.0 + micromark: 4.0.0 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-decode-string: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-stringify-position: 4.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-frontmatter@2.0.1: + resolution: {integrity: sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + escape-string-regexp: 5.0.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + micromark-extension-frontmatter: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-autolink-literal@2.0.0: + resolution: {integrity: sha512-FyzMsduZZHSc3i0Px3PQcBT4WJY/X/RCtEJKuybiC6sjPqLv7h1yqAkmILZtuxMSsUyaLUWNp71+vQH2zqp5cg==} + dependencies: + '@types/mdast': 4.0.3 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-find-and-replace: 3.0.1 + micromark-util-character: 2.0.1 + dev: false + + /mdast-util-gfm-footnote@2.0.0: + resolution: {integrity: sha512-5jOT2boTSVkMnQ7LTrd6n/18kqwjmuYqo7JUPe+tRCY6O7dAuTFMtTPauYYrMPpox9hlN0uOx/FL8XvEfG9/mQ==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + micromark-util-normalize-identifier: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-table@2.0.0: + resolution: {integrity: sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + markdown-table: 3.0.3 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-task-list-item@2.0.0: + resolution: {integrity: sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm@3.0.0: + resolution: {integrity: sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==} + dependencies: + mdast-util-from-markdown: 2.0.0 + mdast-util-gfm-autolink-literal: 2.0.0 + mdast-util-gfm-footnote: 2.0.0 + mdast-util-gfm-strikethrough: 2.0.0 + mdast-util-gfm-table: 2.0.0 + mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx-expression@2.0.0: + resolution: {integrity: sha512-fGCu8eWdKUKNu5mohVGkhBXCXGnOTLuFqOvGMvdikr+J1w7lDJgxThOKpwRWzzbyXAU2hhSwsmssOY4yTokluw==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx-jsx@3.0.0: + resolution: {integrity: sha512-XZuPPzQNBPAlaqsTTgRrcJnyFbSOBovSadFgbFu8SnuNgm+6Bdx1K+IWoitsmj6Lq6MNtI+ytOqwN70n//NaBA==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + parse-entities: 4.0.1 + stringify-entities: 4.0.3 + unist-util-remove-position: 5.0.0 + unist-util-stringify-position: 4.0.0 + vfile-message: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx@3.0.0: + resolution: {integrity: sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==} + dependencies: + mdast-util-from-markdown: 2.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdxjs-esm@2.0.1: + resolution: {integrity: sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-phrasing@4.0.0: + resolution: {integrity: sha512-xadSsJayQIucJ9n053dfQwVu1kuXg7jCTdYsMK8rqzKZh52nLfSH/k0sAxE0u+pj/zKZX+o5wB+ML5mRayOxFA==} + dependencies: + '@types/mdast': 4.0.3 + unist-util-is: 6.0.0 + dev: false + + /mdast-util-to-hast@12.3.0: + resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} + dependencies: + '@types/hast': 2.3.9 + '@types/mdast': 3.0.15 + mdast-util-definitions: 5.1.2 + micromark-util-sanitize-uri: 1.2.0 + trim-lines: 3.0.1 + unist-util-generated: 2.0.1 + unist-util-position: 4.0.4 + unist-util-visit: 4.1.2 + dev: false + + /mdast-util-to-hast@13.0.2: + resolution: {integrity: sha512-U5I+500EOOw9e3ZrclN3Is3fRpw8c19SMyNZlZ2IS+7vLsNzb2Om11VpIVOR+/0137GhZsFEF6YiKD5+0Hr2Og==} + dependencies: + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + '@ungap/structured-clone': 1.2.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.0 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + dev: false + + /mdast-util-to-markdown@2.1.0: + resolution: {integrity: sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + longest-streak: 3.1.0 + mdast-util-phrasing: 4.0.0 + mdast-util-to-string: 4.0.0 + micromark-util-decode-string: 2.0.0 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + dev: false + + /mdast-util-to-string@3.2.0: + resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} + dependencies: + '@types/mdast': 3.0.15 + dev: false + + /mdast-util-to-string@4.0.0: + resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + dependencies: + '@types/mdast': 4.0.3 + dev: false + + /mdn-data@2.0.14: + resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} + dev: false + + /mdn-data@2.0.28: + resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} + dev: false + + /mdn-data@2.0.30: + resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + dev: false + + /media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + dev: false + + /memfs@3.5.3: + resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==} + engines: {node: '>= 4.0.0'} + dependencies: + fs-monkey: 1.0.5 + dev: false + + /merge-descriptors@1.0.1: + resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} + dev: false + + /merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: false + + /merge2@1.4.1: + resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} + engines: {node: '>= 8'} + dev: false + + /mermaid@10.6.1: + resolution: {integrity: sha512-Hky0/RpOw/1il9X8AvzOEChfJtVvmXm+y7JML5C//ePYMy0/9jCEmW1E1g86x9oDfW9+iVEdTV/i+M6KWRNs4A==} + dependencies: + '@braintree/sanitize-url': 6.0.4 + '@types/d3-scale': 4.0.8 + '@types/d3-scale-chromatic': 3.0.3 + cytoscape: 3.28.1 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.28.1) + cytoscape-fcose: 2.2.0(cytoscape@3.28.1) + d3: 7.8.5 + d3-sankey: 0.12.3 + dagre-d3-es: 7.0.10 + dayjs: 1.11.10 + dompurify: 3.0.6 + elkjs: 0.8.2 + khroma: 2.1.0 + lodash-es: 4.17.21 + mdast-util-from-markdown: 1.3.1 + non-layered-tidy-tree-layout: 2.0.2 + stylis: 4.3.1 + ts-dedent: 2.2.0 + uuid: 9.0.1 + web-worker: 1.2.0 + transitivePeerDependencies: + - supports-color + dev: false + + /methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + dev: false + + /micromark-core-commonmark@1.1.0: + resolution: {integrity: sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-factory-destination: 1.1.0 + micromark-factory-label: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-factory-title: 1.1.0 + micromark-factory-whitespace: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-classify-character: 1.1.0 + micromark-util-html-tag-name: 1.2.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-core-commonmark@2.0.0: + resolution: {integrity: sha512-jThOz/pVmAYUtkroV3D5c1osFXAMv9e0ypGDOIZuCeAe91/sD6BoE2Sjzt30yuXtwOYUmySOhMas/PVyh02itA==} + dependencies: + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-factory-destination: 2.0.0 + micromark-factory-label: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-factory-title: 2.0.0 + micromark-factory-whitespace: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-chunked: 2.0.0 + micromark-util-classify-character: 2.0.0 + micromark-util-html-tag-name: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-subtokenize: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-directive@3.0.0: + resolution: {integrity: sha512-61OI07qpQrERc+0wEysLHMvoiO3s2R56x5u7glHq2Yqq6EHbH4dW25G9GfDdGCDYqA21KE6DWgNSzxSwHc2hSg==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-factory-whitespace: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + parse-entities: 4.0.1 + dev: false + + /micromark-extension-frontmatter@2.0.0: + resolution: {integrity: sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==} + dependencies: + fault: 2.0.1 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-autolink-literal@2.0.0: + resolution: {integrity: sha512-rTHfnpt/Q7dEAK1Y5ii0W8bhfJlVJFnJMHIPisfPK3gpVNuOP0VnRl96+YJ3RYWV/P4gFeQoGKNlT3RhuvpqAg==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-footnote@2.0.0: + resolution: {integrity: sha512-6Rzu0CYRKDv3BfLAUnZsSlzx3ak6HAoI85KTiijuKIz5UxZxbUI+pD6oHgw+6UtQuiRwnGRhzMmPRv4smcz0fg==} + dependencies: + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-c3BR1ClMp5fxxmwP6AoOY2fXO9U8uFMKs4ADD66ahLTNcwzSCyRVU4k7LPV5Nxo/VJiR4TdzxRQY2v3qIUceCw==} + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.0 + micromark-util-classify-character: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-table@2.0.0: + resolution: {integrity: sha512-PoHlhypg1ItIucOaHmKE8fbin3vTLpDOUg8KAr8gRCF1MOZI9Nquq2i/44wFvviM4WuxJzc3demT8Y3dkfvYrw==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-tagfilter@2.0.0: + resolution: {integrity: sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-task-list-item@2.0.1: + resolution: {integrity: sha512-cY5PzGcnULaN5O7T+cOzfMoHjBW7j+T9D2sucA5d/KbsBTPcYdebm9zUd9zzdgJGCwahV+/W78Z3nbulBYVbTw==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm@3.0.0: + resolution: {integrity: sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==} + dependencies: + micromark-extension-gfm-autolink-literal: 2.0.0 + micromark-extension-gfm-footnote: 2.0.0 + micromark-extension-gfm-strikethrough: 2.0.0 + micromark-extension-gfm-table: 2.0.0 + micromark-extension-gfm-tagfilter: 2.0.0 + micromark-extension-gfm-task-list-item: 2.0.1 + micromark-util-combine-extensions: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdx-expression@3.0.0: + resolution: {integrity: sha512-sI0nwhUDz97xyzqJAbHQhp5TfaxEvZZZ2JDqUo+7NvyIYG6BZ5CPPqj2ogUoPJlmXHBnyZUzISg9+oUmU6tUjQ==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-factory-mdx-expression: 2.0.1 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdx-jsx@3.0.0: + resolution: {integrity: sha512-uvhhss8OGuzR4/N17L1JwvmJIpPhAd8oByMawEKx6NVdBCbesjH4t+vjEp3ZXft9DwvlKSD07fCeI44/N0Vf2w==} + dependencies: + '@types/acorn': 4.0.6 + '@types/estree': 1.0.5 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + micromark-factory-mdx-expression: 2.0.1 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-extension-mdx-md@2.0.0: + resolution: {integrity: sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdxjs-esm@3.0.0: + resolution: {integrity: sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-extension-mdxjs@3.0.0: + resolution: {integrity: sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==} + dependencies: + acorn: 8.11.3 + acorn-jsx: 5.3.2(acorn@8.11.3) + micromark-extension-mdx-expression: 3.0.0 + micromark-extension-mdx-jsx: 3.0.0 + micromark-extension-mdx-md: 2.0.0 + micromark-extension-mdxjs-esm: 3.0.0 + micromark-util-combine-extensions: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-destination@1.1.0: + resolution: {integrity: sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-destination@2.0.0: + resolution: {integrity: sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-label@1.1.0: + resolution: {integrity: sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-factory-label@2.0.0: + resolution: {integrity: sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==} + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-mdx-expression@2.0.1: + resolution: {integrity: sha512-F0ccWIUHRLRrYp5TC9ZYXmZo+p2AM13ggbsW4T0b5CRKP8KHVRB8t4pwtBgTxtjRmwrK0Irwm7vs2JOZabHZfg==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-factory-space@1.1.0: + resolution: {integrity: sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-space@2.0.0: + resolution: {integrity: sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-title@1.1.0: + resolution: {integrity: sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-title@2.0.0: + resolution: {integrity: sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==} + dependencies: + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-whitespace@1.1.0: + resolution: {integrity: sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-whitespace@2.0.0: + resolution: {integrity: sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==} + dependencies: + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-character@1.2.0: + resolution: {integrity: sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==} + dependencies: + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-character@2.0.1: + resolution: {integrity: sha512-3wgnrmEAJ4T+mGXAUfMvMAbxU9RDG43XmGce4j6CwPtVxB3vfwXSZ6KhFwDzZ3mZHhmPimMAXg71veiBGzeAZw==} + dependencies: + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-chunked@1.1.0: + resolution: {integrity: sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-chunked@2.0.0: + resolution: {integrity: sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-classify-character@1.1.0: + resolution: {integrity: sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-classify-character@2.0.0: + resolution: {integrity: sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-combine-extensions@1.1.0: + resolution: {integrity: sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==} + dependencies: + micromark-util-chunked: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-combine-extensions@2.0.0: + resolution: {integrity: sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==} + dependencies: + micromark-util-chunked: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-decode-numeric-character-reference@1.1.0: + resolution: {integrity: sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-decode-numeric-character-reference@2.0.1: + resolution: {integrity: sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-decode-string@1.1.0: + resolution: {integrity: sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-decode-string@2.0.0: + resolution: {integrity: sha512-r4Sc6leeUTn3P6gk20aFMj2ntPwn6qpDZqWvYmAG6NgvFTIlj4WtrAudLi65qYoaGdXYViXYw2pkmn7QnIFasA==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-util-character: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-encode@1.1.0: + resolution: {integrity: sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==} + dev: false + + /micromark-util-encode@2.0.0: + resolution: {integrity: sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==} + dev: false + + /micromark-util-events-to-acorn@2.0.2: + resolution: {integrity: sha512-Fk+xmBrOv9QZnEDguL9OI9/NQQp6Hz4FuQ4YmCb/5V7+9eAh1s6AYSvL20kHkD67YIg7EpE54TiSlcsf3vyZgA==} + dependencies: + '@types/acorn': 4.0.6 + '@types/estree': 1.0.5 + '@types/unist': 3.0.2 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-util-html-tag-name@1.2.0: + resolution: {integrity: sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==} + dev: false + + /micromark-util-html-tag-name@2.0.0: + resolution: {integrity: sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==} + dev: false + + /micromark-util-normalize-identifier@1.1.0: + resolution: {integrity: sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-normalize-identifier@2.0.0: + resolution: {integrity: sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-resolve-all@1.1.0: + resolution: {integrity: sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==} + dependencies: + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-resolve-all@2.0.0: + resolution: {integrity: sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-sanitize-uri@1.2.0: + resolution: {integrity: sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-encode: 1.1.0 + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-sanitize-uri@2.0.0: + resolution: {integrity: sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-encode: 2.0.0 + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-subtokenize@1.1.0: + resolution: {integrity: sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==} + dependencies: + micromark-util-chunked: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-util-subtokenize@2.0.0: + resolution: {integrity: sha512-vc93L1t+gpR3p8jxeVdaYlbV2jTYteDje19rNSS/H5dlhxUYll5Fy6vJ2cDwP8RnsXi818yGty1ayP55y3W6fg==} + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-symbol@1.1.0: + resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==} + dev: false + + /micromark-util-symbol@2.0.0: + resolution: {integrity: sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==} + dev: false + + /micromark-util-types@1.1.0: + resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==} + dev: false + + /micromark-util-types@2.0.0: + resolution: {integrity: sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==} + dev: false + + /micromark@3.2.0: + resolution: {integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==} + dependencies: + '@types/debug': 4.1.12 + debug: 4.3.4 + decode-named-character-reference: 1.0.2 + micromark-core-commonmark: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-combine-extensions: 1.1.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-encode: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-sanitize-uri: 1.2.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + transitivePeerDependencies: + - supports-color + dev: false + + /micromark@4.0.0: + resolution: {integrity: sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==} + dependencies: + '@types/debug': 4.1.12 + debug: 4.3.4 + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-chunked: 2.0.0 + micromark-util-combine-extensions: 2.0.0 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-encode: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-subtokenize: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /micromatch@4.0.5: + resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} + engines: {node: '>=8.6'} + dependencies: + braces: 3.0.2 + picomatch: 2.3.1 + dev: false + + /mime-db@1.33.0: + resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} + engines: {node: '>= 0.6'} + dev: false + + /mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: false + + /mime-types@2.1.18: + resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.33.0 + dev: false + + /mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false + + /mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + dev: false + + /mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + dev: false + + /mimic-response@4.0.0: + resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /mini-create-react-context@0.4.1(prop-types@15.8.1)(react@18.2.0): + resolution: {integrity: sha512-YWCYEmd5CQeHGSAKrYvXgmzzkrvssZcuuQDDeqkT+PziKGMgE+0MCCtcKbROzocGBG1meBLl2FotlRwf4gAzbQ==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + peerDependencies: + prop-types: ^15.0.0 + react: ^0.14.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + dependencies: + '@babel/runtime': 7.23.7 + prop-types: 15.8.1 + react: 18.2.0 + tiny-warning: 1.0.3 + dev: false + + /mini-css-extract-plugin@2.7.6(webpack@5.89.0): + resolution: {integrity: sha512-Qk7HcgaPkGG6eD77mLvZS1nmxlao3j+9PkrT9Uc7HAE1id3F41+DdBRYRYkbyfNRGzm8/YWtzhw7nVPmwhqTQw==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^5.0.0 + dependencies: + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /minimalistic-assert@1.0.1: + resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} + dev: false + + /minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + dependencies: + brace-expansion: 1.1.11 + dev: false + + /minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + dev: false + + /mri@1.2.0: + resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} + engines: {node: '>=4'} + dev: false + + /mrmime@2.0.0: + resolution: {integrity: sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==} + engines: {node: '>=10'} + dev: false + + /ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + dev: false + + /ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + dev: false + + /ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + dev: false + + /multicast-dns@7.2.5: + resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} + hasBin: true + dependencies: + dns-packet: 5.6.1 + thunky: 1.1.0 + dev: false + + /nanoid@3.3.7: + resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: false + + /negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + dev: false + + /neo-async@2.6.2: + resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + dev: false + + /no-case@3.0.4: + resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + dependencies: + lower-case: 2.0.2 + tslib: 2.6.2 + dev: false + + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-emoji@2.1.3: + resolution: {integrity: sha512-E2WEOVsgs7O16zsURJ/eH8BqhF029wGpEOnv7Urwdo2wmQanOACwJQh0devF9D9RhoZru0+9JXIS0dBXIAz+lA==} + engines: {node: '>=18'} + dependencies: + '@sindresorhus/is': 4.6.0 + char-regex: 1.0.2 + emojilib: 2.4.0 + skin-tone: 2.0.0 + dev: false + + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + + /node-forge@1.3.1: + resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} + engines: {node: '>= 6.13.0'} + dev: false + + /node-releases@2.0.14: + resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} + dev: false + + /non-layered-tidy-tree-layout@2.0.2: + resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==} + dev: false + + /normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + dev: false + + /normalize-range@0.1.2: + resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} + engines: {node: '>=0.10.0'} + dev: false + + /normalize-url@6.1.0: + resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==} + engines: {node: '>=10'} + dev: false + + /normalize-url@8.0.0: + resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==} + engines: {node: '>=14.16'} + dev: false + + /npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + dependencies: + path-key: 3.1.1 + dev: false + + /nprogress@0.2.0: + resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} + dev: false + + /nth-check@2.0.1: + resolution: {integrity: sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==} + dependencies: + boolbase: 1.0.0 + dev: false + + /object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + dev: false + + /object-inspect@1.13.1: + resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + dev: false + + /object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + dev: false + + /object.assign@4.1.5: + resolution: {integrity: sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + has-symbols: 1.0.3 + object-keys: 1.1.1 + dev: false + + /obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + dev: false + + /on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + dependencies: + ee-first: 1.1.1 + dev: false + + /on-headers@1.0.2: + resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} + engines: {node: '>= 0.8'} + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + dependencies: + mimic-fn: 2.1.0 + dev: false + + /ono@4.0.11: + resolution: {integrity: sha512-jQ31cORBFE6td25deYeD80wxKBMj+zBmHTrVxnc6CKhx8gho6ipmWM5zj/oeoqioZ99yqBls9Z/9Nss7J26G2g==} + dependencies: + format-util: 1.0.5 + dev: false + + /open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} + engines: {node: '>=12'} + dependencies: + define-lazy-prop: 2.0.0 + is-docker: 2.2.1 + is-wsl: 2.2.0 + dev: false + + /opener@1.5.2: + resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} + hasBin: true + dev: false + + /p-cancelable@3.0.0: + resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} + engines: {node: '>=12.20'} + dev: false + + /p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + dependencies: + p-try: 2.2.0 + dev: false + + /p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + dependencies: + yocto-queue: 0.1.0 + dev: false + + /p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + yocto-queue: 1.0.0 + dev: false + + /p-locate@3.0.0: + resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} + engines: {node: '>=6'} + dependencies: + p-limit: 2.3.0 + dev: false + + /p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + dependencies: + p-limit: 3.1.0 + dev: false + + /p-locate@6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-limit: 4.0.0 + dev: false + + /p-map@4.0.0: + resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} + engines: {node: '>=10'} + dependencies: + aggregate-error: 3.1.0 + dev: false + + /p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + dev: false + + /p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + dev: false + + /package-json@8.1.1: + resolution: {integrity: sha512-cbH9IAIJHNj9uXi196JVsRlt7cHKak6u/e6AkL/bkRelZ7rlL3X1YKxsZwa36xipOEKAsdtmaG6aAJoM1fx2zA==} + engines: {node: '>=14.16'} + dependencies: + got: 12.6.1 + registry-auth-token: 5.0.2 + registry-url: 6.0.1 + semver: 7.5.4 + dev: false + + /param-case@3.0.4: + resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==} + dependencies: + dot-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + dependencies: + callsites: 3.1.0 + dev: false + + /parse-entities@4.0.1: + resolution: {integrity: sha512-SWzvYcSJh4d/SGLIOQfZ/CoNv6BTlI6YEQ7Nj82oDVnRpwe/Z/F1EMx42x3JAOwGBlCjeCH0BRJQbQ/opHL17w==} + dependencies: + '@types/unist': 2.0.10 + character-entities: 2.0.2 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.0.2 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + dev: false + + /parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + dependencies: + '@babel/code-frame': 7.23.5 + error-ex: 1.3.2 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + dev: false + + /parse-numeric-range@1.3.0: + resolution: {integrity: sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==} + dev: false + + /parse-srcset@1.0.2: + resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==} + dev: false + + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + + /parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + dev: false + + /pascal-case@3.1.2: + resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} + dependencies: + no-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /path-exists@3.0.0: + resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==} + engines: {node: '>=4'} + dev: false + + /path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + dev: false + + /path-exists@5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + dev: false + + /path-is-inside@1.0.2: + resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} + dev: false + + /path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + dev: false + + /path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + dev: false + + /path-to-regexp@0.1.7: + resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} + dev: false + + /path-to-regexp@1.8.0: + resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==} + dependencies: + isarray: 0.0.1 + dev: false + + /path-to-regexp@2.2.1: + resolution: {integrity: sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ==} + dev: false + + /path-type@4.0.0: + resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} + engines: {node: '>=8'} + dev: false + + /periscopic@3.1.0: + resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} + dependencies: + '@types/estree': 1.0.5 + estree-walker: 3.0.3 + is-reference: 3.0.2 + dev: false + + /picocolors@1.0.0: + resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} + dev: false + + /picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + dev: false + + /pkg-dir@7.0.0: + resolution: {integrity: sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==} + engines: {node: '>=14.16'} + dependencies: + find-up: 6.3.0 + dev: false + + /pkg-up@3.1.0: + resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==} + engines: {node: '>=8'} + dependencies: + find-up: 3.0.0 + dev: false + + /postcss-calc@8.2.4(postcss@8.4.32): + resolution: {integrity: sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q==} + peerDependencies: + postcss: ^8.2.2 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-calc@9.0.1(postcss@8.4.32): + resolution: {integrity: sha512-TipgjGyzP5QzEhsOZUaIkeO5mKeMFpebWzRogWG/ysonUlnHcq5aJe0jOjpfzUU8PeSaBQnrE8ehR0QA5vs8PQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.2.2 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-colormin@5.3.1(postcss@8.4.32): + resolution: {integrity: sha512-UsWQG0AqTFQmpBegeLLc1+c3jIqBNB0zlDGRWR+dQ3pRKJL1oeMzyqmH3o2PIfn9MBdNrVPWhDbT769LxCTLJQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + colord: 2.9.3 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-colormin@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-TXKOxs9LWcdYo5cgmcSHPkyrLAh86hX1ijmyy6J8SbOhyv6ua053M3ZAM/0j44UsnQNIWdl8gb5L7xX2htKeLw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + colord: 2.9.3 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@5.1.3(postcss@8.4.32): + resolution: {integrity: sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-zTd4Vh0HxGkhg5aHtfCogcRHzGkvblfdWlQ53lIh1cJhYcGyIxh2hgtKoVh40AMktRERet+JKdB04nNG19kjmA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-aeBmaTnGQ+NUSVQT8aY0sKyAD/BaLJenEKZ03YK0JnDE1w1Rr8XShoxdal2V2H26xTJKr3v5haByOhJuyT4UYw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-discard-comments@5.1.2(postcss@8.4.32): + resolution: {integrity: sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-comments@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-f1KYNPtqYLUeZGCHQPKzzFtsHaRuECe6jLakf/RjSRqvF5XHLZnM2+fXLhb8Qh/HBFHs3M4cSLb1k3B899RYIg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-duplicates@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-duplicates@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-1hvUs76HLYR8zkScbwyJ8oJEugfPV+WchpnA+26fpJ7Smzs51CzGBHC32RS03psuX/2l0l0UKh2StzNxOrKCYg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-empty@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-empty@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-yitcmKwmVWtNsrrRqGJ7/C0YRy53i0mjexBDQ9zYxDwTWVBgbU4+C9jIZLmQlTDT9zhml+u0OMFJh8+31krmOg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-overridden@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-overridden@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-qs0ehZMMZpSESbRkw1+inkf51kak6OOzNRaoLd/U7Fatp0aN2HQ1rxGOrJvYcRAN9VpX8kUF13R2ofn8OlvFVA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-unused@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-KwLWymI9hbwXmJa0dkrzpRbSJEh0vVUd7r8t0yOGPcfKzyJJxFM8kLyC5Ev9avji6nY95pOp1W6HqIrfT+0VGw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-discard-unused@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-wr3lRPahxARmjow5BWML+9bD9D1u6FpfxlWg4lZqCIwvQLBZQD/S0Rq6A/juQwVFVXvMeRGa9TX1vpXuQ6FhTQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-loader@7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-iW5WTTBSC5BfsBJ9daFMPVrLT36MrNiC6fqOZTTaHjBNX6Pfd5p+hSBqe/fEeNd7pc13QiAyGt7VdGMw4eRC4A==} + engines: {node: '>= 14.15.0'} + peerDependencies: + postcss: ^7.0.0 || ^8.0.1 + webpack: ^5.0.0 + dependencies: + cosmiconfig: 8.3.6(typescript@5.3.3) + jiti: 1.21.0 + postcss: 8.4.32 + semver: 7.5.4 + webpack: 5.89.0 + transitivePeerDependencies: + - typescript + dev: false + + /postcss-merge-idents@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-pCijL1TREiCoog5nQp7wUe+TUonA2tC2sQ54UGeMmryK3UFGIYKqDyjnqd6RcuI4znFn9hWSLNN8xKE/vWcUQw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-merge-idents@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-ApqNUkzl3MJP+43DIIvoer98t7tcDVAcnLeAKjuTIM7HkMk8NXB6eqscMIjwQISwoSeE0WrEyIqVy+HoHAVcZw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-merge-longhand@5.1.7(postcss@8.4.32): + resolution: {integrity: sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 5.1.1(postcss@8.4.32) + dev: false + + /postcss-merge-longhand@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-vmr/HZQzaPXc45FRvSctqFTF05UaDnTn5ABX+UtQPJznDWT/QaFbVc/pJ5C2YPxx2J2XcfmWowlKwtCDwiQ5hA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 6.0.2(postcss@8.4.32) + dev: false + + /postcss-merge-longhand@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-+yfVB7gEM8SrCo9w2lCApKIEzrTKl5yS1F4yGhV3kSim6JzbfLGJyhR1B6X+6vOT0U33Mgx7iv4X9MVWuaSAfw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 6.0.2(postcss@8.4.32) + dev: false + + /postcss-merge-rules@5.1.4(postcss@8.4.32): + resolution: {integrity: sha512-0R2IuYpgU93y9lhVbO/OylTtKMVcHb67zjWIfCiKR9rWL3GUk1677LAqD/BcHizukdZEjT8Ru3oHRoAYoJy44g==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-merge-rules@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-6lm8bl0UfriSfxI+F/cezrebqqP8w702UC6SjZlUlBYwuRVNbmgcJuQU7yePIvD4MNT53r/acQCUAyulrpgmeQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-merge-rules@6.0.3(postcss@8.4.32): + resolution: {integrity: sha512-yfkDqSHGohy8sGYIJwBmIGDv4K4/WrJPX355XrxQb/CSsT4Kc/RxDi6akqn5s9bap85AWgv21ArcUWwWdGNSHA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-font-values@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-font-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-tIwmF1zUPoN6xOtA/2FgVk1ZKrLcCvE0dpZLtzyyte0j9zUeB8RTbCqrHZGjJlxOvNWKMYtunLrrl7HPOiR46w==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-gradients@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + colord: 2.9.3 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-gradients@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-M1RJWVjd6IOLPl1hYiOd5HQHgpp6cvJVLrieQYS9y07Yo8itAr6jaekzJphaJFR0tcg4kRewCk3kna9uHBxn/w==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + colord: 2.9.3 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-params@5.1.4(postcss@8.4.32): + resolution: {integrity: sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-params@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-zwQtbrPEBDj+ApELZ6QylLf2/c5zmASoOuA4DzolyVGdV38iR2I5QRMsZcHkcdkZzxpN8RS4cN7LPskOkTwTZw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-selectors@5.2.1(postcss@8.4.32): + resolution: {integrity: sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-selectors@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-mfReq5wrS6vkunxvJp6GDuOk+Ak6JV7134gp8L+ANRnV9VwqzTvBtX6lpohooVU750AR0D3pVx2Zn6uCCwOAfQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-selectors@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-0b+m+w7OAvZejPQdN2GjsXLv5o0jqYHX3aoV0e7RBKPCsB7TYG5KKWBFhGnB/iP3213Ts8c5H4wLPLMm7z28Sg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-modules-extract-imports@3.0.0(postcss@8.4.32): + resolution: {integrity: sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-modules-local-by-default@4.0.3(postcss@8.4.32): + resolution: {integrity: sha512-2/u2zraspoACtrbFRnTijMiQtb4GW4BvatjaG/bCjYQo8kLTdevCUlwuBHx2sCnSyrI3x3qj4ZK1j5LQBgzmwA==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-modules-scope@3.1.0(postcss@8.4.32): + resolution: {integrity: sha512-SaIbK8XW+MZbd0xHPf7kdfA/3eOt7vxJ72IRecn3EzuZVLr1r0orzf0MX/pN8m+NMDoo6X/SQd8oeKqGZd8PXg==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-modules-values@4.0.0(postcss@8.4.32): + resolution: {integrity: sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + dev: false + + /postcss-normalize-charset@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-normalize-charset@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-aW5LbMNRZ+oDV57PF9K+WI1Z8MPnF+A8qbajg/T8PP126YrGX1f9IQx21GI2OlGz7XFJi/fNi0GTbY948XJtXg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-normalize-display-values@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-display-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-mc3vxp2bEuCb4LgCcmG1y6lKJu1Co8T+rKHrcbShJwUmKJiEl761qb/QQCfFwlrvSeET3jksolCR/RZuMURudw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-positions@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-positions@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-HRsq8u/0unKNvm0cvwxcOUEcakFXqZ41fv3FOdPn916XFUrympjr+03oaLkuZENz3HE9RrQE9yU0Xv43ThWjQg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-repeat-style@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-repeat-style@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-Gbb2nmCy6tTiA7Sh2MBs3fj9W8swonk6lw+dFFeQT68B0Pzwp1kvisJQkdV6rbbMSd9brMlS8I8ts52tAGWmGQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-string@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-string@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-5Fhx/+xzALJD9EI26Aq23hXwmv97Zfy2VFrt5PLT8lAhnBIZvmaT5pQk+NuJ/GWj/QWaKSKbnoKDGLbV6qnhXg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-timing-functions@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-timing-functions@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-4zcczzHqmCU7L5dqTB9rzeqPWRMc0K2HoR+Bfl+FSMbqGBUcP5LRfgcH4BdRtLuzVQK1/FHdFoGT3F7rkEnY+g==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-unicode@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-unicode@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-Ff2VdAYCTGyMUwpevTZPZ4w0+mPjbZzLLyoLh/RMpqUqeQKZ+xMm31hkxBavDcGKcxm6ACzGk0nBfZ8LZkStKA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-url@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + normalize-url: 6.1.0 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-url@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-jEXL15tXSvbjm0yzUV7FBiEXwhIa9H88JOXDGQzmcWoB4mSjZIsmtto066s2iW9FYuIrIF4k04HA2BKAOpbsaQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-whitespace@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-whitespace@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-76i3NpWf6bB8UHlVuLRxG4zW2YykF9CTEcq/9LGAiz2qBuX5cBStadkk0jSkg9a9TCIXbMQz7yzrygKoCW9JuA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-ordered-values@5.1.3(postcss@8.4.32): + resolution: {integrity: sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-ordered-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-XXbb1O/MW9HdEhnBxitZpPFbIvDgbo9NK4c/5bOfiKpnIGZDoL2xd7/e6jW5DYLsWxBbs+1nZEnVgnjnlFViaA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-idents@5.2.0(postcss@8.4.32): + resolution: {integrity: sha512-BTrLjICoSB6gxbc58D5mdBK8OhXRDqud/zodYfdSi52qvDHdMwk+9kB9xsM8yJThH/sZU5A6QVSmMmaN001gIg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-idents@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-GKgyBLS5hMCJC8T36h4IH9u0XhmRHRwLwlxP6xVYbAuxKqn3LezEDDIxnb1/Cu2DXGc20jvWK9VZdCVtYAoTyg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-initial@5.1.2(postcss@8.4.32): + resolution: {integrity: sha512-dE/y2XRaqAi6OvjzD22pjTUQ8eOfc6m/natGHgKFBK9DxFmIm69YmaRVQrGgFlEfc1HePIurY0TmDeROK05rIg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + postcss: 8.4.32 + dev: false + + /postcss-reduce-initial@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-YGKalhNlCLcjcLvjU5nF8FyeCTkCO5UtvJEt0hrPZVCTtRLSOH4z00T1UntQPj4dUmIYZgMj8qK77JbSX95hSw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + postcss: 8.4.32 + dev: false + + /postcss-reduce-transforms@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-transforms@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-fUbV81OkUe75JM+VYO1gr/IoA2b/dRiH6HvMwhrIBSUrxq3jNZQZitSnugcTLDi1KkQh1eR/zi+iyxviUNBkcQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-selector-parser@6.0.15: + resolution: {integrity: sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==} + engines: {node: '>=4'} + dependencies: + cssesc: 3.0.0 + util-deprecate: 1.0.2 + dev: false + + /postcss-sort-media-queries@4.4.1(postcss@8.4.32): + resolution: {integrity: sha512-QDESFzDDGKgpiIh4GYXsSy6sek2yAwQx1JASl5AxBtU1Lq2JfKBljIPNdil989NcSKRQX1ToiaKphImtBuhXWw==} + engines: {node: '>=10.0.0'} + peerDependencies: + postcss: ^8.4.16 + dependencies: + postcss: 8.4.32 + sort-css-media-queries: 2.1.0 + dev: false + + /postcss-svgo@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + svgo: 2.8.0 + dev: false + + /postcss-svgo@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-IH5R9SjkTkh0kfFOQDImyy1+mTCb+E830+9SV1O+AaDcoHTvfsvt6WwJeo7KwcHbFnevZVCsXhDmjFiGVuwqFQ==} + engines: {node: ^14 || ^16 || >= 18} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + svgo: 3.2.0 + dev: false + + /postcss-unique-selectors@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-unique-selectors@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-8IZGQ94nechdG7Y9Sh9FlIY2b4uS8/k8kdKRX040XHsS3B6d1HrJAkXrBSsSu4SuARruSsUjW3nlSw8BHkaAYQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + dev: false + + /postcss-zindex@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-fgFMf0OtVSBR1va1JNHYgMxYk73yhn/qb4uQDq1DLGYolz8gHCyr/sesEuGUaYs58E3ZJRcpoGuPVoB7Meiq9A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-zindex@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-wQF95TIerYvPlsjwldO7iGP3Z3arhuYRK/gndq4NAdZaEsdUkmQYtRqkrEPMzJOQFBk06wFtzkHZKJoQlqFgXQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss@8.4.32: + resolution: {integrity: sha512-D/kj5JNu6oo2EIy+XL/26JEDTlIbB8hw85G8StOE6L74RQAVVP5rej6wxCNqyMbR4RkPfqvezVbPw81Ngd6Kcw==} + engines: {node: ^10 || ^12 || >=14} + dependencies: + nanoid: 3.3.7 + picocolors: 1.0.0 + source-map-js: 1.0.2 + dev: false + + /preact@10.19.3: + resolution: {integrity: sha512-nHHTeFVBTHRGxJXKkKu5hT8C/YWBkPso4/Gad6xuj5dbptt9iF9NZr9pHbPhBrnT2klheu7mHTxTZ/LjwJiEiQ==} + dev: false + + /pretty-error@4.0.0: + resolution: {integrity: sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==} + dependencies: + lodash: 4.17.21 + renderkid: 3.0.0 + dev: false + + /pretty-time@1.1.0: + resolution: {integrity: sha512-28iF6xPQrP8Oa6uxE6a1biz+lWeTOAPKggvjB8HAs6nVMKZwf5bG++632Dx614hIWgUPkgivRfG+a8uAXGTIbA==} + engines: {node: '>=4'} + dev: false + + /prism-react-renderer@2.3.1(react@18.2.0): + resolution: {integrity: sha512-Rdf+HzBLR7KYjzpJ1rSoxT9ioO85nZngQEoFIhL07XhtJHlCU3SOz0GJ6+qvMyQe0Se+BV3qpe6Yd/NmQF5Juw==} + peerDependencies: + react: '>=16.0.0' + dependencies: + '@types/prismjs': 1.26.3 + clsx: 2.1.0 + react: 18.2.0 + dev: false + + /prismjs@1.29.0: + resolution: {integrity: sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==} + engines: {node: '>=6'} + dev: false + + /process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: false + + /prompts@2.4.2: + resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} + engines: {node: '>= 6'} + dependencies: + kleur: 3.0.3 + sisteransi: 1.0.5 + dev: false + + /prop-types@15.8.1: + resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + dependencies: + loose-envify: 1.4.0 + object-assign: 4.1.1 + react-is: 16.13.1 + dev: false + + /property-information@6.4.0: + resolution: {integrity: sha512-9t5qARVofg2xQqKtytzt+lZ4d1Qvj8t5B8fEwXK6qOfgRLgH/b13QlgEyDh033NOS31nXeFbYv7CLUDG1CeifQ==} + dev: false + + /proto-list@1.2.4: + resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} + dev: false + + /proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + dev: false + + /punycode@1.4.1: + resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} + dev: false + + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + dev: false + + /pupa@3.1.0: + resolution: {integrity: sha512-FLpr4flz5xZTSJxSeaheeMKN/EDzMdK7b8PTOC6a5PYFKTucWbdqjgqaEyH0shFiSJrVB1+Qqi4Tk19ccU6Aug==} + engines: {node: '>=12.20'} + dependencies: + escape-goat: 4.0.0 + dev: false + + /qs@6.11.0: + resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.4 + dev: false + + /queue-microtask@1.2.3: + resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + dev: false + + /queue@6.0.2: + resolution: {integrity: sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==} + dependencies: + inherits: 2.0.4 + dev: false + + /quick-lru@5.1.1: + resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} + engines: {node: '>=10'} + dev: false + + /randombytes@2.1.0: + resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /range-parser@1.2.0: + resolution: {integrity: sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==} + engines: {node: '>= 0.6'} + dev: false + + /range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + dev: false + + /raw-body@2.5.1: + resolution: {integrity: sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==} + engines: {node: '>= 0.8'} + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + dev: false + + /rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + dev: false + + /react-dev-utils@12.0.1(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=2.7' + webpack: '>=4' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@babel/code-frame': 7.23.5 + address: 1.2.2 + browserslist: 4.22.2 + chalk: 4.1.2 + cross-spawn: 7.0.3 + detect-port-alt: 1.1.6 + escape-string-regexp: 4.0.0 + filesize: 8.0.7 + find-up: 5.0.0 + fork-ts-checker-webpack-plugin: 6.5.3(typescript@5.3.3)(webpack@5.89.0) + global-modules: 2.0.0 + globby: 11.1.0 + gzip-size: 6.0.0 + immer: 9.0.21 + is-root: 2.1.0 + loader-utils: 3.2.1 + open: 8.4.2 + pkg-up: 3.1.0 + prompts: 2.4.2 + react-error-overlay: 6.0.11 + recursive-readdir: 2.2.3 + shell-quote: 1.8.1 + strip-ansi: 6.0.1 + text-table: 0.2.0 + typescript: 5.3.3 + webpack: 5.89.0 + transitivePeerDependencies: + - eslint + - supports-color + - vue-template-compiler + dev: false + + /react-dom@18.2.0(react@18.2.0): + resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} + peerDependencies: + react: ^18.2.0 + dependencies: + loose-envify: 1.4.0 + react: 18.2.0 + scheduler: 0.23.0 + dev: false + + /react-error-overlay@6.0.11: + resolution: {integrity: sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg==} + dev: false + + /react-fast-compare@3.2.2: + resolution: {integrity: sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==} + dev: false + + /react-helmet-async@1.3.0(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-9jZ57/dAn9t3q6hneQS0wukqC2ENOBgMNVEhb/ZG9ZSxUetzVIw4iAmEU38IaVg3QGYauQPhSeUTuIUtFglWpg==} + peerDependencies: + react: ^16.6.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.6.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@babel/runtime': 7.23.7 + invariant: 2.2.4 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-fast-compare: 3.2.2 + shallowequal: 1.1.0 + dev: false + + /react-helmet-async@2.0.4(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-yxjQMWposw+akRfvpl5+8xejl4JtUlHnEBcji6u8/e6oc7ozT+P9PNTWMhCbz2y9tc5zPegw2BvKjQA+NwdEjQ==} + peerDependencies: + react: ^16.6.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.6.0 || ^17.0.0 || ^18.0.0 + dependencies: + invariant: 2.2.4 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-fast-compare: 3.2.2 + shallowequal: 1.1.0 + dev: false + + /react-is@16.13.1: + resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + dev: false + + /react-is@18.2.0: + resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} + dev: false + + /react-json-view-lite@1.2.1(react@18.2.0): + resolution: {integrity: sha512-Itc0g86fytOmKZoIoJyGgvNqohWSbh3NXIKNgH6W6FT9PC1ck4xas1tT3Rr/b3UlFXyA9Jjaw9QSXdZy2JwGMQ==} + engines: {node: '>=14'} + peerDependencies: + react: ^16.13.1 || ^17.0.0 || ^18.0.0 + dependencies: + react: 18.2.0 + dev: false + + /react-loadable-ssr-addon-v5-slorber@1.0.1(@docusaurus/react-loadable@5.5.2)(webpack@5.89.0): + resolution: {integrity: sha512-lq3Lyw1lGku8zUEJPDxsNm1AfYHBrO9Y1+olAYwpUJ2IGFBskM0DMKok97A6LWUpHm+o7IvQBOWu9MLenp9Z+A==} + engines: {node: '>=10.13.0'} + peerDependencies: + react-loadable: '*' + webpack: '>=4.41.1 || 5.x' + dependencies: + '@babel/runtime': 7.23.7 + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + webpack: 5.89.0 + dev: false + + /react-markdown@8.0.7(@types/react@18.2.46)(react@18.2.0): + resolution: {integrity: sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + dependencies: + '@types/hast': 2.3.9 + '@types/prop-types': 15.7.11 + '@types/react': 18.2.46 + '@types/unist': 2.0.10 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 2.0.1 + prop-types: 15.8.1 + property-information: 6.4.0 + react: 18.2.0 + react-is: 18.2.0 + remark-parse: 10.0.2 + remark-rehype: 10.1.0 + space-separated-tokens: 2.0.2 + style-to-object: 0.4.4 + unified: 10.1.2 + unist-util-visit: 4.1.2 + vfile: 5.3.7 + transitivePeerDependencies: + - supports-color + dev: false + + /react-router-config@5.1.1(react-router@5.3.4)(react@18.2.0): + resolution: {integrity: sha512-DuanZjaD8mQp1ppHjgnnUnyOlqYXZVjnov/JzFhjLEwd3Z4dYjMSnqrEzzGThH47vpCOqPPwJM2FtthLeJ8Pbg==} + peerDependencies: + react: '>=15' + react-router: '>=5' + dependencies: + '@babel/runtime': 7.23.7 + react: 18.2.0 + react-router: 5.3.4(react@18.2.0) + dev: false + + /react-router-dom@5.3.4(react@18.2.0): + resolution: {integrity: sha512-m4EqFMHv/Ih4kpcBCONHbkT68KoAeHN4p3lAGoNryfHi0dMy0kCzEZakiKRsvg5wHZ/JLrLW8o8KomWiz/qbYQ==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + loose-envify: 1.4.0 + prop-types: 15.8.1 + react: 18.2.0 + react-router: 5.3.4(react@18.2.0) + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react-router@5.3.3(react@18.2.0): + resolution: {integrity: sha512-mzQGUvS3bM84TnbtMYR8ZjKnuPJ71IjSzR+DE6UkUqvN4czWIqEs17yLL8xkAycv4ev0AiN+IGrWu88vJs/p2w==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + hoist-non-react-statics: 3.3.2 + loose-envify: 1.4.0 + mini-create-react-context: 0.4.1(prop-types@15.8.1)(react@18.2.0) + path-to-regexp: 1.8.0 + prop-types: 15.8.1 + react: 18.2.0 + react-is: 16.13.1 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react-router@5.3.4(react@18.2.0): + resolution: {integrity: sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + hoist-non-react-statics: 3.3.2 + loose-envify: 1.4.0 + path-to-regexp: 1.8.0 + prop-types: 15.8.1 + react: 18.2.0 + react-is: 16.13.1 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react@18.2.0: + resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==} + engines: {node: '>=0.10.0'} + dependencies: + loose-envify: 1.4.0 + dev: false + + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + dev: false + + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + dev: false + + /readdirp@3.6.0: + resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} + engines: {node: '>=8.10.0'} + dependencies: + picomatch: 2.3.1 + dev: false + + /reading-time@1.5.0: + resolution: {integrity: sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg==} + dev: false + + /rechoir@0.6.2: + resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} + engines: {node: '>= 0.10'} + dependencies: + resolve: 1.22.8 + dev: false + + /recursive-readdir@2.2.3: + resolution: {integrity: sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==} + engines: {node: '>=6.0.0'} + dependencies: + minimatch: 3.1.2 + dev: false + + /regenerate-unicode-properties@10.1.1: + resolution: {integrity: sha512-X007RyZLsCJVVrjgEFVpLUTZwyOZk3oiL75ZcuYjlIWd6rNJtOjkBwQc5AsRrpbKVkxN6sklw/k/9m2jJYOf8Q==} + engines: {node: '>=4'} + dependencies: + regenerate: 1.4.2 + dev: false + + /regenerate@1.4.2: + resolution: {integrity: sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==} + dev: false + + /regenerator-runtime@0.14.1: + resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} + dev: false + + /regenerator-transform@0.15.2: + resolution: {integrity: sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==} + dependencies: + '@babel/runtime': 7.23.7 + dev: false + + /regexpu-core@5.3.2: + resolution: {integrity: sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==} + engines: {node: '>=4'} + dependencies: + '@babel/regjsgen': 0.8.0 + regenerate: 1.4.2 + regenerate-unicode-properties: 10.1.1 + regjsparser: 0.9.1 + unicode-match-property-ecmascript: 2.0.0 + unicode-match-property-value-ecmascript: 2.1.0 + dev: false + + /registry-auth-token@5.0.2: + resolution: {integrity: sha512-o/3ikDxtXaA59BmZuZrJZDJv8NMDGSj+6j6XaeBmHw8eY1i1qd9+6H+LjVvQXx3HN6aRCGa1cUdJ9RaJZUugnQ==} + engines: {node: '>=14'} + dependencies: + '@pnpm/npm-conf': 2.2.2 + dev: false + + /registry-url@6.0.1: + resolution: {integrity: sha512-+crtS5QjFRqFCoQmvGduwYWEBng99ZvmFvF+cUJkGYF1L1BfU8C6Zp9T7f5vPAwyLkUExpvK+ANVZmGU49qi4Q==} + engines: {node: '>=12'} + dependencies: + rc: 1.2.8 + dev: false + + /regjsparser@0.9.1: + resolution: {integrity: sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==} + hasBin: true + dependencies: + jsesc: 0.5.0 + dev: false + + /rehype-raw@7.0.0: + resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} + dependencies: + '@types/hast': 3.0.3 + hast-util-raw: 9.0.1 + vfile: 6.0.1 + dev: false + + /relateurl@0.2.7: + resolution: {integrity: sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==} + engines: {node: '>= 0.10'} + dev: false + + /remark-directive@3.0.0: + resolution: {integrity: sha512-l1UyWJ6Eg1VPU7Hm/9tt0zKtReJQNOA4+iDMAxTyZNWnJnFlbS/7zhiel/rogTLQ2vMYwDzSJa4BiVNqGlqIMA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-directive: 3.0.0 + micromark-extension-directive: 3.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-emoji@4.0.1: + resolution: {integrity: sha512-fHdvsTR1dHkWKev9eNyhTo4EFwbUvJ8ka9SgeWkMPYFX4WoI7ViVBms3PjlQYgw5TLvNQso3GUB/b/8t3yo+dg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + '@types/mdast': 4.0.3 + emoticon: 4.0.1 + mdast-util-find-and-replace: 3.0.1 + node-emoji: 2.1.3 + unified: 11.0.4 + dev: false + + /remark-frontmatter@5.0.0: + resolution: {integrity: sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-frontmatter: 2.0.1 + micromark-extension-frontmatter: 2.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-gfm@4.0.0: + resolution: {integrity: sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-gfm: 3.0.0 + micromark-extension-gfm: 3.0.0 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-mdx@3.0.0: + resolution: {integrity: sha512-O7yfjuC6ra3NHPbRVxfflafAj3LTwx3b73aBvkEFU5z4PsD6FD4vrqJAkE5iNGLz71GdjXfgRqm3SQ0h0VuE7g==} + dependencies: + mdast-util-mdx: 3.0.0 + micromark-extension-mdxjs: 3.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-parse@10.0.2: + resolution: {integrity: sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw==} + dependencies: + '@types/mdast': 3.0.15 + mdast-util-from-markdown: 1.3.1 + unified: 10.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-parse@11.0.0: + resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-from-markdown: 2.0.0 + micromark-util-types: 2.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-rehype@10.1.0: + resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} + dependencies: + '@types/hast': 2.3.9 + '@types/mdast': 3.0.15 + mdast-util-to-hast: 12.3.0 + unified: 10.1.2 + dev: false + + /remark-rehype@11.0.0: + resolution: {integrity: sha512-vx8x2MDMcxuE4lBmQ46zYUDfcFMmvg80WYX+UNLeG6ixjdCCLcw1lrgAukwBTuOFsS78eoAedHGn9sNM0w7TPw==} + dependencies: + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + mdast-util-to-hast: 13.0.2 + unified: 11.0.4 + vfile: 6.0.1 + dev: false + + /remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-to-markdown: 2.1.0 + unified: 11.0.4 + dev: false + + /renderkid@3.0.0: + resolution: {integrity: sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==} + dependencies: + css-select: 4.3.0 + dom-converter: 0.2.0 + htmlparser2: 6.1.0 + lodash: 4.17.21 + strip-ansi: 6.0.1 + dev: false + + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + + /require-like@0.1.2: + resolution: {integrity: sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==} + dev: false + + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + + /resolve-alpn@1.2.1: + resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} + dev: false + + /resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + dev: false + + /resolve-pathname@3.0.0: + resolution: {integrity: sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==} + dev: false + + /resolve@1.22.8: + resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} + hasBin: true + dependencies: + is-core-module: 2.13.1 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + dev: false + + /responselike@3.0.0: + resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} + engines: {node: '>=14.16'} + dependencies: + lowercase-keys: 3.0.0 + dev: false + + /retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + + /reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + dev: false + + /rimraf@3.0.2: + resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + hasBin: true + dependencies: + glob: 7.2.3 + dev: false + + /robust-predicates@3.0.2: + resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} + dev: false + + /rtl-detect@1.1.2: + resolution: {integrity: sha512-PGMBq03+TTG/p/cRB7HCLKJ1MgDIi07+QU1faSjiYRfmY5UsAttV9Hs08jDAHVwcOwmVLcSJkpwyfXszVjWfIQ==} + dev: false + + /rtlcss@4.1.1: + resolution: {integrity: sha512-/oVHgBtnPNcggP2aVXQjSy6N1mMAfHg4GSag0QtZBlD5bdDgAHwr4pydqJGd+SUCu9260+Pjqbjwtvu7EMH1KQ==} + engines: {node: '>=12.0.0'} + hasBin: true + dependencies: + escalade: 3.1.1 + picocolors: 1.0.0 + postcss: 8.4.32 + strip-json-comments: 3.1.1 + dev: false + + /run-parallel@1.2.0: + resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + dependencies: + queue-microtask: 1.2.3 + dev: false + + /rw@1.3.3: + resolution: {integrity: sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==} + dev: false + + /sade@1.8.1: + resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} + engines: {node: '>=6'} + dependencies: + mri: 1.2.0 + dev: false + + /safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: false + + /safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + dev: false + + /safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + dev: false + + /sanitize-html@2.11.0: + resolution: {integrity: sha512-BG68EDHRaGKqlsNjJ2xUB7gpInPA8gVx/mvjO743hZaeMCZ2DwzW7xvsqZ+KNU4QKwj86HJ3uu2liISf2qBBUA==} + dependencies: + deepmerge: 4.3.1 + escape-string-regexp: 4.0.0 + htmlparser2: 8.0.2 + is-plain-object: 5.0.0 + parse-srcset: 1.0.2 + postcss: 8.4.32 + dev: false + + /sax@1.3.0: + resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} + dev: false + + /scheduler@0.23.0: + resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} + dependencies: + loose-envify: 1.4.0 + dev: false + + /schema-utils@2.7.0: + resolution: {integrity: sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A==} + engines: {node: '>= 8.9.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 6.12.6 + ajv-keywords: 3.5.2(ajv@6.12.6) + dev: false + + /schema-utils@3.3.0: + resolution: {integrity: sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==} + engines: {node: '>= 10.13.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 6.12.6 + ajv-keywords: 3.5.2(ajv@6.12.6) + dev: false + + /schema-utils@4.2.0: + resolution: {integrity: sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==} + engines: {node: '>= 12.13.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + ajv-keywords: 5.1.0(ajv@8.12.0) + dev: false + + /search-insights@2.13.0: + resolution: {integrity: sha512-Orrsjf9trHHxFRuo9/rzm0KIWmgzE8RMlZMzuhZOJ01Rnz3D0YBAe+V6473t6/H6c7irs6Lt48brULAiRWb3Vw==} + dev: false + + /section-matter@1.0.0: + resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} + engines: {node: '>=4'} + dependencies: + extend-shallow: 2.0.1 + kind-of: 6.0.3 + dev: false + + /select-hose@2.0.0: + resolution: {integrity: sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==} + dev: false + + /selfsigned@2.4.1: + resolution: {integrity: sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==} + engines: {node: '>=10'} + dependencies: + '@types/node-forge': 1.3.11 + node-forge: 1.3.1 + dev: false + + /semver-diff@4.0.0: + resolution: {integrity: sha512-0Ju4+6A8iOnpL/Thra7dZsSlOHYAHIeMxfhWQRI1/VLcT3WDBZKKtQt/QkBOsiIN9ZpuvHE6cGZ0x4glCMmfiA==} + engines: {node: '>=12'} + dependencies: + semver: 7.5.4 + dev: false + + /semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + dev: false + + /semver@7.5.4: + resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} + engines: {node: '>=10'} + hasBin: true + dependencies: + lru-cache: 6.0.0 + dev: false + + /send@0.18.0: + resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} + engines: {node: '>= 0.8.0'} + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /serialize-javascript@6.0.1: + resolution: {integrity: sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==} + dependencies: + randombytes: 2.1.0 + dev: false + + /serve-handler@6.1.5: + resolution: {integrity: sha512-ijPFle6Hwe8zfmBxJdE+5fta53fdIY0lHISJvuikXB3VYFafRjMRpOffSPvCYsbKyBA7pvy9oYr/BT1O3EArlg==} + dependencies: + bytes: 3.0.0 + content-disposition: 0.5.2 + fast-url-parser: 1.1.3 + mime-types: 2.1.18 + minimatch: 3.1.2 + path-is-inside: 1.0.2 + path-to-regexp: 2.2.1 + range-parser: 1.2.0 + dev: false + + /serve-index@1.9.1: + resolution: {integrity: sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==} + engines: {node: '>= 0.8.0'} + dependencies: + accepts: 1.3.8 + batch: 0.6.1 + debug: 2.6.9 + escape-html: 1.0.3 + http-errors: 1.6.3 + mime-types: 2.1.35 + parseurl: 1.3.3 + transitivePeerDependencies: + - supports-color + dev: false + + /serve-static@1.15.0: + resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==} + engines: {node: '>= 0.8.0'} + dependencies: + encodeurl: 1.0.2 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.18.0 + transitivePeerDependencies: + - supports-color + dev: false + + /set-function-length@1.1.1: + resolution: {integrity: sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.1 + get-intrinsic: 1.2.2 + gopd: 1.0.1 + has-property-descriptors: 1.0.1 + dev: false + + /setprototypeof@1.1.0: + resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} + dev: false + + /setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + dev: false + + /shallow-clone@3.0.1: + resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==} + engines: {node: '>=8'} + dependencies: + kind-of: 6.0.3 + dev: false + + /shallowequal@1.1.0: + resolution: {integrity: sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==} + dev: false + + /shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + dependencies: + shebang-regex: 3.0.0 + dev: false + + /shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + dev: false + + /shell-quote@1.8.1: + resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==} + dev: false + + /shelljs@0.8.5: + resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} + engines: {node: '>=4'} + hasBin: true + dependencies: + glob: 7.2.3 + interpret: 1.4.0 + rechoir: 0.6.2 + dev: false + + /side-channel@1.0.4: + resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + dependencies: + call-bind: 1.0.5 + get-intrinsic: 1.2.2 + object-inspect: 1.13.1 + dev: false + + /signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + dev: false + + /sirv@2.0.4: + resolution: {integrity: sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==} + engines: {node: '>= 10'} + dependencies: + '@polka/url': 1.0.0-next.24 + mrmime: 2.0.0 + totalist: 3.0.1 + dev: false + + /sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + dev: false + + /sitemap@7.1.1: + resolution: {integrity: sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==} + engines: {node: '>=12.0.0', npm: '>=5.6.0'} + hasBin: true + dependencies: + '@types/node': 17.0.45 + '@types/sax': 1.2.7 + arg: 5.0.2 + sax: 1.3.0 + dev: false + + /skin-tone@2.0.0: + resolution: {integrity: sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==} + engines: {node: '>=8'} + dependencies: + unicode-emoji-modifier-base: 1.0.0 + dev: false + + /slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + dev: false + + /slash@4.0.0: + resolution: {integrity: sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==} + engines: {node: '>=12'} + dev: false + + /sockjs@0.3.24: + resolution: {integrity: sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==} + dependencies: + faye-websocket: 0.11.4 + uuid: 8.3.2 + websocket-driver: 0.7.4 + dev: false + + /sort-css-media-queries@2.1.0: + resolution: {integrity: sha512-IeWvo8NkNiY2vVYdPa27MCQiR0MN0M80johAYFVxWWXQ44KU84WNxjslwBHmc/7ZL2ccwkM7/e6S5aiKZXm7jA==} + engines: {node: '>= 6.3.0'} + dev: false + + /source-map-js@1.0.2: + resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} + engines: {node: '>=0.10.0'} + dev: false + + /source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + dev: false + + /source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + dev: false + + /source-map@0.7.4: + resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} + engines: {node: '>= 8'} + dev: false + + /space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + dev: false + + /spdy-transport@3.0.0: + resolution: {integrity: sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==} + dependencies: + debug: 4.3.4 + detect-node: 2.1.0 + hpack.js: 2.1.6 + obuf: 1.1.2 + readable-stream: 3.6.2 + wbuf: 1.7.3 + transitivePeerDependencies: + - supports-color + dev: false + + /spdy@4.0.2: + resolution: {integrity: sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==} + engines: {node: '>=6.0.0'} + dependencies: + debug: 4.3.4 + handle-thing: 2.0.1 + http-deceiver: 1.2.7 + select-hose: 2.0.0 + spdy-transport: 3.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + dev: false + + /srcset@4.0.0: + resolution: {integrity: sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==} + engines: {node: '>=12'} + dev: false + + /stable@0.1.8: + resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==} + deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility' + dev: false + + /statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + dev: false + + /statuses@2.0.1: + resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} + engines: {node: '>= 0.8'} + dev: false + + /std-env@3.7.0: + resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==} + dev: false + + /string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + dev: false + + /string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + dev: false + + /string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + dependencies: + safe-buffer: 5.1.2 + dev: false + + /string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /stringify-entities@4.0.3: + resolution: {integrity: sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g==} + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + dev: false + + /stringify-object@3.3.0: + resolution: {integrity: sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==} + engines: {node: '>=4'} + dependencies: + get-own-enumerable-property-symbols: 3.0.2 + is-obj: 1.0.1 + is-regexp: 1.0.0 + dev: false + + /strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + dependencies: + ansi-regex: 5.0.1 + dev: false + + /strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: false + + /strip-bom-string@1.0.0: + resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} + engines: {node: '>=0.10.0'} + dev: false + + /strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + dev: false + + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: false + + /strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + dev: false + + /style-to-object@0.4.4: + resolution: {integrity: sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==} + dependencies: + inline-style-parser: 0.1.1 + dev: false + + /style-to-object@1.0.5: + resolution: {integrity: sha512-rDRwHtoDD3UMMrmZ6BzOW0naTjMsVZLIjsGleSKS/0Oz+cgCfAPRspaqJuE8rDzpKha/nEvnM0IF4seEAZUTKQ==} + dependencies: + inline-style-parser: 0.2.2 + dev: false + + /stylehacks@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /stylehacks@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-00zvJGnCu64EpMjX8b5iCZ3us2Ptyw8+toEkb92VdmkEaRaSGBNKAoK6aWZckhXxmQP8zWiTaFaiMGIU8Ve8sg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /stylis@4.3.1: + resolution: {integrity: sha512-EQepAV+wMsIaGVGX1RECzgrcqRRU/0sYOHkeLsZ3fzHaHXZy4DaOOX0vOlGQdlsjkh3mFHAIlVimpwAs4dslyQ==} + dev: false + + /supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + dependencies: + has-flag: 3.0.0 + dev: false + + /supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + dependencies: + has-flag: 4.0.0 + dev: false + + /supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + dependencies: + has-flag: 4.0.0 + dev: false + + /supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + dev: false + + /svg-parser@2.0.4: + resolution: {integrity: sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==} + dev: false + + /svgo@2.8.0: + resolution: {integrity: sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==} + engines: {node: '>=10.13.0'} + hasBin: true + dependencies: + '@trysound/sax': 0.2.0 + commander: 7.2.0 + css-select: 4.3.0 + css-tree: 1.1.3 + csso: 4.2.0 + picocolors: 1.0.0 + stable: 0.1.8 + dev: false + + /svgo@3.2.0: + resolution: {integrity: sha512-4PP6CMW/V7l/GmKRKzsLR8xxjdHTV4IMvhTnpuHwwBazSIlw5W/5SmPjN8Dwyt7lKbSJrRDgp4t9ph0HgChFBQ==} + engines: {node: '>=14.0.0'} + hasBin: true + dependencies: + '@trysound/sax': 0.2.0 + commander: 7.2.0 + css-select: 5.1.0 + css-tree: 2.3.1 + css-what: 6.1.0 + csso: 5.0.5 + picocolors: 1.0.0 + dev: false + + /tapable@1.1.3: + resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==} + engines: {node: '>=6'} + dev: false + + /tapable@2.2.1: + resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} + engines: {node: '>=6'} + dev: false + + /terser-webpack-plugin@5.3.10(webpack@5.89.0): + resolution: {integrity: sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==} + engines: {node: '>= 10.13.0'} + peerDependencies: + '@swc/core': '*' + esbuild: '*' + uglify-js: '*' + webpack: ^5.1.0 + peerDependenciesMeta: + '@swc/core': + optional: true + esbuild: + optional: true + uglify-js: + optional: true + dependencies: + '@jridgewell/trace-mapping': 0.3.20 + jest-worker: 27.5.1 + schema-utils: 3.3.0 + serialize-javascript: 6.0.1 + terser: 5.26.0 + webpack: 5.89.0 + dev: false + + /terser@5.26.0: + resolution: {integrity: sha512-dytTGoE2oHgbNV9nTzgBEPaqAWvcJNl66VZ0BkJqlvp71IjO8CxdBx/ykCNb47cLnCmCvRZ6ZR0tLkqvZCdVBQ==} + engines: {node: '>=10'} + hasBin: true + dependencies: + '@jridgewell/source-map': 0.3.5 + acorn: 8.11.3 + commander: 2.20.3 + source-map-support: 0.5.21 + dev: false + + /text-table@0.2.0: + resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} + dev: false + + /thunky@1.1.0: + resolution: {integrity: sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==} + dev: false + + /tiny-invariant@1.3.1: + resolution: {integrity: sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==} + dev: false + + /tiny-warning@1.0.3: + resolution: {integrity: sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==} + dev: false + + /to-fast-properties@2.0.0: + resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} + engines: {node: '>=4'} + dev: false + + /to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + dependencies: + is-number: 7.0.0 + dev: false + + /toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + dev: false + + /totalist@3.0.1: + resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} + engines: {node: '>=6'} + dev: false + + /trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + dev: false + + /trim@0.0.3: + resolution: {integrity: sha512-h82ywcYhHK7veeelXrCScdH7HkWfbIT1D/CgYO+nmDarz3SGNssVBMws6jU16Ga60AJCRAvPV6w6RLuNerQqjg==} + deprecated: Use String.prototype.trim() instead + dev: false + + /trough@2.1.0: + resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==} + dev: false + + /ts-dedent@2.2.0: + resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==} + engines: {node: '>=6.10'} + dev: false + + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: false + + /type-fest@1.4.0: + resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} + engines: {node: '>=10'} + dev: false + + /type-fest@2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + dev: false + + /type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + dev: false + + /typedarray-to-buffer@3.1.5: + resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} + dependencies: + is-typedarray: 1.0.0 + dev: false + + /typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + dev: false + + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: false + + /unicode-canonical-property-names-ecmascript@2.0.0: + resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} + engines: {node: '>=4'} + dev: false + + /unicode-emoji-modifier-base@1.0.0: + resolution: {integrity: sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==} + engines: {node: '>=4'} + dev: false + + /unicode-match-property-ecmascript@2.0.0: + resolution: {integrity: sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==} + engines: {node: '>=4'} + dependencies: + unicode-canonical-property-names-ecmascript: 2.0.0 + unicode-property-aliases-ecmascript: 2.1.0 + dev: false + + /unicode-match-property-value-ecmascript@2.1.0: + resolution: {integrity: sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==} + engines: {node: '>=4'} + dev: false + + /unicode-property-aliases-ecmascript@2.1.0: + resolution: {integrity: sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==} + engines: {node: '>=4'} + dev: false + + /unified@10.1.2: + resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + dependencies: + '@types/unist': 2.0.10 + bail: 2.0.2 + extend: 3.0.2 + is-buffer: 2.0.5 + is-plain-obj: 4.1.0 + trough: 2.1.0 + vfile: 5.3.7 + dev: false + + /unified@11.0.4: + resolution: {integrity: sha512-apMPnyLjAX+ty4OrNap7yumyVAMlKx5IWU2wlzzUdYJO9A8f1p9m/gywF/GM2ZDFcjQPrx59Mc90KwmxsoklxQ==} + dependencies: + '@types/unist': 3.0.2 + bail: 2.0.2 + devlop: 1.1.0 + extend: 3.0.2 + is-plain-obj: 4.1.0 + trough: 2.1.0 + vfile: 6.0.1 + dev: false + + /unique-string@3.0.0: + resolution: {integrity: sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==} + engines: {node: '>=12'} + dependencies: + crypto-random-string: 4.0.0 + dev: false + + /unist-builder@4.0.0: + resolution: {integrity: sha512-wmRFnH+BLpZnTKpc5L7O67Kac89s9HMrtELpnNaE6TAobq5DTZZs5YaTQfAZBA9bFPECx2uVAPO31c+GVug8mg==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-generated@2.0.1: + resolution: {integrity: sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==} + dev: false + + /unist-util-is@5.2.1: + resolution: {integrity: sha512-u9njyyfEh43npf1M+yGKDGVPbY/JWEemg5nH05ncKPfi+kBbKBJoTdsogMu33uhytuLlv9y0O7GH7fEdwLdLQw==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-is@6.0.0: + resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-position-from-estree@2.0.0: + resolution: {integrity: sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-position@4.0.4: + resolution: {integrity: sha512-kUBE91efOWfIVBo8xzh/uZQ7p9ffYRtUbMRZBNFYwf0RK8koUMx6dGUfwylLOKmaT2cs4wSW96QoYUSXAyEtpg==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-remove-position@5.0.0: + resolution: {integrity: sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==} + dependencies: + '@types/unist': 3.0.2 + unist-util-visit: 5.0.0 + dev: false + + /unist-util-select@5.1.0: + resolution: {integrity: sha512-4A5mfokSHG/rNQ4g7gSbdEs+H586xyd24sdJqF1IWamqrLHvYb+DH48fzxowyOhOfK7YSqX+XlCojAyuuyyT2A==} + dependencies: + '@types/unist': 3.0.2 + css-selector-parser: 3.0.4 + devlop: 1.1.0 + nth-check: 2.0.1 + zwitch: 2.0.4 + dev: false + + /unist-util-stringify-position@3.0.3: + resolution: {integrity: sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-visit-parents@5.1.3: + resolution: {integrity: sha512-x6+y8g7wWMyQhL1iZfhIPhDAs7Xwbn9nRosDXl7qoPTSCy0yNxnKc+hWokFifWQIDGi154rdUqKvbCa4+1kLhg==} + dependencies: + '@types/unist': 2.0.10 + unist-util-is: 5.2.1 + dev: false + + /unist-util-visit-parents@6.0.1: + resolution: {integrity: sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-is: 6.0.0 + dev: false + + /unist-util-visit@4.1.2: + resolution: {integrity: sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg==} + dependencies: + '@types/unist': 2.0.10 + unist-util-is: 5.2.1 + unist-util-visit-parents: 5.1.3 + dev: false + + /unist-util-visit@5.0.0: + resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} + dependencies: + '@types/unist': 3.0.2 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 + dev: false + + /universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} + engines: {node: '>= 10.0.0'} + dev: false + + /unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + dev: false + + /update-browserslist-db@1.0.13(browserslist@4.22.2): + resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + dependencies: + browserslist: 4.22.2 + escalade: 3.1.1 + picocolors: 1.0.0 + dev: false + + /update-notifier@6.0.2: + resolution: {integrity: sha512-EDxhTEVPZZRLWYcJ4ZXjGFN0oP7qYvbXWzEgRm/Yql4dHX5wDbvh89YHP6PK1lzZJYrMtXUuZZz8XGK+U6U1og==} + engines: {node: '>=14.16'} + dependencies: + boxen: 7.1.1 + chalk: 5.3.0 + configstore: 6.0.0 + has-yarn: 3.0.0 + import-lazy: 4.0.0 + is-ci: 3.0.1 + is-installed-globally: 0.4.0 + is-npm: 6.0.0 + is-yarn-global: 0.4.1 + latest-version: 7.0.0 + pupa: 3.1.0 + semver: 7.5.4 + semver-diff: 4.0.0 + xdg-basedir: 5.1.0 + dev: false + + /uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + dependencies: + punycode: 2.3.1 + dev: false + + /url-loader@4.1.1(file-loader@6.2.0)(webpack@5.89.0): + resolution: {integrity: sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==} + engines: {node: '>= 10.13.0'} + peerDependencies: + file-loader: '*' + webpack: ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + file-loader: + optional: true + dependencies: + file-loader: 6.2.0(webpack@5.89.0) + loader-utils: 2.0.4 + mime-types: 2.1.35 + schema-utils: 3.3.0 + webpack: 5.89.0 + dev: false + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /utila@0.4.0: + resolution: {integrity: sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==} + dev: false + + /utility-types@3.10.0: + resolution: {integrity: sha512-O11mqxmi7wMKCo6HKFt5AhO4BwY3VV68YU07tgxfz8zJTIxr4BpsezN49Ffwy9j3ZpwwJp4fkRwjRzq3uWE6Rg==} + engines: {node: '>= 4'} + dev: false + + /utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + dev: false + + /uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + + /uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + dev: false + + /uvu@0.5.6: + resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} + engines: {node: '>=8'} + hasBin: true + dependencies: + dequal: 2.0.3 + diff: 5.1.0 + kleur: 4.1.5 + sade: 1.8.1 + dev: false + + /value-equal@1.0.1: + resolution: {integrity: sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw==} + dev: false + + /vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + dev: false + + /vfile-location@5.0.2: + resolution: {integrity: sha512-NXPYyxyBSH7zB5U6+3uDdd6Nybz6o6/od9rk8bp9H8GR3L+cm/fC0uUTbqBmUTnMCUDslAGBOIKNfvvb+gGlDg==} + dependencies: + '@types/unist': 3.0.2 + vfile: 6.0.1 + dev: false + + /vfile-message@3.1.4: + resolution: {integrity: sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw==} + dependencies: + '@types/unist': 2.0.10 + unist-util-stringify-position: 3.0.3 + dev: false + + /vfile-message@4.0.2: + resolution: {integrity: sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-stringify-position: 4.0.0 + dev: false + + /vfile@5.3.7: + resolution: {integrity: sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g==} + dependencies: + '@types/unist': 2.0.10 + is-buffer: 2.0.5 + unist-util-stringify-position: 3.0.3 + vfile-message: 3.1.4 + dev: false + + /vfile@6.0.1: + resolution: {integrity: sha512-1bYqc7pt6NIADBJ98UiG0Bn/CHIVOoZ/IyEkqIruLg0mE1BKzkOXY2D6CSqQIcKqgadppE5lrxgWXJmXd7zZJw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-stringify-position: 4.0.0 + vfile-message: 4.0.2 + dev: false + + /watchpack@2.4.0: + resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} + engines: {node: '>=10.13.0'} + dependencies: + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + dev: false + + /wbuf@1.7.3: + resolution: {integrity: sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==} + dependencies: + minimalistic-assert: 1.0.1 + dev: false + + /web-namespaces@2.0.1: + resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + dev: false + + /web-streams-polyfill@3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false + + /web-worker@1.2.0: + resolution: {integrity: sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA==} + dev: false + + /webpack-bundle-analyzer@4.10.1: + resolution: {integrity: sha512-s3P7pgexgT/HTUSYgxJyn28A+99mmLq4HsJepMPzu0R8ImJc52QNqaFYW1Z2z2uIb1/J3eYgaAWVpaC+v/1aAQ==} + engines: {node: '>= 10.13.0'} + hasBin: true + dependencies: + '@discoveryjs/json-ext': 0.5.7 + acorn: 8.11.3 + acorn-walk: 8.3.1 + commander: 7.2.0 + debounce: 1.2.1 + escape-string-regexp: 4.0.0 + gzip-size: 6.0.0 + html-escaper: 2.0.2 + is-plain-object: 5.0.0 + opener: 1.5.2 + picocolors: 1.0.0 + sirv: 2.0.4 + ws: 7.5.9 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + + /webpack-dev-middleware@5.3.3(webpack@5.89.0): + resolution: {integrity: sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^4.0.0 || ^5.0.0 + dependencies: + colorette: 2.0.20 + memfs: 3.5.3 + mime-types: 2.1.35 + range-parser: 1.2.1 + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /webpack-dev-server@4.15.1(webpack@5.89.0): + resolution: {integrity: sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==} + engines: {node: '>= 12.13.0'} + hasBin: true + peerDependencies: + webpack: ^4.37.0 || ^5.0.0 + webpack-cli: '*' + peerDependenciesMeta: + webpack: + optional: true + webpack-cli: + optional: true + dependencies: + '@types/bonjour': 3.5.13 + '@types/connect-history-api-fallback': 1.5.4 + '@types/express': 4.17.21 + '@types/serve-index': 1.9.4 + '@types/serve-static': 1.15.5 + '@types/sockjs': 0.3.36 + '@types/ws': 8.5.10 + ansi-html-community: 0.0.8 + bonjour-service: 1.2.0 + chokidar: 3.5.3 + colorette: 2.0.20 + compression: 1.7.4 + connect-history-api-fallback: 2.0.0 + default-gateway: 6.0.3 + express: 4.18.2 + graceful-fs: 4.2.11 + html-entities: 2.4.0 + http-proxy-middleware: 2.0.6(@types/express@4.17.21) + ipaddr.js: 2.1.0 + launch-editor: 2.6.1 + open: 8.4.2 + p-retry: 4.6.2 + rimraf: 3.0.2 + schema-utils: 4.2.0 + selfsigned: 2.4.1 + serve-index: 1.9.1 + sockjs: 0.3.24 + spdy: 4.0.2 + webpack: 5.89.0 + webpack-dev-middleware: 5.3.3(webpack@5.89.0) + ws: 8.16.0 + transitivePeerDependencies: + - bufferutil + - debug + - supports-color + - utf-8-validate + dev: false + + /webpack-dev-server@4.9.2(webpack@5.89.0): + resolution: {integrity: sha512-H95Ns95dP24ZsEzO6G9iT+PNw4Q7ltll1GfJHV4fKphuHWgKFzGHWi4alTlTnpk1SPPk41X+l2RB7rLfIhnB9Q==} + engines: {node: '>= 12.13.0'} + hasBin: true + peerDependencies: + webpack: ^4.37.0 || ^5.0.0 + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + dependencies: + '@types/bonjour': 3.5.13 + '@types/connect-history-api-fallback': 1.5.4 + '@types/express': 4.17.21 + '@types/serve-index': 1.9.4 + '@types/serve-static': 1.15.5 + '@types/sockjs': 0.3.36 + '@types/ws': 8.5.10 + ansi-html-community: 0.0.8 + bonjour-service: 1.2.0 + chokidar: 3.5.3 + colorette: 2.0.20 + compression: 1.7.4 + connect-history-api-fallback: 1.6.0 + default-gateway: 6.0.3 + express: 4.18.2 + graceful-fs: 4.2.11 + html-entities: 2.4.0 + http-proxy-middleware: 2.0.6(@types/express@4.17.21) + ipaddr.js: 2.1.0 + open: 8.4.2 + p-retry: 4.6.2 + rimraf: 3.0.2 + schema-utils: 4.2.0 + selfsigned: 2.4.1 + serve-index: 1.9.1 + sockjs: 0.3.24 + spdy: 4.0.2 + webpack: 5.89.0 + webpack-dev-middleware: 5.3.3(webpack@5.89.0) + ws: 8.16.0 + transitivePeerDependencies: + - bufferutil + - debug + - supports-color + - utf-8-validate + dev: false + + /webpack-merge@5.10.0: + resolution: {integrity: sha512-+4zXKdx7UnO+1jaN4l2lHVD+mFvnlZQP/6ljaJVb4SZiwIKeUnrT5l0gkT8z+n4hKpC+jpOv6O9R+gLtag7pSA==} + engines: {node: '>=10.0.0'} + dependencies: + clone-deep: 4.0.1 + flat: 5.0.2 + wildcard: 2.0.1 + dev: false + + /webpack-sources@3.2.3: + resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} + engines: {node: '>=10.13.0'} + dev: false + + /webpack@5.89.0: + resolution: {integrity: sha512-qyfIC10pOr70V+jkmud8tMfajraGCZMBWJtrmuBymQKCrLTRejBI8STDp1MCyZu/QTdZSeacCQYpYNQVOzX5kw==} + engines: {node: '>=10.13.0'} + hasBin: true + peerDependencies: + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + dependencies: + '@types/eslint-scope': 3.7.7 + '@types/estree': 1.0.5 + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/wasm-edit': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + acorn: 8.11.3 + acorn-import-assertions: 1.9.0(acorn@8.11.3) + browserslist: 4.22.2 + chrome-trace-event: 1.0.3 + enhanced-resolve: 5.15.0 + es-module-lexer: 1.4.1 + eslint-scope: 5.1.1 + events: 3.3.0 + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + json-parse-even-better-errors: 2.3.1 + loader-runner: 4.3.0 + mime-types: 2.1.35 + neo-async: 2.6.2 + schema-utils: 3.3.0 + tapable: 2.2.1 + terser-webpack-plugin: 5.3.10(webpack@5.89.0) + watchpack: 2.4.0 + webpack-sources: 3.2.3 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + dev: false + + /webpackbar@5.0.2(webpack@5.89.0): + resolution: {integrity: sha512-BmFJo7veBDgQzfWXl/wwYXr/VFus0614qZ8i9znqcl9fnEdiVkdbi0TedLQ6xAK92HZHDJ0QmyQ0fmuZPAgCYQ==} + engines: {node: '>=12'} + peerDependencies: + webpack: 3 || 4 || 5 + dependencies: + chalk: 4.1.2 + consola: 2.15.3 + pretty-time: 1.1.0 + std-env: 3.7.0 + webpack: 5.89.0 + dev: false + + /websocket-driver@0.7.4: + resolution: {integrity: sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==} + engines: {node: '>=0.8.0'} + dependencies: + http-parser-js: 0.5.8 + safe-buffer: 5.2.1 + websocket-extensions: 0.1.4 + dev: false + + /websocket-extensions@0.1.4: + resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==} + engines: {node: '>=0.8.0'} + dev: false + + /which@1.3.1: + resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: false + + /which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: false + + /widest-line@4.0.1: + resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==} + engines: {node: '>=12'} + dependencies: + string-width: 5.1.2 + dev: false + + /wildcard@2.0.1: + resolution: {integrity: sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==} + dev: false + + /wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false + + /write-file-atomic@3.0.3: + resolution: {integrity: sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==} + dependencies: + imurmurhash: 0.1.4 + is-typedarray: 1.0.0 + signal-exit: 3.0.7 + typedarray-to-buffer: 3.1.5 + dev: false + + /ws@7.5.9: + resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==} + engines: {node: '>=8.3.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + + /ws@8.16.0: + resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + + /xdg-basedir@5.1.0: + resolution: {integrity: sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==} + engines: {node: '>=12'} + dev: false + + /xml-js@1.6.11: + resolution: {integrity: sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==} + hasBin: true + dependencies: + sax: 1.3.0 + dev: false + + /yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + dev: false + + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false + + /yaml-loader@0.8.0: + resolution: {integrity: sha512-LjeKnTzVBKWiQBeE2L9ssl6WprqaUIxCSNs5tle8PaDydgu3wVFXTbMfsvF2MSErpy9TDVa092n4q6adYwJaWg==} + engines: {node: '>= 12.13'} + dependencies: + javascript-stringify: 2.1.0 + loader-utils: 2.0.4 + yaml: 2.3.4 + dev: false + + /yaml@1.10.2: + resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} + engines: {node: '>= 6'} + dev: false + + /yaml@2.3.4: + resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} + engines: {node: '>= 14'} + dev: false + + /yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + dev: false + + /yocto-queue@1.0.0: + resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + engines: {node: '>=12.20'} + dev: false + + /zwitch@2.0.4: + resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + dev: false diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index 080a84f1dd4c..b5f2bf9e8afc 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -45,8 +45,6 @@ to: /enterprise-setup/ - from: /enterprise-setup/self-managed/implementation-guide to: /enterprise-setup/implementation-guide -- from: /enterprise-setup/self-managed/sso - to: /enterprise-setup/sso - from: - /project-overview/security - /operator-guides/securing-airbyte @@ -90,3 +88,11 @@ to: /using-airbyte/workspaces - from: /operator-guides/locating-files-local-destination to: /integrations/locating-files-local-destination +# End November 2023 restructure +- from: + - /enterprise-setup/self-managed/sso + - /enterprise-setup/sso + to: /access-management/sso +- from: + - /pyairbyte + to: /using-airbyte/airbyte-lib/getting-started diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index d6a5a7811bd3..138790ac2deb 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -225,7 +225,6 @@ const buildAConnector = { }, items: [ "connector-development/testing-connectors/connector-acceptance-tests-reference", - "connector-development/testing-connectors/testing-a-local-catalog-in-development", ], }, { @@ -273,7 +272,8 @@ const connectorCatalog = { type: "category", label: "Sources", link: { - type: "generated-index", + type: "doc", + id: "integrations/sources/README", }, items: [sourcePostgres, sourceMysql, ...getSourceConnectors()].sort((itemA, itemB) => itemA.label.localeCompare(itemB.label)), }, @@ -281,7 +281,8 @@ const connectorCatalog = { type: "category", label: "Destinations", link: { - type: "generated-index", + type: "doc", + id: "integrations/destinations/README", }, items: getDestinationConnectors(), }, @@ -311,6 +312,7 @@ const contributeToAirbyte = { "contributing-to-airbyte/resources/pull-requests-handbook", "contributing-to-airbyte/resources/code-style", "contributing-to-airbyte/resources/code-formatting", + "contributing-to-airbyte/resources/qa-checks", "contributing-to-airbyte/resources/developing-locally", "contributing-to-airbyte/resources/developing-on-docker", ], @@ -382,17 +384,18 @@ const understandingAirbyte = { type: "category", label: "Understand Airbyte", items: [ - "understanding-airbyte/beginners-guide-to-catalog", + "understanding-airbyte/high-level-view", "understanding-airbyte/airbyte-protocol", "understanding-airbyte/airbyte-protocol-docker", - "understanding-airbyte/operations", - "understanding-airbyte/high-level-view", "understanding-airbyte/jobs", - "understanding-airbyte/tech-stack", - "understanding-airbyte/cdc", + "understanding-airbyte/database-data-catalog", + "understanding-airbyte/beginners-guide-to-catalog", "understanding-airbyte/supported-data-types", + "understanding-airbyte/operations", + "understanding-airbyte/cdc", "understanding-airbyte/json-avro-conversion", - "understanding-airbyte/database-data-catalog", + "understanding-airbyte/schemaless-sources-and-destinations", + "understanding-airbyte/tech-stack", ], }; @@ -411,6 +414,7 @@ module.exports = { id: "using-airbyte/getting-started/readme", }, items: [ + "using-airbyte/core-concepts/readme", "using-airbyte/getting-started/add-a-source", "using-airbyte/getting-started/add-a-destination", "using-airbyte/getting-started/set-up-a-connection", @@ -418,10 +422,10 @@ module.exports = { }, { type: "category", - label: "Core Concepts", + label: "Configuring Connections", link: { type: "doc", - id: "using-airbyte/core-concepts/readme" + id: "cloud/managing-airbyte-cloud/configuring-connections" }, items: [ "using-airbyte/core-concepts/sync-schedules", @@ -440,21 +444,18 @@ module.exports = { "using-airbyte/core-concepts/sync-modes/full-refresh-overwrite", ], }, - "using-airbyte/core-concepts/typing-deduping", - "using-airbyte/core-concepts/basic-normalization", - ], - }, - { - type: "category", - label: "Configuring Connections", - link: { - type: "doc", - id: "cloud/managing-airbyte-cloud/configuring-connections" - }, - items: [ + { + type: "category", + label: "Typing and Deduping", + link: { + type: "doc", + id: "using-airbyte/core-concepts/typing-deduping" + }, + items: [ + "using-airbyte/core-concepts/basic-normalization" + ], + }, "cloud/managing-airbyte-cloud/manage-schema-changes", - "cloud/managing-airbyte-cloud/manage-data-residency", - "cloud/managing-airbyte-cloud/manage-connection-state", { type: "category", label: "Transformations", @@ -475,30 +476,33 @@ module.exports = { "cloud/managing-airbyte-cloud/review-sync-history", "operator-guides/browsing-output-logs", "operator-guides/reset", + "cloud/managing-airbyte-cloud/manage-connection-state", ], }, { type: "category", label: "Workspace Management", items: [ + "cloud/managing-airbyte-cloud/manage-data-residency", "using-airbyte/workspaces", "cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications", "cloud/managing-airbyte-cloud/manage-credits", "operator-guides/using-custom-connectors", + ] }, - sectionHeader("Operating Airbyte"), + sectionHeader("Managing Airbyte"), deployAirbyte, { type: "category", - label: "Airbyte Enterprise", + label: "Self-Managed Enterprise", link: { type: "doc", id: "enterprise-setup/README", }, items: [ "enterprise-setup/implementation-guide", - "enterprise-setup/sso", + "enterprise-setup/upgrading-from-community", ] }, "operator-guides/upgrading-airbyte", @@ -512,6 +516,24 @@ module.exports = { items: [ "operator-guides/configuring-airbyte-db", "operator-guides/configuring-connector-resources", + "operator-guides/telemetry", + ] + }, + { + type: "category", + label: "Access Management", + items: [ + { + type: "category", + label: "Single Sign-On (SSO)", + link: { + type: "doc", + id: "access-management/sso" + }, + items: [ + { type: "autogenerated", dirName: "access-management/sso-providers" }, + ] + }, ] }, { @@ -543,10 +565,6 @@ module.exports = { type: "doc", id: "terraform-documentation", }, - { - type: "doc", - id: "cli-documentation", - }, understandingAirbyte, contributeToAirbyte, { @@ -579,6 +597,9 @@ module.exports = { type: "generated-index", }, items: [ + "release_notes/january_2024", + "release_notes/december_2023", + "release_notes/november_2023", "release_notes/october_2023", "release_notes/upgrading_to_destinations_v2", "release_notes/september_2023", diff --git a/docusaurus/src/components/AirbyteLibConnectors.jsx b/docusaurus/src/components/AirbyteLibConnectors.jsx new file mode 100644 index 000000000000..2a546bda679b --- /dev/null +++ b/docusaurus/src/components/AirbyteLibConnectors.jsx @@ -0,0 +1,22 @@ +export default function AirbyteLibConnectors({ + connectorsJSON, + }) { + const connectors = JSON.parse(connectorsJSON); + return +} + +function getRelativeDocumentationUrl(connector) { + // get the relative path from the the dockerRepository_oss (e.g airbyte/source-amazon-sqs -> /integrations/sources/amazon-sqs) + + const fullDockerImage = connector.dockerRepository_oss; + console.log(fullDockerImage); + const dockerImage = fullDockerImage.split("airbyte/")[1]; + + const [integrationType, ...integrationName] = dockerImage.split("-"); + + return `/integrations/${integrationType}s/${integrationName.join("-")}`; +} \ No newline at end of file diff --git a/docusaurus/src/components/AirbyteLibDefinitions.jsx b/docusaurus/src/components/AirbyteLibDefinitions.jsx new file mode 100644 index 000000000000..e08462f8bfa4 --- /dev/null +++ b/docusaurus/src/components/AirbyteLibDefinitions.jsx @@ -0,0 +1,17 @@ +import React from 'react'; + +// Add additional modules here +import main_docs from "../../../airbyte-lib/docs/generated/airbyte_lib.html"; +import caches_docs from "../../../airbyte-lib/docs/generated/airbyte_lib/caches.html"; + +const docs = { + "airbyte_lib": main_docs, + "airbyte_lib.caches": caches_docs, +} + + +export default function AirbyteLibDefinitions({ module }) { + return <> +
      + +} diff --git a/docusaurus/src/components/AirbyteLibExample.jsx b/docusaurus/src/components/AirbyteLibExample.jsx new file mode 100644 index 000000000000..403c80d99270 --- /dev/null +++ b/docusaurus/src/components/AirbyteLibExample.jsx @@ -0,0 +1,52 @@ +import React, { useMemo } from "react"; +import { JSONSchemaFaker } from "json-schema-faker"; +import CodeBlock from '@theme/CodeBlock'; + +/** + * Generate a fake config based on the spec. + * + * As our specs are not 100% consistent, errors may occur. + * Try to generate a few times before giving up. + */ +function generateFakeConfig(spec) { + let tries = 5; + while (tries > 0) { + try { + return JSON.stringify(JSONSchemaFaker.generate(spec), null, 2) + } + catch (e) { + tries--; + } + } + return "{ ... }"; +} + +export const AirbyteLibExample = ({ + specJSON, + connector, +}) => { + const spec = useMemo(() => JSON.parse(specJSON), [specJSON]); + const fakeConfig = useMemo(() => generateFakeConfig(spec), [spec]); + return <> +

      + Install the Python library via: +

      + {"pip install airbyte-lib"} +

      Then, execute a sync by loading the connector like this:

      + {`import airbyte_lib as ab + +config = ${fakeConfig} + +result = ab.get_source( + "${connector}", + config=config, +).read() + +for record in result.cache.streams["my_stream:name"]: + print(record)`} +

      You can find more information in the airbyte_lib quickstart guide.

      + ; +}; diff --git a/docusaurus/src/components/Arcade.jsx b/docusaurus/src/components/Arcade.jsx new file mode 100644 index 000000000000..a2f3d2d646ef --- /dev/null +++ b/docusaurus/src/components/Arcade.jsx @@ -0,0 +1,7 @@ +export const Arcade = (props) => { + return ( +
      +